From 0e57a8062b9a8c0f1229ec1be80040d896ec107c Mon Sep 17 00:00:00 2001 From: orbiter Date: Fri, 22 Jun 2007 14:29:14 +0000 Subject: [PATCH] added network definition for different YaCy networks (needs much more work) git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3919 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- doc/index.html | 38 +++++----- htroot/Network.html | 10 +-- htroot/Network.java | 2 + htroot/NetworkPicture.java | 3 +- htroot/Status.java | 4 -- source/de/anomic/http/httpc.java | 27 +++++++ source/de/anomic/plasma/plasmaCrawlLURL.java | 1 + source/de/anomic/plasma/plasmaGrafics.java | 29 ++++---- .../de/anomic/plasma/plasmaSwitchboard.java | 46 ++++++++++-- .../anomic/server/serverAbstractSwitch.java | 14 +++- source/de/anomic/server/serverSwitch.java | 1 + source/de/anomic/yacy/yacyClient.java | 28 +++++--- source/de/anomic/yacy/yacyCore.java | 4 +- source/yacy.java | 3 +- yacy.init | 47 +++--------- yacy.network.group | 29 ++++++++ yacy.network.unit | 72 +++++++++++++++++++ 17 files changed, 255 insertions(+), 103 deletions(-) create mode 100644 yacy.network.group create mode 100644 yacy.network.unit diff --git a/doc/index.html b/doc/index.html index 422d75594..8a40d6c2f 100644 --- a/doc/index.html +++ b/doc/index.html @@ -100,32 +100,36 @@ globalheader();   - -YaCy is a peer-to-peer application
for web search.

- - - + +

Peer-to-Peer Web Search Engine

+Anonymous and uncensored search in a distributed and community-driven network of search-engine peers. +


+ +

Personal Search Appliance

+Create your own search portal with up to 10 million pages in your private web index.   - -YaCy can be used as your own,
private search engine

- + + + + + +
We are supported by
SuMa-eV
Gemeinnütziger Verein zur Förderung der Suchmaschinen-Technologie und des freien Wissenszugangs


+ + + + +
We collaborate with
Karlsruhe Institute of Technology
Liebel-Lab at Forschungszentrum Karlsruhe

+ +  

YaCy is GPL licensed, free software. -

Für eine deutsche Dokumentation sehen sie bitte hier

+

Für eine deutsche Dokumentation sehen sie bitte hier

diff --git a/htroot/Network.html b/htroot/Network.html index 86e34b7a2..1c82147d7 100644 --- a/htroot/Network.html +++ b/htroot/Network.html @@ -14,20 +14,20 @@

YaCy Network

#(page)# -

Network Overview

+

Overview of '#[networkName]#' Network

:: -

Active Peers

+

Active Peers in '#[networkName]#' Network

:: -

Passive Peers

+

Passive Peers in '#[networkName]#' Network

:: -

Potential Peers

+

Potential Peers in '#[networkName]#' Network

::

Manually contacting Peer

#(/page)# diff --git a/htroot/Network.java b/htroot/Network.java index d7cfab1aa..827fa1c68 100644 --- a/htroot/Network.java +++ b/htroot/Network.java @@ -78,6 +78,8 @@ public class Network { // return variable that accumulates replacements final serverObjects prop = new serverObjects(); + prop.put("page_networkTitle", sb.getConfig("network.unit.description", "unspecified")); + prop.put("page_networkName", sb.getConfig("network.unit.name", "unspecified")); final boolean overview = (post == null) || (post.get("page", "0").equals("0")); final String mySeedType = yacyCore.seedDB.mySeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN); diff --git a/htroot/NetworkPicture.java b/htroot/NetworkPicture.java index 01df35e05..4570f5b79 100644 --- a/htroot/NetworkPicture.java +++ b/htroot/NetworkPicture.java @@ -43,7 +43,6 @@ // the intact and unchanged copyright notice. // Contributions and changes to the program code must be marked as such. - import de.anomic.http.httpHeader; import de.anomic.plasma.plasmaGrafics; import de.anomic.server.serverObjects; @@ -79,7 +78,7 @@ public class NetworkPicture { if (passiveLimit > 1000000) passiveLimit = 1000000; if (potentialLimit > 1000000) potentialLimit = 1000000; if (maxCount > 1000) maxCount = 1000; - return plasmaGrafics.getNetworkPicture(10000, width, height, passiveLimit, potentialLimit, maxCount, corona); + return plasmaGrafics.getNetworkPicture(10000, width, height, passiveLimit, potentialLimit, maxCount, corona, env.getConfig("network.unit.name", "unspecified"), env.getConfig("network.unit.description", "unspecified")); } } diff --git a/htroot/Status.java b/htroot/Status.java index 347708571..030a69dc3 100644 --- a/htroot/Status.java +++ b/htroot/Status.java @@ -119,10 +119,6 @@ public class Status { } } - /* - versionProbe=http://www.anomic.de/AnomicHTTPProxy/release.txt - superseedFile=superseed.txt - */ // update seed info yacyCore.peerActions.updateMySeed(); diff --git a/source/de/anomic/http/httpc.java b/source/de/anomic/http/httpc.java index f509e557d..3f3e0159d 100644 --- a/source/de/anomic/http/httpc.java +++ b/source/de/anomic/http/httpc.java @@ -1372,6 +1372,33 @@ do upload // return result return a; } + + public static Map loadHashMap(URL url, httpRemoteProxyConfig proxy) { + try { + // should we use the proxy? + boolean useProxy = (proxy != null) && + (proxy.useProxy()) && + (proxy.useProxy4Yacy()); + + // sending request + final HashMap result = nxTools.table( + httpc.wget( + url, + url.getHost(), + 8000, + null, + null, + (useProxy) ? proxy : null, + null + ) + , "UTF-8"); + + if (result == null) return new HashMap(); + return result; + } catch (Exception e) { + return new HashMap(); + } + } public static httpHeader whead( URL url, diff --git a/source/de/anomic/plasma/plasmaCrawlLURL.java b/source/de/anomic/plasma/plasmaCrawlLURL.java index bcb25edae..cdb36a851 100644 --- a/source/de/anomic/plasma/plasmaCrawlLURL.java +++ b/source/de/anomic/plasma/plasmaCrawlLURL.java @@ -187,6 +187,7 @@ public final class plasmaCrawlLURL { oldEntry = null; } } catch (Exception e) { + e.printStackTrace(); oldEntry = null; } if ((oldEntry != null) && (entry.isOlder(oldEntry))) { diff --git a/source/de/anomic/plasma/plasmaGrafics.java b/source/de/anomic/plasma/plasmaGrafics.java index d419a427d..40401d7bd 100644 --- a/source/de/anomic/plasma/plasmaGrafics.java +++ b/source/de/anomic/plasma/plasmaGrafics.java @@ -127,7 +127,7 @@ public class plasmaGrafics { if (primarySearches == null) return null; // this was a local search and there are no threads // get a copy of a recent network picture - ymageMatrix eventPicture = getNetworkPicture(120000); + ymageMatrix eventPicture = getNetworkPicture(120000, plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", "unspecified"), plasmaSwitchboard.getSwitchboard().getConfig("network.unit.description", "unspecified")); if (eventPicture instanceof ymageMatrix) eventPicture = (ymageMatrix) eventPicture; //new ymageMatrix((ymageMatrix) eventPicture); // TODO: fix cloning of ymageMatrix pictures @@ -172,18 +172,18 @@ public class plasmaGrafics { return eventPicture; } - public static ymageMatrix getNetworkPicture(long maxAge) { - return getNetworkPicture(maxAge, 640, 480, 300, 300, 1000, true); + public static ymageMatrix getNetworkPicture(long maxAge, String networkName, String networkTitle) { + return getNetworkPicture(maxAge, 640, 480, 300, 300, 1000, true, networkName, networkTitle); } - public static ymageMatrix getNetworkPicture(long maxAge, int width, int height, int passiveLimit, int potentialLimit, int maxCount, boolean corona) { + public static ymageMatrix getNetworkPicture(long maxAge, int width, int height, int passiveLimit, int potentialLimit, int maxCount, boolean corona, String networkName, String networkTitle) { if ((networkPicture == null) || ((System.currentTimeMillis() - networkPictureDate) > maxAge)) { - drawNetworkPicture(width, height, passiveLimit, potentialLimit, maxCount, corona); + drawNetworkPicture(width, height, passiveLimit, potentialLimit, maxCount, corona, networkName, networkTitle); } return networkPicture; } - private static void drawNetworkPicture(int width, int height, int passiveLimit, int potentialLimit, int maxCount, boolean corona) { + private static void drawNetworkPicture(int width, int height, int passiveLimit, int potentialLimit, int maxCount, boolean corona, String networkName, String networkTitle) { int innerradius = Math.min(width, height) / 5; int outerradius = innerradius + innerradius * yacyCore.seedDB.sizeConnected() / 100; @@ -196,7 +196,7 @@ public class plasmaGrafics { // draw network circle networkPicture.setColor(COL_DHTCIRCLE); - networkPicture.arc(width / 2, height / 2, innerradius - 20, innerradius + 20, 0, 360); + networkPicture.arc(width / 2, height / 2 + 20, innerradius - 20, innerradius + 20, 0, 360); //System.out.println("Seed Maximum distance is " + yacySeed.maxDHTDistance); //System.out.println("Seed Minimum distance is " + yacySeed.minDHTNumber); @@ -212,7 +212,7 @@ public class plasmaGrafics { while (e.hasMoreElements() && count < maxCount) { seed = (yacySeed) e.nextElement(); if (seed != null) { - drawNetworkPicturePeer(networkPicture, width / 2, height / 2, innerradius, outerradius, seed, COL_ACTIVE_DOT, COL_ACTIVE_LINE, COL_ACTIVE_TEXT, corona); + drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seed, COL_ACTIVE_DOT, COL_ACTIVE_LINE, COL_ACTIVE_TEXT, corona); count++; } } @@ -226,7 +226,7 @@ public class plasmaGrafics { if (seed != null) { lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenUTC()) / 1000 / 60); if (lastseen > passiveLimit) break; // we have enough, this list is sorted so we don't miss anything - drawNetworkPicturePeer(networkPicture, width / 2, height / 2, innerradius, outerradius, seed, COL_PASSIVE_DOT, COL_PASSIVE_LINE, COL_PASSIVE_TEXT, corona); + drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seed, COL_PASSIVE_DOT, COL_PASSIVE_LINE, COL_PASSIVE_TEXT, corona); count++; } } @@ -240,22 +240,23 @@ public class plasmaGrafics { if (seed != null) { lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenUTC()) / 1000 / 60); if (lastseen > potentialLimit) break; // we have enough, this list is sorted so we don't miss anything - drawNetworkPicturePeer(networkPicture, width / 2, height / 2, innerradius, outerradius, seed, COL_POTENTIAL_DOT, COL_POTENTIAL_LINE, COL_POTENTIAL_TEXT, corona); + drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seed, COL_POTENTIAL_DOT, COL_POTENTIAL_LINE, COL_POTENTIAL_TEXT, corona); count++; } } totalCount += count; // draw my own peer - drawNetworkPicturePeer(networkPicture, width / 2, height / 2, innerradius, outerradius, yacyCore.seedDB.mySeed, COL_WE_DOT, COL_WE_LINE, COL_WE_TEXT, corona); + drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, yacyCore.seedDB.mySeed, COL_WE_DOT, COL_WE_LINE, COL_WE_TEXT, corona); // draw description networkPicture.setColor(COL_HEADLINE); networkPicture.setMode(ymageMatrix.MODE_SUB); - ymageToolPrint.print(networkPicture, 2, 8, 0, "THE YACY NETWORK", -1); - ymageToolPrint.print(networkPicture, 2, 16, 0, "DRAWING OF " + totalCount + " SELECTED PEERS", -1); + ymageToolPrint.print(networkPicture, 2, 8, 0, "YACY NETWORK '" + networkName.toUpperCase() + "'", -1); + ymageToolPrint.print(networkPicture, 2, 16, 0, networkTitle.toUpperCase(), -1); ymageToolPrint.print(networkPicture, width - 2, 8, 0, "SNAPSHOT FROM " + new Date().toString().toUpperCase(), 1); - + ymageToolPrint.print(networkPicture, width - 2, 16, 0, "DRAWING OF " + totalCount + " SELECTED PEERS", 1); + // set timestamp networkPictureDate = System.currentTimeMillis(); } diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 0217cf71f..86c4b7814 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -893,9 +893,47 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // set loglevel and log setLog(new serverLog("PLASMA")); - if (applyPro) this.log.logInfo("This is the pro-version of YaCy"); + // remote proxy configuration + this.remoteProxyConfig = httpRemoteProxyConfig.init(this); + this.log.logConfig("Remote proxy configuration:\n" + this.remoteProxyConfig.toString()); + + // load network configuration into settings + String networkUnitDefinition = getConfig("network.unit.definition", "yacy.network.unit"); + String networkGroupDefinition = getConfig("network.group.definition", "yacy.network.group"); + + // include additional network definition properties into our settings + // note that these properties cannot be set in the application because they are + // _always_ overwritten each time with the default values. This is done so on purpose. + // the network definition should be made either consistent for all peers, + // or independently using a bootstrap URL + Map initProps; + if (networkUnitDefinition.startsWith("http://")) { + try { + this.setConfig(httpc.loadHashMap(new URL(networkUnitDefinition), remoteProxyConfig)); + } catch (MalformedURLException e) { + } + } else { + File networkUnitDefinitionFile = new File(rootPath, networkUnitDefinition); + if (networkUnitDefinitionFile.exists()) { + initProps = serverFileUtils.loadHashMap(networkUnitDefinitionFile); + this.setConfig(initProps); + } + } + if (networkGroupDefinition.startsWith("http://")) { + try { + this.setConfig(httpc.loadHashMap(new URL(networkGroupDefinition), remoteProxyConfig)); + } catch (MalformedURLException e) { + } + } else { + File networkGroupDefinitionFile = new File(rootPath, networkGroupDefinition); + if (networkGroupDefinitionFile.exists()) { + initProps = serverFileUtils.loadHashMap(networkGroupDefinitionFile); + this.setConfig(initProps); + } + } + // load values from configs this.plasmaPath = new File(rootPath, getConfig(DBPATH, DBPATH_DEFAULT)); this.log.logConfig("Plasma DB Path: " + this.plasmaPath.toString()); @@ -913,12 +951,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser this.workPath = new File(rootPath, getConfig(WORK_PATH, WORK_PATH_DEFAULT)); this.log.logConfig("Work Path: " + this.workPath.toString()); - /* ============================================================================ - * Remote Proxy configuration - * ============================================================================ */ - this.remoteProxyConfig = httpRemoteProxyConfig.init(this); - this.log.logConfig("Remote proxy configuration:\n" + this.remoteProxyConfig.toString()); - // set up local robots.txt this.robotstxtConfig = httpdRobotsTxtConfig.init(this); diff --git a/source/de/anomic/server/serverAbstractSwitch.java b/source/de/anomic/server/serverAbstractSwitch.java index 030e7cbd7..8b7564798 100644 --- a/source/de/anomic/server/serverAbstractSwitch.java +++ b/source/de/anomic/server/serverAbstractSwitch.java @@ -84,7 +84,6 @@ public abstract class serverAbstractSwitch implements serverSwitch { // predefine init's Map initProps; - configRemoved = new HashMap(); if (initFile.exists()) initProps = serverFileUtils.loadHashMap(initFile); else @@ -117,9 +116,9 @@ public abstract class serverAbstractSwitch implements serverSwitch { else configProps = new HashMap(); + // remove all values from config that do not appear in init + configRemoved = new HashMap(); synchronized (configProps) { - // remove all values from config that do not appear in init - // (out-dated settings) i = configProps.keySet().iterator(); String key; while (i.hasNext()) { @@ -212,6 +211,15 @@ public abstract class serverAbstractSwitch implements serverSwitch { return accessTracker.keySet().iterator(); } + public void setConfig(Map otherConfigs) { + Iterator i = otherConfigs.entrySet().iterator(); + Map.Entry entry; + while (i.hasNext()) { + entry = (Map.Entry) i.next(); + setConfig((String) entry.getKey(), (String) entry.getValue()); + } + } + public void setConfig(String key, boolean value) { setConfig(key, (value) ? "true" : "false"); } diff --git a/source/de/anomic/server/serverSwitch.java b/source/de/anomic/server/serverSwitch.java index f1573e157..9818e5123 100644 --- a/source/de/anomic/server/serverSwitch.java +++ b/source/de/anomic/server/serverSwitch.java @@ -96,6 +96,7 @@ public interface serverSwitch { public Iterator /*of serverThread-Names (String)*/ threadNames(); // the switchboard can be used to set and read properties + public void setConfig(Map otherConfigs); public void setConfig(String key, long value); public void setConfig(String key, String value); public String getConfig(String key, String dflt); diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index b68034b0a..1f3d62106 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -259,9 +259,9 @@ public final class yacyClient { final String key = crypt.randomSalt(); try { // should we use the proxy? - boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) && - (yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) && - (yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy()); + boolean useProxy = (plasmaSwitchboard.getSwitchboard().remoteProxyConfig != null) && + (plasmaSwitchboard.getSwitchboard().remoteProxyConfig.useProxy()) && + (plasmaSwitchboard.getSwitchboard().remoteProxyConfig.useProxy4Yacy()); // sending request final HashMap result = nxTools.table( @@ -278,7 +278,7 @@ public final class yacyClient { 8000, null, null, - (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null, + (useProxy)? plasmaSwitchboard.getSwitchboard().remoteProxyConfig:null, null ) , "UTF-8"); @@ -526,18 +526,28 @@ public final class yacyClient { assert (urlEntry.hash().length() == 12) : "urlEntry.hash() = " + urlEntry.hash(); if (urlEntry.hash().length() != 12) continue; // bad url hash indexURLEntry.Components comp = urlEntry.comp(); - if (blacklist.isListed(plasmaURLPattern.BLACKLIST_SEARCH, comp.url())) continue; // block with backlist - urlManager.store(urlEntry); - urlManager.stack(urlEntry, yacyCore.seedDB.mySeed.hash, targetPeer.hash, 2); - + if (blacklist.isListed(plasmaURLPattern.BLACKLIST_SEARCH, comp.url())) { + yacyCore.log.logInfo("remote search (client): filtered blacklisted url " + comp.url() + " from peer " + targetPeer.getName()); + continue; // block with backlist + } + // save the url entry indexRWIEntry entry; if (urlEntry.word() == null) { - yacyCore.log.logWarning("DEBUG-SEARCH: no word attached from peer " + targetPeer.getName() + ", version " + targetPeer.getVersion()); + yacyCore.log.logWarning("remote search (client): no word attached from peer " + targetPeer.getName() + ", version " + targetPeer.getVersion()); continue; // no word attached } + // the search-result-url transports all the attributes of word indexes entry = urlEntry.word(); + if (!(entry.urlHash().equals(urlEntry.hash()))) { + yacyCore.log.logInfo("remote search (client): url-hash " + urlEntry.hash() + " does not belong to word-attached-hash " + entry.urlHash() + "; url = " + comp.url() + " from peer " + targetPeer.getName()); + continue; // spammed + } + + // passed all checks, store url + urlManager.store(urlEntry); + urlManager.stack(urlEntry, yacyCore.seedDB.mySeed.hash, targetPeer.hash, 2); if (urlEntry.snippet() != null) { // we don't store the snippets along the url entry, because they are search-specific. diff --git a/source/de/anomic/yacy/yacyCore.java b/source/de/anomic/yacy/yacyCore.java index e78dbaaeb..2e50bc85a 100644 --- a/source/de/anomic/yacy/yacyCore.java +++ b/source/de/anomic/yacy/yacyCore.java @@ -215,8 +215,8 @@ public class yacyCore { // deploy peer actions peerActions = new yacyPeerActions(seedDB, switchboard, - new File(sb.getRootPath(), sb.getConfig("superseedFile", "superseed.txt")), - switchboard.getConfig("superseedLocation", "http://www.yacy.net/yacy/superseed.txt")); + new File(sb.getRootPath(), sb.getConfig("network.unit.bootstrap.file", "superseed.txt")), + switchboard.getConfig("network.unit.bootstrap.url", "http://www.yacy.net/yacy/superseed.txt")); dhtAgent = new yacyDHTAction(seedDB); peerActions.deploy(dhtAgent); peerActions.deploy(new yacyNewsAction(newsPool)); diff --git a/source/yacy.java b/source/yacy.java index 79f7c24fb..69bfd8b4e 100644 --- a/source/yacy.java +++ b/source/yacy.java @@ -253,8 +253,7 @@ public final class yacy { f = new File(homePath, "DATA/yacy.running"); if (f.exists()) { // another instance running? VM crash? User will have to care about this - serverLog.logSevere("STARTUP", "the file " + f + " exists, this usually means that another instance of YaCy is using this DATA-folder."); - serverLog.logSevere("STARTUP", "please make sure that DATA can be used exclusively by one YaCy. Quitting..."); + serverLog.logSevere("STARTUP", "WARNING: the file " + f + " exists, this usually means that a YaCy instance is still running"); f.delete(); } f.createNewFile(); diff --git a/yacy.init b/yacy.init index 6a88b0806..59ee10e40 100644 --- a/yacy.init +++ b/yacy.init @@ -64,43 +64,14 @@ pkcs12ImportPwd = # value is in milliseconds, default is one hour server.maxTrackingTime = 3600000 -# peer-to-peer construction for distributed search -# we have several stages: -# 1st: a file within every distribution that has a list of URLs: -# -> this is the superseed file -# 2nd: the files that can be retrieved by the superseeds' URLs -# are called seed list-files. -# -> the seed list-files contain IP/port combinations of running -# AnomicHTTPProxies -# 3rd: the peers that are targeted within the seed files are called superpeers -# 4th: the superpeers hold and share a list of all client/search/crawl peers -# -# some superpeers should be able to create again seed list-files. -# These superpeers must upload their IP or their list of peer-IP's to a -# ftp location to create the seed list-file. -# Everyone who do so should mail his/her new seed location to mcanomic.de -# The seed list-file location will then be included in the superseed file. -# This superseed file is available then at two localtions: -# - it is included in every distribution and -# - updated through a specific URL-location -# we see the file name and the URL of the superseed here: -superseedFile=superseed.txt -superseedLocation=http://www.yacy.net/superseed.txt - -# network definition -# we distiguish local and global networks. Each network type can have different user groups -# groups can be uncontrolled, moderated or controlled -# Each group has a group master, but in case the group is uncontrolled the only task of the -# group master is to provide a bootstrap file which contains peer addresses of peers that -# are member of the group. All attributes are given in a single network description string -# of the form: -# network = :::{'uncontrolled'|'moderated'|'controlled'}: -# in case that the network is the uncontrolled global dht-managed community for everybody, -# the details defaults to: -# network = all:world:global:uncontrolled:http://yacy.net/ -# the network-uri must have a sub-path yacy/seed.txt containing a list of urls pointing to the -# peer-address of peers within the group of that network -# several network definition strings can be listed +# Network Definition +# There can be separate YaCy networks, and managed sub-groups of the general network. +# The essentials of the network definition are attached in separate property files. +# The property here can also be a url where the definition can be loaded. +# In case of privately managed networks, this configuration must be changed BEFORE it is released +# to the members of the separated network peers. +network.unit.definition = yacy.network.unit +network.group.definition = yacy.network.group # clusters within a network: # every network can have an unlimited number of clusters. Clusters may be also completely @@ -716,7 +687,7 @@ useYacyReferer = true useYacyReferer__pro = false # allow only 443(https-port) for https-proxy? -# if you want to tunnel other protokols, set to false +# if you want to tunnel other protocols, set to false secureHttps = true # specifies if the httpdFileHandler should cache diff --git a/yacy.network.group b/yacy.network.group new file mode 100644 index 000000000..2bf87d354 --- /dev/null +++ b/yacy.network.group @@ -0,0 +1,29 @@ +# YaCy Network Group Definition +# ----------------------------- +# This is an addition to the yacy.network.unit configuration file. +# This file is adressed by the network.group.definition property in yacy.init +# The purpose of a group within a network is that some parts of a network may be managed independently, +# while the content of the network stays private for the whole network, mostly for a special purpose. +# This file needs to be configured if someone wants to participate with several peers to the network, +# but wants to manage the group of own peers with a single master peer. +# One example application would be the use of computing-teams + +# Defintion of property domains: +# network.group.name = +# network.group.description = +# network.group.administration.control = 'uncontrolled'|'moderated'|'controlled' +# network.group.administration.manager = +# network.group.administration.request.authentification.method = 'salted-magic' +# network.group.administration.request.authentification.essentials = + +# -----------------------------------------------------------------# +# this is a work in progress. disabled properties are not yet used # +# -----------------------------------------------------------------# + +# properties for group access administrator-authentification: +#network.group.name = everybody +#network.group.description = Our Peer-Group +#network.group.administration.control = uncontrolled +#network.group.administration.manager = +#network.group.administration.request.authentification.method = salted-magic +#network.group.administration.request.authentification.essentials = \ No newline at end of file diff --git a/yacy.network.unit b/yacy.network.unit new file mode 100644 index 000000000..a877f1903 --- /dev/null +++ b/yacy.network.unit @@ -0,0 +1,72 @@ +# YaCy Network Unit Definition +# ---------------------------- +# There is not only a single global YaCy network, but there can be also separated networks +# This file describes the properties of the network, how it is bootstraped, if it is moderated +# and so on. It is also possible to describe authentification methods for in-protocol communication +# To distinguish between index areas where the shared web index is for a intranet or the public internet, +# we distinguish local and global networks. Each network type can have different user groups. +# Groups can be uncontrolled, moderated or controlled. The whole network may also have a global master. +# This file is adressed by the network.unit.definition property in yacy.init. If necessary this +# property can also address a URL where a network definition file can be hosted, so that a network +# can easily be moderated from a single central. In case of the public network of all independent peers +# for global indexing, the network definition is attached to the release and defines no network master or +# any other supervision. + +# peer-to-peer construction using the bootstrap properties: +# 1st: a file within every distribution that has a list of URLs, +# or a location with a file containing a list of URLs: +# -> these are the superseed files (there are only two of these files) +# 2nd: the files that can be retrieved by the superseeds' URLs +# are called seed list-files. (there can be many of those) +# -> the seed list-files contain b.o. IP/port combinations of running YaCy peers +# -> seed-list files are written by principal peers + +# Control Modes for Peer Groups: +# we have three different types of moderation: +# uncontrolled: only the peer's administrator may use protected pages of the YaCy interface +# moderated : the unit/group moderator may access some protected pages i.e. Crawl Start +# controlled : the unit/group moderator may access all pages of the peers within the same unit/group + +# Defintion of property domains: +# network.unit.name = +# network.unit.description = +# network.unit.domain = 'global'|'local' +# network.unit.dhtredundancy = +# network.unit.bootstrap.file = +# network.unit.bootstrap.url = +# network.unit.protocol.control = 'uncontrolled'|'moderated'|'controlled' +# network.unit.protocol.request.authentification.method = 'salted-magic' +# network.unit.protocol.request.authentification.essentials = +# network.unit.protocol.response.authentification.method = 'challenge-pwresponse' +# network.unit.protocol.response.authentification.essentials = ':' +# network.unit.administration.control = 'uncontrolled'|'moderated'|'controlled' +# network.unit.administration.manager = +# network.unit.administration.request.authentification.method = 'salted-magic' +# network.unit.administration.request.authentification.essentials = + + +# -----------------------------------------------------------------# +# this is a work in progress. disabled properties are not yet used # +# -----------------------------------------------------------------# + +# general network definition +network.unit.name = freeworld +network.unit.description = Public YaCy Community +#network.unit.domain = global +network.unit.dhtredundancy.junior = 1 +network.unit.dhtredundancy.senior = 3 +network.unit.bootstrap.file = superseed.txt +network.unit.bootstrap.url = http://www.yacy.net/superseed.txt + +# properties for in-protocol response authentification: +#network.unit.protocol.control = uncontrolled +#network.unit.protocol.request.authentification.method = salted-magic +#network.unit.protocol.request.authentification.essentials = +#network.unit.protocol.response.authentification.method = challenge-pwresponse +#network.unit.protocol.response.authentification.essentials = + +# properties for unit access administrator-authentification: +#network.unit.administration.control = uncontrolled +#network.unit.administration.manager = +#network.unit.administration.request.authentification.method = salted-magic +#network.unit.administration.request.authentification.essentials =