added network definition for different YaCy networks

(needs much more work)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3919 6c8d7289-2bf4-0310-a012-ef5d649a1542

doc/index.html

@@ -101,31 +101,35 @@ globalheader();
 <td width="20">&nbsp;</td>
 
 <td width="50%">
-<b>YaCy is a peer-to-peer application<br>for web search.</b><br><br>
-<ul>
-<li>This is your web search engine for freedom of information:
-search requests are anonymous, independend and uncensored</li>
-<li>No central server, no storage of user behaviour</li>
-</ul>
-</td>
+<h3>Peer-to-Peer Web Search Engine</h3>
+Anonymous and uncensored search in a distributed and community-driven network of search-engine peers.
+<br><br><br>
 
+<h3>Personal Search Appliance</h3>
+Create your own search portal with up to 10 million pages in your private web index.
 <td width="40">&nbsp;</td>
 
-<td width="50%">
-<b>YaCy can be used as your own,<br>private search engine</b><br><br>
-<ul>
-<li>Set up a search portal</li>
-<li>Use YaCy to provide a search function for your own web pages</li>
-<li>Host web pages with the built-in http-server</li>
-</ul>
+<td width="20%">
+<table>
+<tr><td>We are supported by</td></tr>
+<tr><td><a href="http://suma-ev.de/"><img src="http://suma-ev.de/images/suma-logo-small.jpg" border="0" alt="SuMa-eV"></a></td></tr>
+<tr><td><a href="http://suma-ev.de/">Gemeinn&uuml;tziger Verein zur F&ouml;rderung der Suchmaschinen-Technologie und des freien Wissenszugangs</a></td></tr>
+</table><br><br>
+<table>
+<tr><td>We collaborate with</td></tr>
+<tr><td><a href="http://liebel.fzk.de/"><img src="http://www.kit.edu/fzk/resources/kit/img/kit_logo_large.gif" border="0" alt="Karlsruhe Institute of Technology"></a></td></tr>
+<tr><td><a href="http://liebel.fzk.de/">Liebel-Lab at Forschungszentrum Karlsruhe</a></td></tr>
+</table><br>
 </td>
 
+
+
 <td width="20">&nbsp;</td>
 </tr></table>
 
 <p>YaCy is GPL licensed, free software.
 
-<p><i>F&uuml;r eine deutsche Dokumentation sehen sie bitte <a href="http://www.yacy-websuche.de/">hier</a></p>
+<p><i>F&uuml;r eine deutsche Dokumentation sehen sie bitte <a href="http://yacy-websuche.de/wiki/index.php/De:Start">hier</a></p>
 
 
 <!-- ----- HERE ENDS CONTENT PART ----- -->

htroot/Network.html

@@ -14,20 +14,20 @@
     <div class="SubMenu">
       <h3>YaCy Network</h3>
       <ul class="SubMenu">
-        <li><a href="/Network.html" class="MenuItemLink">Network&nbsp;Overview</a></li>
+        <li><a href="/Network.html" class="MenuItemLink">Network Overview</a></li>
         <li><a href="/Network.html?page=1" class="MenuItemLink">Active&nbsp;Peers</a></li>
         <li><a href="/Network.html?page=2" class="MenuItemLink">Passive&nbsp;Peers</a></li>
         <li><a href="/Network.html?page=3" class="MenuItemLink">Potential&nbsp;Peers</a></li>
       </ul>
     </div>
     #(page)#
-    <h2>Network Overview</h2>
+    <h2>Overview of '#[networkName]#' Network</h2>
     ::
-    <h2>Active Peers</h2>
+    <h2>Active Peers in '#[networkName]#' Network</h2>
     ::
-    <h2>Passive Peers</h2>
+    <h2>Passive Peers in '#[networkName]#' Network</h2>
     ::
-    <h2>Potential Peers</h2>
+    <h2>Potential Peers in '#[networkName]#' Network</h2>
     ::
     <h2>Manually contacting Peer</h2>
     #(/page)#

htroot/Network.java

@@ -78,6 +78,8 @@ public class Network {
         
         // return variable that accumulates replacements
         final serverObjects prop = new serverObjects();
+        prop.put("page_networkTitle", sb.getConfig("network.unit.description", "unspecified"));
+        prop.put("page_networkName", sb.getConfig("network.unit.name", "unspecified"));
         final boolean overview = (post == null) || (post.get("page", "0").equals("0"));
 
         final String mySeedType = yacyCore.seedDB.mySeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN);

htroot/NetworkPicture.java

@@ -43,7 +43,6 @@
 // the intact and unchanged copyright notice.
 // Contributions and changes to the program code must be marked as such.
 
-
 import de.anomic.http.httpHeader;
 import de.anomic.plasma.plasmaGrafics;
 import de.anomic.server.serverObjects;
@@ -79,7 +78,7 @@ public class NetworkPicture {
         if (passiveLimit > 1000000) passiveLimit = 1000000;
         if (potentialLimit > 1000000) potentialLimit = 1000000;
         if (maxCount > 1000) maxCount = 1000;
-        return plasmaGrafics.getNetworkPicture(10000, width, height, passiveLimit, potentialLimit, maxCount, corona);
+        return plasmaGrafics.getNetworkPicture(10000, width, height, passiveLimit, potentialLimit, maxCount, corona, env.getConfig("network.unit.name", "unspecified"), env.getConfig("network.unit.description", "unspecified"));
     }
     
 }

htroot/Status.java

@@ -119,10 +119,6 @@ public class Status {
         	}
         }
         
-        /*
-          versionProbe=http://www.anomic.de/AnomicHTTPProxy/release.txt
-          superseedFile=superseed.txt
-         */
         // update seed info
         yacyCore.peerActions.updateMySeed();
 

source/de/anomic/http/httpc.java

@@ -1373,6 +1373,33 @@ do upload
         return a;
     }
     
+    public static Map loadHashMap(URL url, httpRemoteProxyConfig proxy) {
+        try {
+            // should we use the proxy?
+            boolean useProxy = (proxy != null) &&  
+                               (proxy.useProxy()) && 
+                               (proxy.useProxy4Yacy());
+            
+            // sending request
+            final HashMap result = nxTools.table(
+                    httpc.wget(
+                            url,
+                            url.getHost(),
+                            8000, 
+                            null, 
+                            null, 
+                            (useProxy) ? proxy : null,
+                            null
+                    )
+                    , "UTF-8");
+            
+            if (result == null) return new HashMap();
+            return result;
+        } catch (Exception e) {
+            return new HashMap();
+        }
+    }
+
     public static httpHeader whead(
             URL url,
             String vhost,

source/de/anomic/plasma/plasmaCrawlLURL.java

@@ -187,6 +187,7 @@ public final class plasmaCrawlLURL {
                 oldEntry = null;
             }
         } catch (Exception e) {
+            e.printStackTrace();
             oldEntry = null;
         }
         if ((oldEntry != null) && (entry.isOlder(oldEntry))) {

source/de/anomic/plasma/plasmaGrafics.java

@@ -127,7 +127,7 @@ public class plasmaGrafics {
         if (primarySearches == null) return null; // this was a local search and there are no threads
 
         // get a copy of a recent network picture
-        ymageMatrix eventPicture = getNetworkPicture(120000);
+        ymageMatrix eventPicture = getNetworkPicture(120000, plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", "unspecified"), plasmaSwitchboard.getSwitchboard().getConfig("network.unit.description", "unspecified"));
         if (eventPicture instanceof ymageMatrix) eventPicture = (ymageMatrix) eventPicture; //new ymageMatrix((ymageMatrix) eventPicture);
         // TODO: fix cloning of ymageMatrix pictures
         
@@ -172,18 +172,18 @@ public class plasmaGrafics {
         return eventPicture;
     }
 
-    public static ymageMatrix getNetworkPicture(long maxAge) {
-        return getNetworkPicture(maxAge, 640, 480, 300, 300, 1000, true);
+    public static ymageMatrix getNetworkPicture(long maxAge, String networkName, String networkTitle) {
+        return getNetworkPicture(maxAge, 640, 480, 300, 300, 1000, true, networkName, networkTitle);
     }
 
-    public static ymageMatrix getNetworkPicture(long maxAge, int width, int height, int passiveLimit, int potentialLimit, int maxCount, boolean corona) {
+    public static ymageMatrix getNetworkPicture(long maxAge, int width, int height, int passiveLimit, int potentialLimit, int maxCount, boolean corona, String networkName, String networkTitle) {
         if ((networkPicture == null) || ((System.currentTimeMillis() - networkPictureDate) > maxAge)) {
-            drawNetworkPicture(width, height, passiveLimit, potentialLimit, maxCount, corona);
+            drawNetworkPicture(width, height, passiveLimit, potentialLimit, maxCount, corona, networkName, networkTitle);
         }
         return networkPicture;
     }
 
-    private static void drawNetworkPicture(int width, int height, int passiveLimit, int potentialLimit, int maxCount, boolean corona) {
+    private static void drawNetworkPicture(int width, int height, int passiveLimit, int potentialLimit, int maxCount, boolean corona, String networkName, String networkTitle) {
 
         int innerradius = Math.min(width, height) / 5;
         int outerradius = innerradius + innerradius * yacyCore.seedDB.sizeConnected() / 100;
@@ -196,7 +196,7 @@ public class plasmaGrafics {
 
         // draw network circle
         networkPicture.setColor(COL_DHTCIRCLE);
-        networkPicture.arc(width / 2, height / 2, innerradius - 20, innerradius + 20, 0, 360);
+        networkPicture.arc(width / 2, height / 2 + 20, innerradius - 20, innerradius + 20, 0, 360);
 
         //System.out.println("Seed Maximum distance is       " + yacySeed.maxDHTDistance);
         //System.out.println("Seed Minimum distance is       " + yacySeed.minDHTNumber);
@@ -212,7 +212,7 @@ public class plasmaGrafics {
         while (e.hasMoreElements() && count < maxCount) {
             seed = (yacySeed) e.nextElement();
             if (seed != null) {
-                drawNetworkPicturePeer(networkPicture, width / 2, height / 2, innerradius, outerradius, seed, COL_ACTIVE_DOT, COL_ACTIVE_LINE, COL_ACTIVE_TEXT, corona);
+                drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seed, COL_ACTIVE_DOT, COL_ACTIVE_LINE, COL_ACTIVE_TEXT, corona);
                 count++;
             }
         }
@@ -226,7 +226,7 @@ public class plasmaGrafics {
             if (seed != null) {
                 lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenUTC()) / 1000 / 60);
                 if (lastseen > passiveLimit) break; // we have enough, this list is sorted so we don't miss anything
-                drawNetworkPicturePeer(networkPicture, width / 2, height / 2, innerradius, outerradius, seed, COL_PASSIVE_DOT, COL_PASSIVE_LINE, COL_PASSIVE_TEXT, corona);
+                drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seed, COL_PASSIVE_DOT, COL_PASSIVE_LINE, COL_PASSIVE_TEXT, corona);
                 count++;
             }
         }
@@ -240,21 +240,22 @@ public class plasmaGrafics {
             if (seed != null) {
                 lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenUTC()) / 1000 / 60);
                 if (lastseen > potentialLimit) break; // we have enough, this list is sorted so we don't miss anything
-                drawNetworkPicturePeer(networkPicture, width / 2, height / 2, innerradius, outerradius, seed, COL_POTENTIAL_DOT, COL_POTENTIAL_LINE, COL_POTENTIAL_TEXT, corona);
+                drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seed, COL_POTENTIAL_DOT, COL_POTENTIAL_LINE, COL_POTENTIAL_TEXT, corona);
                 count++;
             }
         }
         totalCount += count;
 
         // draw my own peer
-        drawNetworkPicturePeer(networkPicture, width / 2, height / 2, innerradius, outerradius, yacyCore.seedDB.mySeed, COL_WE_DOT, COL_WE_LINE, COL_WE_TEXT, corona);
+        drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, yacyCore.seedDB.mySeed, COL_WE_DOT, COL_WE_LINE, COL_WE_TEXT, corona);
 
         // draw description
         networkPicture.setColor(COL_HEADLINE);
         networkPicture.setMode(ymageMatrix.MODE_SUB);
-        ymageToolPrint.print(networkPicture, 2, 8, 0, "THE YACY NETWORK", -1);
-        ymageToolPrint.print(networkPicture, 2, 16, 0, "DRAWING OF " + totalCount + " SELECTED PEERS", -1);
+        ymageToolPrint.print(networkPicture, 2, 8, 0, "YACY NETWORK '" + networkName.toUpperCase() + "'", -1);
+        ymageToolPrint.print(networkPicture, 2, 16, 0, networkTitle.toUpperCase(), -1);
         ymageToolPrint.print(networkPicture, width - 2, 8, 0, "SNAPSHOT FROM " + new Date().toString().toUpperCase(), 1);
+        ymageToolPrint.print(networkPicture, width - 2, 16, 0, "DRAWING OF " + totalCount + " SELECTED PEERS", 1);
         
         // set timestamp
         networkPictureDate = System.currentTimeMillis();

source/de/anomic/plasma/plasmaSwitchboard.java

@@ -893,9 +893,47 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
         
         // set loglevel and log
         setLog(new serverLog("PLASMA"));
-        
         if (applyPro) this.log.logInfo("This is the pro-version of YaCy");
         
+        // remote proxy configuration
+        this.remoteProxyConfig = httpRemoteProxyConfig.init(this);
+        this.log.logConfig("Remote proxy configuration:\n" + this.remoteProxyConfig.toString());
+        
+        // load network configuration into settings
+        String networkUnitDefinition = getConfig("network.unit.definition", "yacy.network.unit");
+        String networkGroupDefinition = getConfig("network.group.definition", "yacy.network.group");
+        
+        // include additional network definition properties into our settings
+        // note that these properties cannot be set in the application because they are
+        // _always_ overwritten each time with the default values. This is done so on purpose.
+        // the network definition should be made either consistent for all peers,
+        // or independently using a bootstrap URL
+        Map initProps;
+        if (networkUnitDefinition.startsWith("http://")) {
+            try {
+                this.setConfig(httpc.loadHashMap(new URL(networkUnitDefinition), remoteProxyConfig));
+            } catch (MalformedURLException e) {
+            }
+        } else {
+            File networkUnitDefinitionFile = new File(rootPath, networkUnitDefinition);
+            if (networkUnitDefinitionFile.exists()) {
+                initProps = serverFileUtils.loadHashMap(networkUnitDefinitionFile);
+                this.setConfig(initProps);
+            }
+        }
+        if (networkGroupDefinition.startsWith("http://")) {
+            try {
+                this.setConfig(httpc.loadHashMap(new URL(networkGroupDefinition), remoteProxyConfig));
+            } catch (MalformedURLException e) {
+            }
+        } else {
+            File networkGroupDefinitionFile = new File(rootPath, networkGroupDefinition);
+            if (networkGroupDefinitionFile.exists()) {
+                initProps = serverFileUtils.loadHashMap(networkGroupDefinitionFile);
+                this.setConfig(initProps);
+            }
+        }
+
         // load values from configs
         this.plasmaPath   = new File(rootPath, getConfig(DBPATH, DBPATH_DEFAULT));
         this.log.logConfig("Plasma DB Path: " + this.plasmaPath.toString());
@@ -913,12 +951,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
         this.workPath   = new File(rootPath, getConfig(WORK_PATH, WORK_PATH_DEFAULT));
         this.log.logConfig("Work Path:    " + this.workPath.toString());
         
-        /* ============================================================================
-         * Remote Proxy configuration
-         * ============================================================================ */
-        this.remoteProxyConfig = httpRemoteProxyConfig.init(this);
-        this.log.logConfig("Remote proxy configuration:\n" + this.remoteProxyConfig.toString());
-        
         // set up local robots.txt
         this.robotstxtConfig = httpdRobotsTxtConfig.init(this);
         

source/de/anomic/server/serverAbstractSwitch.java

@@ -84,7 +84,6 @@ public abstract class serverAbstractSwitch implements serverSwitch {
 
         // predefine init's
         Map initProps;
-        configRemoved = new HashMap();
         if (initFile.exists())
             initProps = serverFileUtils.loadHashMap(initFile);
         else
@@ -117,9 +116,9 @@ public abstract class serverAbstractSwitch implements serverSwitch {
         else
             configProps = new HashMap();
 
+        // remove all values from config that do not appear in init
+        configRemoved = new HashMap();
         synchronized (configProps) {
-            // remove all values from config that do not appear in init
-            // (out-dated settings)
             i = configProps.keySet().iterator();
             String key;
             while (i.hasNext()) {
@@ -212,6 +211,15 @@ public abstract class serverAbstractSwitch implements serverSwitch {
         return accessTracker.keySet().iterator();
     }
 
+    public void setConfig(Map otherConfigs) {
+        Iterator i = otherConfigs.entrySet().iterator();
+        Map.Entry entry;
+        while (i.hasNext()) {
+            entry = (Map.Entry) i.next();
+            setConfig((String) entry.getKey(), (String) entry.getValue());
+        }
+    }
+
     public void setConfig(String key, boolean value) {
         setConfig(key, (value) ? "true" : "false");
     }

source/de/anomic/server/serverSwitch.java

@@ -96,6 +96,7 @@ public interface serverSwitch {
     public Iterator /*of serverThread-Names (String)*/ threadNames();
 
     // the switchboard can be used to set and read properties
+    public void setConfig(Map otherConfigs);
     public void setConfig(String key, long value);
     public void setConfig(String key, String value);
     public String getConfig(String key, String dflt);

source/de/anomic/yacy/yacyClient.java

@@ -259,9 +259,9 @@ public final class yacyClient {
         final String key = crypt.randomSalt();
         try {
             // should we use the proxy?
-            boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) &&  
-                               (yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) && 
-                               (yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
+            boolean useProxy = (plasmaSwitchboard.getSwitchboard().remoteProxyConfig != null) &&  
+                               (plasmaSwitchboard.getSwitchboard().remoteProxyConfig.useProxy()) && 
+                               (plasmaSwitchboard.getSwitchboard().remoteProxyConfig.useProxy4Yacy());
             
             // sending request
             final HashMap result = nxTools.table(
@@ -278,7 +278,7 @@ public final class yacyClient {
                             8000, 
                             null, 
                             null, 
-                            (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
+                            (useProxy)? plasmaSwitchboard.getSwitchboard().remoteProxyConfig:null,
                             null
                     )
                     , "UTF-8");
@@ -526,18 +526,28 @@ public final class yacyClient {
                 assert (urlEntry.hash().length() == 12) : "urlEntry.hash() = " + urlEntry.hash();
                 if (urlEntry.hash().length() != 12) continue; // bad url hash
                 indexURLEntry.Components comp = urlEntry.comp();
-                if (blacklist.isListed(plasmaURLPattern.BLACKLIST_SEARCH, comp.url())) continue; // block with backlist
-                urlManager.store(urlEntry);
-                urlManager.stack(urlEntry, yacyCore.seedDB.mySeed.hash, targetPeer.hash, 2);
+                if (blacklist.isListed(plasmaURLPattern.BLACKLIST_SEARCH, comp.url())) {
+                    yacyCore.log.logInfo("remote search (client): filtered blacklisted url " + comp.url() + " from peer " + targetPeer.getName());
+                    continue; // block with backlist
+                }
                 
                 // save the url entry
                 indexRWIEntry entry;
                 if (urlEntry.word() == null) {
-                    yacyCore.log.logWarning("DEBUG-SEARCH: no word attached from peer " + targetPeer.getName() + ", version " + targetPeer.getVersion());
+                    yacyCore.log.logWarning("remote search (client): no word attached from peer " + targetPeer.getName() + ", version " + targetPeer.getVersion());
                     continue; // no word attached
                 }
+                
                 // the search-result-url transports all the attributes of word indexes
                 entry = urlEntry.word();
+                if (!(entry.urlHash().equals(urlEntry.hash()))) {
+                    yacyCore.log.logInfo("remote search (client): url-hash " + urlEntry.hash() + " does not belong to word-attached-hash " + entry.urlHash() + "; url = " + comp.url() + " from peer " + targetPeer.getName());
+                    continue; // spammed
+                }
+
+                // passed all checks, store url
+                urlManager.store(urlEntry);
+                urlManager.stack(urlEntry, yacyCore.seedDB.mySeed.hash, targetPeer.hash, 2);
 
                 if (urlEntry.snippet() != null) {
                     // we don't store the snippets along the url entry, because they are search-specific.

source/de/anomic/yacy/yacyCore.java

@@ -215,8 +215,8 @@ public class yacyCore {
 
         // deploy peer actions
         peerActions = new yacyPeerActions(seedDB, switchboard,
-                new File(sb.getRootPath(), sb.getConfig("superseedFile", "superseed.txt")),
-                switchboard.getConfig("superseedLocation", "http://www.yacy.net/yacy/superseed.txt"));
+                new File(sb.getRootPath(), sb.getConfig("network.unit.bootstrap.file", "superseed.txt")),
+                switchboard.getConfig("network.unit.bootstrap.url", "http://www.yacy.net/yacy/superseed.txt"));
         dhtAgent = new yacyDHTAction(seedDB);
         peerActions.deploy(dhtAgent);
         peerActions.deploy(new yacyNewsAction(newsPool));

source/yacy.java

@@ -253,8 +253,7 @@ public final class yacy {
             
             f = new File(homePath, "DATA/yacy.running");
             if (f.exists()) {                // another instance running? VM crash? User will have to care about this
-                serverLog.logSevere("STARTUP", "the file " + f + " exists, this usually means that another instance of YaCy is using this DATA-folder.");
-                serverLog.logSevere("STARTUP", "please make sure that DATA can be used exclusively by one YaCy. Quitting...");
+                serverLog.logSevere("STARTUP", "WARNING: the file " + f + " exists, this usually means that a YaCy instance is still running");
                 f.delete();
             }
             f.createNewFile();

yacy.init

@@ -64,43 +64,14 @@ pkcs12ImportPwd =
 # value is in milliseconds, default is one hour
 server.maxTrackingTime = 3600000
 
-# peer-to-peer construction for distributed search
-# we have several stages:
-# 1st: a file within every distribution that has a list of URLs:
-#      -> this is the superseed file
-# 2nd: the files that can be retrieved by the superseeds' URLs
-#      are called seed list-files.
-#      -> the seed list-files contain IP/port combinations of running
-#         AnomicHTTPProxies
-# 3rd: the peers that are targeted within the seed files are called superpeers
-# 4th: the superpeers hold and share a list of all client/search/crawl peers
-#
-# some superpeers should be able to create again seed list-files.
-# These superpeers must upload their IP or their list of peer-IP's to a
-# ftp location to create the seed list-file.
-# Everyone who do so should mail his/her new seed location to mc<at>anomic.de
-# The seed list-file location will then be included in the superseed file.
-# This superseed file is available then at two localtions:
-# - it is included in every distribution and
-# - updated through a specific URL-location
-# we see the file name and the URL of the superseed here:
-superseedFile=superseed.txt
-superseedLocation=http://www.yacy.net/superseed.txt
-
-# network definition
-# we distiguish local and global networks. Each network type can have different user groups
-# groups can be uncontrolled, moderated or controlled
-# Each group has a group master, but in case the group is uncontrolled the only task of the
-# group master is to provide a bootstrap file which contains peer addresses of peers that
-# are member of the group. All attributes are given in a single network description string
-# of the form:
-# network = <group-name>:<network-name>:<network-domain>:{'uncontrolled'|'moderated'|'controlled'}:<bootstrap-uri>
-# in case that the network is the uncontrolled global dht-managed community for everybody,
-# the details defaults to:
-# network = all:world:global:uncontrolled:http://yacy.net/
-# the network-uri must have a sub-path yacy/seed.txt containing a list of urls pointing to the
-# peer-address of peers within the group of that network
-# several network definition strings can be listed
+# Network Definition
+# There can be separate YaCy networks, and managed sub-groups of the general network.
+# The essentials of the network definition are attached in separate property files.
+# The property here can also be a url where the definition can be loaded.
+# In case of privately managed networks, this configuration must be changed BEFORE it is released
+# to the members of the separated network peers.
+network.unit.definition = yacy.network.unit
+network.group.definition = yacy.network.group
 
 # clusters within a network:
 # every network can have an unlimited number of clusters. Clusters may be also completely
@@ -716,7 +687,7 @@ useYacyReferer = true
 useYacyReferer__pro = false
 
 # allow only 443(https-port) for https-proxy?
-# if you want to tunnel other protokols, set to false
+# if you want to tunnel other protocols, set to false
 secureHttps = true
 
 # specifies if the httpdFileHandler should cache 

yacy.network.group

@@ -0,0 +1,29 @@
+# YaCy Network Group Definition
+# -----------------------------
+# This is an addition to the yacy.network.unit configuration file.
+# This file is adressed by the network.group.definition property in yacy.init
+# The purpose of a group within a network is that some parts of a network may be managed independently,
+# while the content of the network stays private for the whole network, mostly for a special purpose.
+# This file needs to be configured if someone wants to participate with several peers to the network,
+# but wants to manage the group of own peers with a single master peer.
+# One example application would be the use of computing-teams
+
+# Defintion of property domains:
+#     network.group.name = <any word, name of network>
+#     network.group.description = <any string, just informal; appears in network graphic>
+#     network.group.administration.control = 'uncontrolled'|'moderated'|'controlled'
+#     network.group.administration.manager = <a .yacy or .yacyh - domain, naming the manager peer>
+#     network.group.administration.request.authentification.method = 'salted-magic'
+#     network.group.administration.request.authentification.essentials = <any string, a secret password>
+
+# -----------------------------------------------------------------#
+# this is a work in progress. disabled properties are not yet used #
+# -----------------------------------------------------------------#
+
+# properties for group access administrator-authentification:
+#network.group.name = everybody
+#network.group.description = Our Peer-Group
+#network.group.administration.control = uncontrolled
+#network.group.administration.manager = 
+#network.group.administration.request.authentification.method = salted-magic
+#network.group.administration.request.authentification.essentials =

yacy.network.unit

@@ -0,0 +1,72 @@
+# YaCy Network Unit Definition
+# ----------------------------
+# There is not only a single global YaCy network, but there can be also separated networks
+# This file describes the properties of the network, how it is bootstraped, if it is moderated
+# and so on. It is also possible to describe authentification methods for in-protocol communication
+# To distinguish between index areas where the shared web index is for a intranet or the public internet,
+# we distinguish local and global networks. Each network type can have different user groups.
+# Groups can be uncontrolled, moderated or controlled. The whole network may also have a global master.
+# This file is adressed by the network.unit.definition property in yacy.init. If necessary this
+# property can also address a URL where a network definition file can be hosted, so that a network
+# can easily be moderated from a single central. In case of the public network of all independent peers
+# for global indexing, the network definition is attached to the release and defines no network master or
+# any other supervision.
+
+# peer-to-peer construction using the bootstrap properties:
+# 1st: a file within every distribution that has a list of URLs,
+#      or a location with a file containing a list of URLs:
+#      -> these are the superseed files (there are only two of these files)
+# 2nd: the files that can be retrieved by the superseeds' URLs
+#      are called seed list-files. (there can be many of those)
+#      -> the seed list-files contain b.o. IP/port combinations of running YaCy peers
+#      -> seed-list files are written by principal peers
+
+# Control Modes for Peer Groups:
+# we have three different types of moderation:
+# uncontrolled: only the peer's administrator may use protected pages of the YaCy interface
+# moderated   : the unit/group moderator may access some protected pages i.e. Crawl Start
+# controlled  : the unit/group moderator may access all pages of the peers within the same unit/group
+
+# Defintion of property domains:
+#     network.unit.name = <any word, name of network; appears in network graphic>
+#     network.unit.description = <any string, just informal; appears in network graphic>
+#     network.unit.domain = 'global'|'local'
+#     network.unit.dhtredundancy = <integer number, 0 means no DHT enabled>
+#     network.unit.bootstrap.file = <a file in the application directory containing urls to seedlists>
+#     network.unit.bootstrap.url = <a url to a bootstrap.file containing urls to seedlists>
+#     network.unit.protocol.control = 'uncontrolled'|'moderated'|'controlled'
+#     network.unit.protocol.request.authentification.method = 'salted-magic'
+#     network.unit.protocol.request.authentification.essentials = <any string, a secret password>
+#     network.unit.protocol.response.authentification.method = 'challenge-pwresponse'
+#     network.unit.protocol.response.authentification.essentials = <a challenge string>':'<a secret response-string>
+#     network.unit.administration.control = 'uncontrolled'|'moderated'|'controlled'
+#     network.unit.administration.manager = <a .yacy or .yacyh - domain, naming the manager peer>
+#     network.unit.administration.request.authentification.method = 'salted-magic'
+#     network.unit.administration.request.authentification.essentials = <any string, a secret password>
+
+
+# -----------------------------------------------------------------#
+# this is a work in progress. disabled properties are not yet used #
+# -----------------------------------------------------------------#
+
+# general network definition
+network.unit.name = freeworld
+network.unit.description = Public YaCy Community
+#network.unit.domain = global
+network.unit.dhtredundancy.junior = 1
+network.unit.dhtredundancy.senior = 3
+network.unit.bootstrap.file = superseed.txt
+network.unit.bootstrap.url = http://www.yacy.net/superseed.txt
+
+# properties for in-protocol response authentification:
+#network.unit.protocol.control = uncontrolled
+#network.unit.protocol.request.authentification.method = salted-magic
+#network.unit.protocol.request.authentification.essentials =
+#network.unit.protocol.response.authentification.method = challenge-pwresponse
+#network.unit.protocol.response.authentification.essentials =
+
+# properties for unit access administrator-authentification:
+#network.unit.administration.control = uncontrolled
+#network.unit.administration.manager =
+#network.unit.administration.request.authentification.method = salted-magic
+#network.unit.administration.request.authentification.essentials =