- better control of chunk size in dht selection

- more restrict values in selection
- step to 4 vertical partitions

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5603 6c8d7289-2bf4-0310-a012-ef5d649a1542

defaults/yacy.network.freeworld.unit

@@ -14,7 +14,7 @@ network.unit.search.time = 4
 network.unit.dht = true
 network.unit.dhtredundancy.junior = 1
 network.unit.dhtredundancy.senior = 3
-network.unit.dht.partitionExponent = 1
+network.unit.dht.partitionExponent = 2
 network.unit.remotecrawl.speed = 6
 network.unit.bootstrap.seedlist0 = http://www.yacy.net/seed.txt
 network.unit.bootstrap.seedlist1 = http://home.arcor.de/hermens/yacy/seed.txt

source/de/anomic/plasma/plasmaSwitchboard.java

@@ -179,9 +179,10 @@ import de.anomic.yacy.dht.PeerSelection;
 public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.QueueEntry> implements serverSwitch<IndexingStack.QueueEntry> {
     
     // load slots
-    public  static int  xstackCrawlSlots      = 2000;
-    private        int  dhtTransferIndexCount = 100;
-    public  static long lastPPMUpdate         = System.currentTimeMillis()- 30000;
+    public  static int  xstackCrawlSlots     = 2000;
+    private        int  dhtMaxContainerCount = 100;
+    private        int  dhtMaxReferenceCount = 1000;
+    public  static long lastPPMUpdate        = System.currentTimeMillis()- 30000;
 
     // colored list management
     public static TreeSet<String> badwords = null;
@@ -567,7 +568,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
                 "global.any".indexOf(getConfig("network.unit.domain", "global")) >= 0);
         
         // initializing dht chunk generation
-        this.dhtTransferIndexCount = (int) getConfigLong(plasmaSwitchboardConstants.INDEX_DIST_CHUNK_SIZE_START, 50);
+        this.dhtMaxReferenceCount = (int) getConfigLong(plasmaSwitchboardConstants.INDEX_DIST_CHUNK_SIZE_START, 50);
         
         // init robinson cluster
         // before we do that, we wait some time until the seed list is loaded.
@@ -1920,7 +1921,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
             int c = this.dhtDispatcher.selectContainersToCache(
                     startHash,
                     limitHash,
-                    dhtTransferIndexCount,
+                    dhtMaxContainerCount,
+                    dhtMaxReferenceCount,
                     2000);
             log.logInfo("dhtTransferJob: Dispatcher selected " + c + " containers");
         } catch (IOException e) {

source/de/anomic/yacy/dht/Dispatcher.java

@@ -142,10 +142,11 @@ public class Dispatcher {
             final String hash,
             final String limitHash,
             final int maxContainerCount,
+            final int maxReferenceCount,
             final int maxtime) throws IOException {
         
     	// prefer file
-        ArrayList<indexContainer> containers = selectContainers(hash, limitHash, maxContainerCount, maxtime, false);
+        ArrayList<indexContainer> containers = selectContainers(hash, limitHash, maxContainerCount, maxReferenceCount, maxtime, false);
 
         // if ram does not provide any result, take from file
         //if (containers.size() == 0) containers = selectContainers(hash, limitHash, maxContainerCount, maxtime, false);
@@ -156,6 +157,7 @@ public class Dispatcher {
             final String hash,
             final String limitHash,
             final int maxContainerCount,
+            final int maxReferenceCount,
             final int maxtime,
             final boolean ram) throws IOException {
         
@@ -168,14 +170,17 @@ public class Dispatcher {
         // first select the container
         final long timeout = (maxtime < 0) ? Long.MAX_VALUE : System.currentTimeMillis() + maxtime;
         while (
-                (maxContainerCount > refcount) &&
+                (containers.size() < maxContainerCount) &&
+                (refcount < maxReferenceCount) &&
                 (indexContainerIterator.hasNext()) &&
+                (System.currentTimeMillis() < timeout) &&
                 ((container = indexContainerIterator.next()) != null) &&
-                (container.size() > 0) &&
                 ((containers.size() == 0) ||
-                 (Base64Order.enhancedComparator.compare(container.getWordHash(), limitHash) < 0)) &&
-                (System.currentTimeMillis() < timeout)
+                 (Base64Order.enhancedComparator.compare(container.getWordHash(), limitHash) < 0))
+                
         ) {
+            if (container.size() == 0) continue;
+            refcount += container.size();
             containers.add(container);
         }
         // then remove the container from the backend
@@ -198,12 +203,13 @@ public class Dispatcher {
             final String hash,
             final String limitHash,
             final int maxContainerCount,
+            final int maxReferenceCount,
             final int maxtime) throws IOException {
         if (this.selectedContainerCache != null && this.selectedContainerCache.size() > 0) {
         	this.log.logInfo("selectContainersToCache: selectedContainerCache is already filled, no selection done.");
         	return 0;
         }
-        this.selectedContainerCache = selectContainers(hash, limitHash, maxContainerCount, maxtime);
+        this.selectedContainerCache = selectContainers(hash, limitHash, maxContainerCount, maxReferenceCount, maxtime);
         this.log.logInfo("selectContainersToCache: selectedContainerCache was filled with " + this.selectedContainerCache.size() + " entries");
     	return this.selectedContainerCache.size();
     }