From 40d9849aa4176e524c946cbfef3502452ec70392 Mon Sep 17 00:00:00 2001 From: orbiter Date: Thu, 12 Feb 2009 14:47:32 +0000 Subject: [PATCH] - better control of chunk size in dht selection - more restrict values in selection - step to 4 vertical partitions git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5603 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- defaults/yacy.network.freeworld.unit | 2 +- source/de/anomic/plasma/plasmaSwitchboard.java | 12 +++++++----- source/de/anomic/yacy/dht/Dispatcher.java | 18 ++++++++++++------ 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/defaults/yacy.network.freeworld.unit b/defaults/yacy.network.freeworld.unit index c866febc4..d8e23e571 100644 --- a/defaults/yacy.network.freeworld.unit +++ b/defaults/yacy.network.freeworld.unit @@ -14,7 +14,7 @@ network.unit.search.time = 4 network.unit.dht = true network.unit.dhtredundancy.junior = 1 network.unit.dhtredundancy.senior = 3 -network.unit.dht.partitionExponent = 1 +network.unit.dht.partitionExponent = 2 network.unit.remotecrawl.speed = 6 network.unit.bootstrap.seedlist0 = http://www.yacy.net/seed.txt network.unit.bootstrap.seedlist1 = http://home.arcor.de/hermens/yacy/seed.txt diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 69854effd..c56b7983a 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -179,9 +179,10 @@ import de.anomic.yacy.dht.PeerSelection; public final class plasmaSwitchboard extends serverAbstractSwitch implements serverSwitch { // load slots - public static int xstackCrawlSlots = 2000; - private int dhtTransferIndexCount = 100; - public static long lastPPMUpdate = System.currentTimeMillis()- 30000; + public static int xstackCrawlSlots = 2000; + private int dhtMaxContainerCount = 100; + private int dhtMaxReferenceCount = 1000; + public static long lastPPMUpdate = System.currentTimeMillis()- 30000; // colored list management public static TreeSet badwords = null; @@ -567,7 +568,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch= 0); // initializing dht chunk generation - this.dhtTransferIndexCount = (int) getConfigLong(plasmaSwitchboardConstants.INDEX_DIST_CHUNK_SIZE_START, 50); + this.dhtMaxReferenceCount = (int) getConfigLong(plasmaSwitchboardConstants.INDEX_DIST_CHUNK_SIZE_START, 50); // init robinson cluster // before we do that, we wait some time until the seed list is loaded. @@ -1920,7 +1921,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch containers = selectContainers(hash, limitHash, maxContainerCount, maxtime, false); + ArrayList containers = selectContainers(hash, limitHash, maxContainerCount, maxReferenceCount, maxtime, false); // if ram does not provide any result, take from file //if (containers.size() == 0) containers = selectContainers(hash, limitHash, maxContainerCount, maxtime, false); @@ -156,6 +157,7 @@ public class Dispatcher { final String hash, final String limitHash, final int maxContainerCount, + final int maxReferenceCount, final int maxtime, final boolean ram) throws IOException { @@ -168,14 +170,17 @@ public class Dispatcher { // first select the container final long timeout = (maxtime < 0) ? Long.MAX_VALUE : System.currentTimeMillis() + maxtime; while ( - (maxContainerCount > refcount) && + (containers.size() < maxContainerCount) && + (refcount < maxReferenceCount) && (indexContainerIterator.hasNext()) && + (System.currentTimeMillis() < timeout) && ((container = indexContainerIterator.next()) != null) && - (container.size() > 0) && ((containers.size() == 0) || - (Base64Order.enhancedComparator.compare(container.getWordHash(), limitHash) < 0)) && - (System.currentTimeMillis() < timeout) + (Base64Order.enhancedComparator.compare(container.getWordHash(), limitHash) < 0)) + ) { + if (container.size() == 0) continue; + refcount += container.size(); containers.add(container); } // then remove the container from the backend @@ -198,12 +203,13 @@ public class Dispatcher { final String hash, final String limitHash, final int maxContainerCount, + final int maxReferenceCount, final int maxtime) throws IOException { if (this.selectedContainerCache != null && this.selectedContainerCache.size() > 0) { this.log.logInfo("selectContainersToCache: selectedContainerCache is already filled, no selection done."); return 0; } - this.selectedContainerCache = selectContainers(hash, limitHash, maxContainerCount, maxtime); + this.selectedContainerCache = selectContainers(hash, limitHash, maxContainerCount, maxReferenceCount, maxtime); this.log.logInfo("selectContainersToCache: selectedContainerCache was filled with " + this.selectedContainerCache.size() + " entries"); return this.selectedContainerCache.size(); }