From 461548698ccc1fcbb36248aaed233768fd81d54b Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 9 May 2006 11:43:10 +0000 Subject: [PATCH] configuration of index transfer chunk size see http://www.yacy-forum.de/viewtopic.php?p=20951#20951 new properties in yacy.init: indexDistribution.minChunkSize = 5 indexDistribution.maxChunkSize = 1000 indexDistribution.startChunkSize = 50 git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2073 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/de/anomic/plasma/plasmaSwitchboard.java | 13 +++++++++---- yacy.init | 5 +++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index e00e16f27..766b5c9eb 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -161,8 +161,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser public static int maxCRLDump = 500000; public static int maxCRGDump = 200000; - private int dhtTransferIndexCount = 50; - private static final int dhtTransferIndexMinimum = 30; + + private int dhtTransferIndexCount = 50; // couloured list management public static TreeSet badwords = null; @@ -548,6 +548,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // initializing dht chunk generation this.dhtTransferChunk = null; + this.dhtTransferIndexCount = (int) getConfigLong("indexDistribution.startChunkSize", 50); // deploy threads log.logConfig("Starting Threads"); @@ -929,7 +930,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // (this.dhtTransferChunk.getStatus() == plasmaDHTChunk.chunkStatus_COMPLETE) || (this.dhtTransferChunk.getStatus() == plasmaDHTChunk.chunkStatus_FAILED))) { // generate new chunk - dhtTransferChunk = new plasmaDHTChunk(this.log, this.wordIndex, this.urlPool.loadedURL, 30, dhtTransferIndexCount); + int minChunkSize = (int) getConfigLong("indexDistribution.minChunkSize", 30); + dhtTransferChunk = new plasmaDHTChunk(this.log, this.wordIndex, this.urlPool.loadedURL, minChunkSize, dhtTransferIndexCount); doneSomething = true; } @@ -2119,7 +2121,10 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser } else { if (dhtTransferChunk.indexCount() >= dhtTransferIndexCount) dhtTransferIndexCount++; } - if (dhtTransferIndexCount < dhtTransferIndexMinimum) dhtTransferIndexCount = dhtTransferIndexMinimum; + int minChunkSize = (int) getConfigLong("indexDistribution.minChunkSize", 30); + int maxChunkSize = (int) getConfigLong("indexDistribution.maxChunkSize", 3000); + if (dhtTransferIndexCount < minChunkSize) dhtTransferIndexCount = minChunkSize; + if (dhtTransferIndexCount > maxChunkSize) dhtTransferIndexCount = maxChunkSize; // show success return true; diff --git a/yacy.init b/yacy.init index 298fcbbbd..f7259376f 100644 --- a/yacy.init +++ b/yacy.init @@ -611,6 +611,11 @@ indexTransfer.timeout = 120000 indexDistribution.maxOpenFiles = 800 indexTransfer.maxOpenFiles = 800 +# sizes for index distribution +indexDistribution.minChunkSize = 5 +indexDistribution.maxChunkSize = 1000 +indexDistribution.startChunkSize = 50 + # Distribution of Citation-Reference (CR-) files # The distribution is done in two steps: # first step to anonymize the records