From be5e80823687acd2a74b9868e3079b738a5fca90 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Tue, 21 Jan 2014 17:48:45 +0100 Subject: [PATCH] - removed hardcoded load-test which is now handled in BusyQueues steering, see /PerformanceQueues_p.html - changed default values for crawler queue load limit (high, because these jobs are started upon user request) --- defaults/yacy.init | 9 ++++----- htroot/yacy/transferRWI.java | 2 +- source/net/yacy/peers/Dispatcher.java | 6 +++++- source/net/yacy/search/Switchboard.java | 5 ----- source/net/yacy/search/SwitchboardConstants.java | 1 - 5 files changed, 10 insertions(+), 13 deletions(-) diff --git a/defaults/yacy.init b/defaults/yacy.init index 615c69f14..b499a6ca9 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -610,22 +610,22 @@ collection=user 50_localcrawl_idlesleep=2000 50_localcrawl_busysleep=10 50_localcrawl_memprereq=12582912 -50_localcrawl_loadprereq=2.0 +50_localcrawl_loadprereq=8.0 50_localcrawl_isPaused=false 60_remotecrawlloader_idlesleep=4000 60_remotecrawlloader_busysleep=800 60_remotecrawlloader_memprereq=12582912 -60_remotecrawlloader_loadprereq=2.0 +60_remotecrawlloader_loadprereq=8.0 60_remotecrawlloader_isPaused=false 62_remotetriggeredcrawl_idlesleep=2000 62_remotetriggeredcrawl_busysleep=200 62_remotetriggeredcrawl_memprereq=12582912 -62_remotetriggeredcrawl_loadprereq=2.0 +62_remotetriggeredcrawl_loadprereq=8.0 62_remotetriggeredcrawl_isPaused=false 70_surrogates_idlesleep=10000 70_surrogates_busysleep=0 70_surrogates_memprereq=12582912 -70_surrogates_loadprereq=2.0 +70_surrogates_loadprereq=8.0 80_searchresult_idlesleep=10000 80_searchresult_busysleep=200 80_searchresult_memprereq=0 @@ -860,7 +860,6 @@ indexControl.gzipBody = true indexControl.timeout = 60000 indexDistribution.timeout = 60000 indexTransfer.timeout = 120000 -indexTransfer.maxload = 2.2 # defining max. allowed amount of open files during index- transfer/distribution indexDistribution.maxOpenFiles = 800 diff --git a/htroot/yacy/transferRWI.java b/htroot/yacy/transferRWI.java index 2367105ae..94cf8667e 100644 --- a/htroot/yacy/transferRWI.java +++ b/htroot/yacy/transferRWI.java @@ -127,7 +127,7 @@ public final class transferRWI { final StringBuilder unknownURLs = new StringBuilder(6000); double load = Memory.load(); - float maxload = sb.getConfigFloat(SwitchboardConstants.INDEX_TRANSFER_MAXLOAD, 1.5f); + float maxload = sb.getConfigFloat(SwitchboardConstants.INDEX_DIST_LOADPREREQ, 2.0f); if (load > maxload) { // too high local load. this is bad but we must reject this to protect ourself! sb.getLog().info("Rejecting RWIs from peer " + otherPeerName + ", system has too high load = " + load + ", maxload = " + maxload); diff --git a/source/net/yacy/peers/Dispatcher.java b/source/net/yacy/peers/Dispatcher.java index a5ee4a1e0..a59aef84a 100644 --- a/source/net/yacy/peers/Dispatcher.java +++ b/source/net/yacy/peers/Dispatcher.java @@ -44,6 +44,8 @@ import net.yacy.kelondro.data.word.WordReference; import net.yacy.kelondro.index.RowHandleSet; import net.yacy.kelondro.rwi.ReferenceContainer; import net.yacy.kelondro.workflow.WorkflowProcessor; +import net.yacy.search.Switchboard; +import net.yacy.search.SwitchboardConstants; import net.yacy.search.index.Segment; public class Dispatcher { @@ -401,7 +403,9 @@ public class Dispatcher { // try to keep the system healthy; sleep as long as System load is too high while (Protocol.metadataRetrievalRunning.get() > 0) try {Thread.sleep(1000);} catch (InterruptedException e) {break;} - while (Memory.load() > 2.0) try {Thread.sleep(10000);} catch (InterruptedException e) {break;} + + // we must test this here again + while (Memory.load() > Switchboard.getSwitchboard().getConfigFloat(SwitchboardConstants.INDEX_DIST_LOADPREREQ, 2.0f)) try {Thread.sleep(10000);} catch (InterruptedException e) {break;} // do the transmission final boolean success = chunk.transmit(); diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index aea45e0ca..734c8a114 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -3417,11 +3417,6 @@ public final class Switchboard extends serverSwitch { + getIndexingProcessorsQueueSize(); } - double load = Memory.load(); - float maxload = getConfigFloat(SwitchboardConstants.INDEX_TRANSFER_MAXLOAD, 1.5f); - if (load > maxload) { - return "no DHT distribution: system has too high load = " + load + ", maxload = " + maxload; - } return null; // this means; yes, please do dht transfer } diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java index ff6fb3908..0a41661af 100644 --- a/source/net/yacy/search/SwitchboardConstants.java +++ b/source/net/yacy/search/SwitchboardConstants.java @@ -266,7 +266,6 @@ public final class SwitchboardConstants { public static final String INDEX_DIST_ALLOW_WHILE_CRAWLING = "allowDistributeIndexWhileCrawling"; public static final String INDEX_DIST_ALLOW_WHILE_INDEXING = "allowDistributeIndexWhileIndexing"; public static final String INDEX_TRANSFER_TIMEOUT = "indexTransfer.timeout"; - public static final String INDEX_TRANSFER_MAXLOAD = "indexTransfer.maxload"; public static final String INDEX_TRANSFER_GZIP_BODY = "indexTransfer.gzipBody"; public static final String PARSER_MIME_DENY = "parser.mime.deny"; public static final String PARSER_EXTENSIONS_DENY = "parser.extensions.deny";