- removed hardcoded load-test which is now handled in BusyQueues

steering, see /PerformanceQueues_p.html
- changed default values for crawler queue load limit (high, because
these jobs are started upon user request)

defaults/yacy.init

@@ -610,22 +610,22 @@ collection=user
 50_localcrawl_idlesleep=2000
 50_localcrawl_busysleep=10
 50_localcrawl_memprereq=12582912
-50_localcrawl_loadprereq=2.0
+50_localcrawl_loadprereq=8.0
 50_localcrawl_isPaused=false
 60_remotecrawlloader_idlesleep=4000
 60_remotecrawlloader_busysleep=800
 60_remotecrawlloader_memprereq=12582912
-60_remotecrawlloader_loadprereq=2.0
+60_remotecrawlloader_loadprereq=8.0
 60_remotecrawlloader_isPaused=false
 62_remotetriggeredcrawl_idlesleep=2000
 62_remotetriggeredcrawl_busysleep=200
 62_remotetriggeredcrawl_memprereq=12582912
-62_remotetriggeredcrawl_loadprereq=2.0
+62_remotetriggeredcrawl_loadprereq=8.0
 62_remotetriggeredcrawl_isPaused=false
 70_surrogates_idlesleep=10000
 70_surrogates_busysleep=0
 70_surrogates_memprereq=12582912
-70_surrogates_loadprereq=2.0
+70_surrogates_loadprereq=8.0
 80_searchresult_idlesleep=10000
 80_searchresult_busysleep=200
 80_searchresult_memprereq=0
@@ -860,7 +860,6 @@ indexControl.gzipBody = true
 indexControl.timeout = 60000
 indexDistribution.timeout = 60000
 indexTransfer.timeout = 120000
-indexTransfer.maxload = 2.2
 
 # defining max. allowed amount of open files during index- transfer/distribution
 indexDistribution.maxOpenFiles = 800

htroot/yacy/transferRWI.java

@@ -127,7 +127,7 @@ public final class transferRWI {
         final StringBuilder unknownURLs = new StringBuilder(6000);
 
         double load = Memory.load();
-        float maxload = sb.getConfigFloat(SwitchboardConstants.INDEX_TRANSFER_MAXLOAD, 1.5f);
+        float maxload = sb.getConfigFloat(SwitchboardConstants.INDEX_DIST_LOADPREREQ, 2.0f);
         if (load > maxload) {
             // too high local load. this is bad but we must reject this to protect ourself!
             sb.getLog().info("Rejecting RWIs from peer " + otherPeerName + ", system has too high load = " + load + ", maxload = " + maxload);

source/net/yacy/peers/Dispatcher.java

@@ -44,6 +44,8 @@ import net.yacy.kelondro.data.word.WordReference;
 import net.yacy.kelondro.index.RowHandleSet;
 import net.yacy.kelondro.rwi.ReferenceContainer;
 import net.yacy.kelondro.workflow.WorkflowProcessor;
+import net.yacy.search.Switchboard;
+import net.yacy.search.SwitchboardConstants;
 import net.yacy.search.index.Segment;
 
 public class Dispatcher {
@@ -401,7 +403,9 @@ public class Dispatcher {
 
         // try to keep the system healthy; sleep as long as System load is too high
         while (Protocol.metadataRetrievalRunning.get() > 0) try {Thread.sleep(1000);} catch (InterruptedException e) {break;}
-        while (Memory.load() > 2.0) try {Thread.sleep(10000);} catch (InterruptedException e) {break;}
+        
+        // we must test this here again
+        while (Memory.load() > Switchboard.getSwitchboard().getConfigFloat(SwitchboardConstants.INDEX_DIST_LOADPREREQ, 2.0f)) try {Thread.sleep(10000);} catch (InterruptedException e) {break;}
         
         // do the transmission
         final boolean success = chunk.transmit();

source/net/yacy/search/Switchboard.java

@@ -3417,11 +3417,6 @@ public final class Switchboard extends serverSwitch {
                 + getIndexingProcessorsQueueSize();
         }
         
-        double load = Memory.load();
-        float maxload = getConfigFloat(SwitchboardConstants.INDEX_TRANSFER_MAXLOAD, 1.5f);
-        if (load > maxload) {
-            return "no DHT distribution: system has too high load = " + load + ", maxload = " + maxload;
-        }
         return null; // this means; yes, please do dht transfer
     }
 

source/net/yacy/search/SwitchboardConstants.java

@@ -266,7 +266,6 @@ public final class SwitchboardConstants {
     public static final String INDEX_DIST_ALLOW_WHILE_CRAWLING  = "allowDistributeIndexWhileCrawling";
     public static final String INDEX_DIST_ALLOW_WHILE_INDEXING  = "allowDistributeIndexWhileIndexing";
     public static final String INDEX_TRANSFER_TIMEOUT           = "indexTransfer.timeout";
-    public static final String INDEX_TRANSFER_MAXLOAD           = "indexTransfer.maxload";
     public static final String INDEX_TRANSFER_GZIP_BODY         = "indexTransfer.gzipBody";
     public static final String PARSER_MIME_DENY                 = "parser.mime.deny";
     public static final String PARSER_EXTENSIONS_DENY           = "parser.extensions.deny";