reduce load during dht transfer

source/net/yacy/peers/Dispatcher.java

@@ -37,6 +37,7 @@ import net.yacy.cora.federate.yacy.Distribution;
 import net.yacy.cora.order.Base64Order;
 import net.yacy.cora.storage.HandleSet;
 import net.yacy.cora.util.ConcurrentLog;
+import net.yacy.cora.util.Memory;
 import net.yacy.cora.util.SpaceExceededException;
 import net.yacy.kelondro.data.meta.URIMetadataRow;
 import net.yacy.kelondro.data.word.Word;
@@ -116,7 +117,7 @@ public class Dispatcher {
             gzipBody,
             timeout);
 
-        final int concurrentSender = Math.min(32, WorkflowProcessor.availableCPU * 2);
+        final int concurrentSender = Math.min(8, WorkflowProcessor.availableCPU);
         this.indexingTransmissionProcessor = new WorkflowProcessor<Transmission.Chunk>(
                 "transferDocumentIndex",
                 "This is the RWI transmission process",
@@ -399,6 +400,10 @@ public class Dispatcher {
      */
     public Transmission.Chunk transferDocumentIndex(final Transmission.Chunk chunk) {
 
+        // try to keep the system healthy; sleep as long as System load is too high
+        while (Protocol.metadataRetrievalRunning) try {Thread.sleep(1000);} catch (InterruptedException e) {break;}
+        while (Memory.load() > 2.0) try {Thread.sleep(10000);} catch (InterruptedException e) {break;}
+        
         // do the transmission
         final boolean success = chunk.transmit();
 

source/net/yacy/peers/Protocol.java

@@ -1311,6 +1311,8 @@ public final class Protocol {
         }
     }
 
+    public static boolean metadataRetrievalRunning = false;
+
     /**
      * transfer the index. If the transmission fails, return a string describing the cause. If everything is
      * ok, return null.
@@ -1386,22 +1388,19 @@ public final class Protocol {
             return null;
         } // all url's known
 
-        // extract the urlCache from the result
+        // extract the urlCache from the result; this is io-intensive;
+        // other transmissions should not be started as long as this is running
         final URIMetadataNode[] urls = new URIMetadataNode[uhs.length];
         byte[] key;
-        for ( int i = 0; i < uhs.length; i++ ) {
+        metadataRetrievalRunning = true;
+        for (int i = 0; i < uhs.length; i++) {
         	key = ASCII.getBytes(uhs[i]);
-        	if ( urlRefs.has(key) ) urls[i] = segment.fulltext().getMetadata(key);
+        	if (urlRefs.has(key)) urls[i] = segment.fulltext().getMetadata(key);
-            if ( urls[i] == null ) {
+            if (urls[i] == null) {
-                if ( Network.log.isFine() ) {
+                if (Network.log.isFine()) Network.log.fine("DEBUG transferIndex: requested url hash '" + uhs[i] + "', unknownURL='" + uhss + "'");
-                    Network.log.fine("DEBUG transferIndex: requested url hash '"
-                        + uhs[i]
-                        + "', unknownURL='"
-                        + uhss
-                        + "'");
-                }
             }
         }
+        metadataRetrievalRunning = false;
 
         in = transferURL(targetSeed, urls, gzipBody, timeout);