diff --git a/source/net/yacy/peers/Dispatcher.java b/source/net/yacy/peers/Dispatcher.java index f30b58cc5..b40404da5 100644 --- a/source/net/yacy/peers/Dispatcher.java +++ b/source/net/yacy/peers/Dispatcher.java @@ -37,6 +37,7 @@ import net.yacy.cora.federate.yacy.Distribution; import net.yacy.cora.order.Base64Order; import net.yacy.cora.storage.HandleSet; import net.yacy.cora.util.ConcurrentLog; +import net.yacy.cora.util.Memory; import net.yacy.cora.util.SpaceExceededException; import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.data.word.Word; @@ -116,7 +117,7 @@ public class Dispatcher { gzipBody, timeout); - final int concurrentSender = Math.min(32, WorkflowProcessor.availableCPU * 2); + final int concurrentSender = Math.min(8, WorkflowProcessor.availableCPU); this.indexingTransmissionProcessor = new WorkflowProcessor( "transferDocumentIndex", "This is the RWI transmission process", @@ -399,6 +400,10 @@ public class Dispatcher { */ public Transmission.Chunk transferDocumentIndex(final Transmission.Chunk chunk) { + // try to keep the system healthy; sleep as long as System load is too high + while (Protocol.metadataRetrievalRunning) try {Thread.sleep(1000);} catch (InterruptedException e) {break;} + while (Memory.load() > 2.0) try {Thread.sleep(10000);} catch (InterruptedException e) {break;} + // do the transmission final boolean success = chunk.transmit(); diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java index aa9619390..a96156aef 100644 --- a/source/net/yacy/peers/Protocol.java +++ b/source/net/yacy/peers/Protocol.java @@ -1311,6 +1311,8 @@ public final class Protocol { } } + public static boolean metadataRetrievalRunning = false; + /** * transfer the index. If the transmission fails, return a string describing the cause. If everything is * ok, return null. @@ -1386,22 +1388,19 @@ public final class Protocol { return null; } // all url's known - // extract the urlCache from the result + // extract the urlCache from the result; this is io-intensive; + // other transmissions should not be started as long as this is running final URIMetadataNode[] urls = new URIMetadataNode[uhs.length]; byte[] key; - for ( int i = 0; i < uhs.length; i++ ) { + metadataRetrievalRunning = true; + for (int i = 0; i < uhs.length; i++) { key = ASCII.getBytes(uhs[i]); - if ( urlRefs.has(key) ) urls[i] = segment.fulltext().getMetadata(key); - if ( urls[i] == null ) { - if ( Network.log.isFine() ) { - Network.log.fine("DEBUG transferIndex: requested url hash '" - + uhs[i] - + "', unknownURL='" - + uhss - + "'"); - } + if (urlRefs.has(key)) urls[i] = segment.fulltext().getMetadata(key); + if (urls[i] == null) { + if (Network.log.isFine()) Network.log.fine("DEBUG transferIndex: requested url hash '" + uhs[i] + "', unknownURL='" + uhss + "'"); } } + metadataRetrievalRunning = false; in = transferURL(targetSeed, urls, gzipBody, timeout);