diff --git a/htroot/Collage.html b/htroot/Collage.html old mode 100755 new mode 100644 diff --git a/htroot/Collage.java b/htroot/Collage.java old mode 100755 new mode 100644 diff --git a/htroot/IndexCleaner_p.html b/htroot/IndexCleaner_p.html old mode 100755 new mode 100644 diff --git a/htroot/IndexCleaner_p.java b/htroot/IndexCleaner_p.java old mode 100755 new mode 100644 diff --git a/htroot/api/bookmarks/posts/add_p.java b/htroot/api/bookmarks/posts/add_p.java old mode 100755 new mode 100644 diff --git a/htroot/api/bookmarks/posts/all.java b/htroot/api/bookmarks/posts/all.java old mode 100755 new mode 100644 diff --git a/htroot/api/bookmarks/posts/delete_p.java b/htroot/api/bookmarks/posts/delete_p.java old mode 100755 new mode 100644 diff --git a/htroot/api/bookmarks/posts/get.java b/htroot/api/bookmarks/posts/get.java old mode 100755 new mode 100644 diff --git a/htroot/api/bookmarks/tags/editTag_p.java b/htroot/api/bookmarks/tags/editTag_p.java old mode 100755 new mode 100644 diff --git a/htroot/api/bookmarks/tags/getTag.java b/htroot/api/bookmarks/tags/getTag.java old mode 100755 new mode 100644 diff --git a/htroot/api/bookmarks/xbel/xbel.java b/htroot/api/bookmarks/xbel/xbel.java old mode 100755 new mode 100644 diff --git a/htroot/api/feed.java b/htroot/api/feed.java old mode 100755 new mode 100644 diff --git a/htroot/api/getpageinfo_p.java b/htroot/api/getpageinfo_p.java old mode 100755 new mode 100644 diff --git a/htroot/api/queues_p.java b/htroot/api/queues_p.java old mode 100755 new mode 100644 diff --git a/htroot/api/ynetSearch.java b/htroot/api/ynetSearch.java old mode 100755 new mode 100644 diff --git a/htroot/compare_yacy.html b/htroot/compare_yacy.html old mode 100755 new mode 100644 diff --git a/htroot/compare_yacy.java b/htroot/compare_yacy.java old mode 100755 new mode 100644 diff --git a/htroot/processing/domaingraph/applet/domaingraph.java b/htroot/processing/domaingraph/applet/domaingraph.java old mode 100755 new mode 100644 diff --git a/htroot/processing/domaingraph/applet/index.html b/htroot/processing/domaingraph/applet/index.html old mode 100755 new mode 100644 diff --git a/htroot/rssTerminal.html b/htroot/rssTerminal.html old mode 100755 new mode 100644 diff --git a/htroot/terminal_p.html b/htroot/terminal_p.html old mode 100755 new mode 100644 diff --git a/source/de/anomic/crawler/CrawlQueues.java b/source/de/anomic/crawler/CrawlQueues.java index 935636d4c..1abc88867 100644 --- a/source/de/anomic/crawler/CrawlQueues.java +++ b/source/de/anomic/crawler/CrawlQueues.java @@ -198,11 +198,7 @@ public class CrawlQueues { public boolean coreCrawlJob() { - final boolean robinsonPrivateCase = (this.sb.isRobinsonMode() && - !this.sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "").equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER) && - !this.sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "").equals(SwitchboardConstants.CLUSTER_MODE_PRIVATE_CLUSTER)); - - if ((robinsonPrivateCase || coreCrawlJobSize() <= 20) && limitCrawlJobSize() > 0) { + if (coreCrawlJobSize() <= 20 && limitCrawlJobSize() > 0) { // move some tasks to the core crawl job so we have something to do final int toshift = Math.min(10, limitCrawlJobSize()); // this cannot be a big number because the balancer makes a forced waiting if it cannot balance for (int i = 0; i < toshift; i++) { @@ -402,7 +398,8 @@ public class CrawlQueues { return false; } - if (coreCrawlJobSize() > 0 /*&& sb.indexingStorageProcessor.queueSize() > 0*/) { + // Determine ratio local/remote, if lower than 1000 do not do any remote jobs + if (!isLocalRemoteRatioReached()) { if (this.log.isFine()) this.log.logFine("remoteCrawlLoaderJob: a local crawl is running, omitting processing"); return false; } @@ -645,4 +642,30 @@ public class CrawlQueues { } + /** + * To determine wether remote-crawling is omitted because of to many local crawls + * + * @return Wether remote-crawling is omitted + */ + private boolean isLocalRemoteRatioReached () { + if (remoteTriggeredCrawlJobSize() == 0) { + // No entries in remote queue + return true; + } + + // Determine ratio + float ratio = (coreCrawlJobSize() / remoteTriggeredCrawlJobSize()); + + // Debug message + Log.logInfo(LoggerNames.LOGGER_CRAWL_QUEUES, + "isLocalRemoteRatioReached: local.size() = " + localCrawlJobSize() + + ", global.size() = " + globalCrawlJobSize() + + ", remoteTriggered.size() = " + remoteTriggeredCrawlJobSize() + + ", ratio = " + new Float(ratio).toString() + ); + + // Check for local/remote ratio is below 1,000 + return (ratio <= sb.getConfigLong("crawler.localRemoteRatio", 1000)); + } + } diff --git a/source/de/anomic/crawler/NoticedURL.java b/source/de/anomic/crawler/NoticedURL.java old mode 100755 new mode 100644 diff --git a/source/de/anomic/crawler/ResultImages.java b/source/de/anomic/crawler/ResultImages.java old mode 100755 new mode 100644 diff --git a/source/de/anomic/crawler/ZURL.java b/source/de/anomic/crawler/ZURL.java old mode 100755 new mode 100644 diff --git a/source/de/anomic/crawler/retrieval/Request.java b/source/de/anomic/crawler/retrieval/Request.java old mode 100755 new mode 100644 diff --git a/source/de/anomic/crawler/retrieval/Response.java b/source/de/anomic/crawler/retrieval/Response.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/ai/example/ConnectFour.java b/source/net/yacy/ai/example/ConnectFour.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/ai/example/testorder.java b/source/net/yacy/ai/example/testorder.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/ai/greedy/AbstractFinding.java b/source/net/yacy/ai/greedy/AbstractFinding.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/ai/greedy/AbstractModel.java b/source/net/yacy/ai/greedy/AbstractModel.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/ai/greedy/Agent.java b/source/net/yacy/ai/greedy/Agent.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/ai/greedy/Asset.java b/source/net/yacy/ai/greedy/Asset.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/ai/greedy/Attempts.java b/source/net/yacy/ai/greedy/Attempts.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/ai/greedy/Battle.java b/source/net/yacy/ai/greedy/Battle.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/ai/greedy/Challenge.java b/source/net/yacy/ai/greedy/Challenge.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/ai/greedy/Context.java b/source/net/yacy/ai/greedy/Context.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/ai/greedy/Engine.java b/source/net/yacy/ai/greedy/Engine.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/ai/greedy/Finding.java b/source/net/yacy/ai/greedy/Finding.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/ai/greedy/Goal.java b/source/net/yacy/ai/greedy/Goal.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/ai/greedy/Model.java b/source/net/yacy/ai/greedy/Model.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/ai/greedy/Role.java b/source/net/yacy/ai/greedy/Role.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/cora/protocol/RequestHeader.java b/source/net/yacy/cora/protocol/RequestHeader.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/cora/protocol/ResponseHeader.java b/source/net/yacy/cora/protocol/ResponseHeader.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/kelondro/blob/ArrayStack.java b/source/net/yacy/kelondro/blob/ArrayStack.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/kelondro/blob/Heap.java b/source/net/yacy/kelondro/blob/Heap.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/kelondro/table/Relations.java b/source/net/yacy/kelondro/table/Relations.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/kelondro/util/ISO639.java b/source/net/yacy/kelondro/util/ISO639.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/peers/dht/Dispatcher.java b/source/net/yacy/peers/dht/Dispatcher.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/peers/dht/FlatWordPartitionScheme.java b/source/net/yacy/peers/dht/FlatWordPartitionScheme.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/peers/dht/PartitionScheme.java b/source/net/yacy/peers/dht/PartitionScheme.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/peers/dht/PeerSelection.java b/source/net/yacy/peers/dht/PeerSelection.java old mode 100755 new mode 100644 diff --git a/source/net/yacy/peers/dht/VerticalWordPartitionScheme.java b/source/net/yacy/peers/dht/VerticalWordPartitionScheme.java old mode 100755 new mode 100644