From 97ba5ddbb7c3dfc54bba305870d24e13dba3531e Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Wed, 4 Feb 2015 01:12:25 +0100 Subject: [PATCH] configuration option for maxload limit for remote search --- defaults/yacy.init | 2 ++ source/net/yacy/peers/RemoteSearch.java | 13 +++++++------ source/net/yacy/search/SwitchboardConstants.java | 3 ++- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/defaults/yacy.init b/defaults/yacy.init index 6f21e0e2d..6fd4ad33c 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -865,6 +865,8 @@ search.verify.delete = true remotesearch.maxcount = 10 remotesearch.maxtime = 3000 remotesearch.result.store=true +remotesearch.maxload.rwi=8.0 +remotesearch.maxload.solr=4.0 # specifies if yacy should set it's own referer if no referer URL # was set by the client. diff --git a/source/net/yacy/peers/RemoteSearch.java b/source/net/yacy/peers/RemoteSearch.java index 69a66b5b4..fae3bae5d 100644 --- a/source/net/yacy/peers/RemoteSearch.java +++ b/source/net/yacy/peers/RemoteSearch.java @@ -147,6 +147,7 @@ public class RemoteSearch extends Thread { final SortedSet clusterselection) { // check own peer status //if (wordIndex.seedDB.mySeed() == null || wordIndex.seedDB.mySeed().getPublicAddress() == null) { return null; } + Switchboard sb = Switchboard.getSwitchboard(); // check the peer memory and lifesign-situation to get a scaling for the number of remote search processes final boolean shortmem = MemoryControl.shortStatus(); @@ -212,12 +213,12 @@ public class RemoteSearch extends Thread { final Collection robinsonPeers = DHTSelection.selectExtraTargets(event.peers, event.query.getQueryGoal().getIncludeHashes(), minage, dhtPeers, robinsoncount, random); if (event.peers != null) { - if (Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_TESTLOCAL, false)) { + if (sb.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_TESTLOCAL, false)) { dhtPeers.clear(); dhtPeers.add(event.peers.mySeed()); } - if (Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL, false)) { + if (sb.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL, false)) { robinsonPeers.clear(); robinsonPeers.add(event.peers.mySeed()); } @@ -229,20 +230,20 @@ public class RemoteSearch extends Thread { // start solr searches final int targets = dhtPeers.size() + robinsonPeers.size(); - if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_OFF, false)) { + if (!sb.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_OFF, false)) { final SolrQuery solrQuery = event.query.solrQuery(event.getQuery().contentdom, start == 0, event.excludeintext_image); for (Seed s: robinsonPeers) { - if (MemoryControl.shortStatus() || Memory.load() > 4.0) continue; + if (MemoryControl.shortStatus() || Memory.load() > sb.getConfigFloat(SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR, 4.0f)) continue; Thread t = solrRemoteSearch(event, solrQuery, start, count, s, targets, blacklist); event.nodeSearchThreads.add(t); } } // start search to YaCy DHT peers - if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_OFF, false)) { + if (!sb.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_OFF, false)) { for (Seed dhtPeer: dhtPeers) { if (dhtPeer == null || dhtPeer.hash == null) continue; - if (MemoryControl.shortStatus() || Memory.load() > 8.0) continue; + if (MemoryControl.shortStatus() || Memory.load() > sb.getConfigFloat(SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI, 8.0f)) continue; try { RemoteSearch rs = new RemoteSearch( event, diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java index 03e32c9c5..53779d0c4 100644 --- a/source/net/yacy/search/SwitchboardConstants.java +++ b/source/net/yacy/search/SwitchboardConstants.java @@ -330,8 +330,9 @@ public final class SwitchboardConstants { public static final String REMOTESEARCH_MAXTIME_DEFAULT = "network.unit.remotesearch.maxtime"; public static final String REMOTESEARCH_MAXCOUNT_USER = "remotesearch.maxcount"; public static final String REMOTESEARCH_MAXTIME_USER = "remotesearch.maxtime"; - public static final String REMOTESEARCH_RESULT_STORE = "remotesearch.result.store"; // add remote results to local index + public static final String REMOTESEARCH_MAXLOAD_RWI = "remotesearch.maxload.rwi"; + public static final String REMOTESEARCH_MAXLOAD_SOLR = "remotesearch.maxload.solr"; public static final String FEDERATED_SERVICE_SOLR_INDEXING_ENABLED = "federated.service.solr.indexing.enabled"; public static final String FEDERATED_SERVICE_SOLR_INDEXING_URL = "federated.service.solr.indexing.url";