From dcc56318bbf2cdf14240c0d8a6d2bd2f2755533e Mon Sep 17 00:00:00 2001 From: luccioman Date: Fri, 30 Jun 2017 11:30:54 +0200 Subject: [PATCH] Made remote search max system load limits configurable from UI. As reported by davide on YaCy forums ( http://forum.yacy-websuche.de/viewtopic.php?f=23&t=6004 ) when the system is on high load, unless reading carefully YaCy configuration file, it could be difficult to understand why remote search results are not fetched. --- htroot/PerformanceQueues_p.html | 30 +++++++++++++++++++ htroot/PerformanceQueues_p.java | 24 +++++++++++++++ source/net/yacy/peers/RemoteSearch.java | 12 ++++++-- .../net/yacy/search/SwitchboardConstants.java | 10 +++++++ 4 files changed, 74 insertions(+), 2 deletions(-) diff --git a/htroot/PerformanceQueues_p.html b/htroot/PerformanceQueues_p.html index a5a9c1a52..2f32ffb64 100644 --- a/htroot/PerformanceQueues_p.html +++ b/htroot/PerformanceQueues_p.html @@ -70,6 +70,36 @@ +
+ +
Remote search requests: + + + + + + + + + + + + + + + + + + + +
TypeMaximum system loadFull Description
RWI loadSearch requests performed on remote peers distributed Reverse Word Index
Solr loadSearch requests performed on remote peers Solr indexes
+ + + Changes take effect immediately +
+
+
+
Cache Settings: diff --git a/htroot/PerformanceQueues_p.java b/htroot/PerformanceQueues_p.java index 539eec912..0b922fd74 100644 --- a/htroot/PerformanceQueues_p.java +++ b/htroot/PerformanceQueues_p.java @@ -251,6 +251,24 @@ public class PerformanceQueues_p { sb.setConfig(SwitchboardConstants.WORDCACHE_MAX_COUNT, Integer.toString(wordCacheMaxCount)); if (rwi != null) rwi.setBufferMaxWordCount(wordCacheMaxCount); } + + /* Setting remote searches max loads */ + if (post != null) { + if(post.containsKey("setRemoteSearchLoads")) { + float loadValue = post.getFloat("remoteSearchRWIMaxLoad", + sb.getConfigFloat(SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI, + SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI_DEFAULT)); + sb.setConfig(SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI, loadValue); + + loadValue = post.getFloat("remoteSearchSolrMaxLoad", + sb.getConfigFloat(SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR, + SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR_DEFAULT)); + sb.setConfig(SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR, loadValue); + } else if(post.containsKey("defaultRemoteSearchLoads")) { + sb.setConfig(SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI, SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI_DEFAULT); + sb.setConfig(SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR, SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR_DEFAULT); + } + } if ((post != null) && (post.containsKey("poolConfig"))) { @@ -323,6 +341,12 @@ public class PerformanceQueues_p { prop.put("pool_2_numActive", ConnectionInfo.getServerCount()); prop.put("pool", "3"); + + /* Remote searches max loads settings */ + prop.put("remoteSearchRWIMaxLoad", sb.getConfigFloat(SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI, + SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI_DEFAULT)); + prop.put("remoteSearchSolrMaxLoad", sb.getConfigFloat(SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR, + SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR_DEFAULT)); // parse initialization memory settings final String Xmx = sb.getConfig("javastart_Xmx", "Xmx600m").substring(3); diff --git a/source/net/yacy/peers/RemoteSearch.java b/source/net/yacy/peers/RemoteSearch.java index 09bace366..595c4306c 100644 --- a/source/net/yacy/peers/RemoteSearch.java +++ b/source/net/yacy/peers/RemoteSearch.java @@ -259,7 +259,11 @@ public class RemoteSearch extends Thread { if (!sb.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_OFF, false)) { final SolrQuery solrQuery = event.query.solrQuery(event.getQuery().contentdom, start == 0, event.excludeintext_image); for (Seed s: robinsonPeers) { - if (MemoryControl.shortStatus() || Memory.load() > sb.getConfigFloat(SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR, 4.0f)) continue; + if (MemoryControl.shortStatus() + || Memory.load() > sb.getConfigFloat(SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR, + SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR_DEFAULT)) { + continue; + } Thread t = solrRemoteSearch(event, solrQuery, start, count, s, targets, blacklist); event.nodeSearchThreads.add(t); } @@ -269,7 +273,11 @@ public class RemoteSearch extends Thread { if (!sb.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_OFF, false)) { for (Seed dhtPeer: dhtPeers) { if (dhtPeer == null || dhtPeer.hash == null) continue; - if (MemoryControl.shortStatus() || Memory.load() > sb.getConfigFloat(SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI, 8.0f)) continue; + if (MemoryControl.shortStatus() + || Memory.load() > sb.getConfigFloat(SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI, + SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI_DEFAULT)) { + continue; + } try { RemoteSearch rs = new RemoteSearch( event, diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java index 098403711..af90841bd 100644 --- a/source/net/yacy/search/SwitchboardConstants.java +++ b/source/net/yacy/search/SwitchboardConstants.java @@ -298,9 +298,19 @@ public final class SwitchboardConstants { public static final String REMOTESEARCH_RESULT_STORE = "remotesearch.result.store"; // add remote results to local index /** Maximum size allowed (in kbytes) for a remote document result to be stored to local index */ public static final String REMOTESEARCH_RESULT_STORE_MAXSIZE= "remotesearch.result.store.maxsize"; + + /** Setting key to configure the maximum system load allowing remote RWI searches */ public static final String REMOTESEARCH_MAXLOAD_RWI = "remotesearch.maxload.rwi"; + + /** Default maximum system load allowing remote RWI searches */ + public static final float REMOTESEARCH_MAXLOAD_RWI_DEFAULT = 8.0f; + + /** Setting key to configure the maximum system load allowing remote Solr searches */ public static final String REMOTESEARCH_MAXLOAD_SOLR = "remotesearch.maxload.solr"; + /** Default maximum system load allowing remote Solr searches */ + public static final float REMOTESEARCH_MAXLOAD_SOLR_DEFAULT = 4.0f; + /** * Setting key to configure whether responses from remote Solr instances * should be binary encoded :