boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Made remote search max system load limits configurable from UI.

Browse Source 
As reported by davide on YaCy forums (
http://forum.yacy-websuche.de/viewtopic.php?f=23&t=6004 ) when the
system is on high load, unless reading carefully YaCy configuration
file, it could be difficult to understand why remote search results are
not fetched.
pull/127/head
luccioman 8 years ago
parent ddd13b776d
commit dcc56318bb
4 changed files with 74 additions and 2 deletions
Show Stats Download Patch File Download Diff File

30
htroot/PerformanceQueues_p.html
Unescape View File

@ -70,6 +70,36 @@
</fieldset>
</form>
<form action="PerformanceQueues_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8">
<input type="hidden" name="transactionToken" value="#[transactionToken]#" />
<fieldset><legend>Remote search requests:</legend>
<table style="border-width:0px;">
<tr class="TableHeader" style="vertical-align:bottom;">
<th style="padding:0.4em;">Type</th>
<th style="padding:0.4em;" title="When the system load average is over the specified value, that type of remote search request is not used to fill search results.">Maximum system load</th>
<th style="padding:0.4em;">Full Description</th>
</tr>
<tr class="TableCellDark">
<td align="left"><abbr title="Reverse Word Index">RWI</abbr></td>
<td align="right"><input name="remoteSearchRWIMaxLoad" size="7" maxlength="7" value="#[remoteSearchRWIMaxLoad]#"/> load</td>
<td align="left">Search requests performed on remote peers distributed Reverse Word Index</td>
</tr>
<tr class="TableCellDark">
<td align="left">Solr</td>
<td align="right"><input name="remoteSearchSolrMaxLoad" size="7" maxlength="7" value="#[remoteSearchSolrMaxLoad]#"/> load</td>
<td align="left">Search requests performed on remote peers Solr indexes</td>
</tr>
<tr class="TableCellLight">
<td align="left" colspan="19">
<input type="submit" name="setRemoteSearchLoads" class="btn btn-sm btn-primary" value="Submit New Values" />
<input type="submit" name="defaultRemoteSearchLoads" class="btn btn-sm btn-primary" value="Re-set to default" />
Changes take effect immediately
</td>
</tr>
</table>
</fieldset>
</form>
<form action="PerformanceQueues_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8">
<input type="hidden" name="transactionToken" value="#[transactionToken]#" />
<fieldset><legend>Cache Settings:</legend>

24
htroot/PerformanceQueues_p.java
Unescape View File

@ -251,6 +251,24 @@ public class PerformanceQueues_p {
sb.setConfig(SwitchboardConstants.WORDCACHE_MAX_COUNT, Integer.toString(wordCacheMaxCount));
if (rwi != null) rwi.setBufferMaxWordCount(wordCacheMaxCount);
}
/* Setting remote searches max loads */
if (post != null) {
if(post.containsKey("setRemoteSearchLoads")) {
float loadValue = post.getFloat("remoteSearchRWIMaxLoad",
sb.getConfigFloat(SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI,
SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI_DEFAULT));
sb.setConfig(SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI, loadValue);
loadValue = post.getFloat("remoteSearchSolrMaxLoad",
sb.getConfigFloat(SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR,
SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR_DEFAULT));
sb.setConfig(SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR, loadValue);
} else if(post.containsKey("defaultRemoteSearchLoads")) {
sb.setConfig(SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI, SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI_DEFAULT);
sb.setConfig(SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR, SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR_DEFAULT);
}
}
if ((post != null) && (post.containsKey("poolConfig"))) {
@ -323,6 +341,12 @@ public class PerformanceQueues_p {
prop.put("pool_2_numActive", ConnectionInfo.getServerCount());
prop.put("pool", "3");
/* Remote searches max loads settings */
prop.put("remoteSearchRWIMaxLoad", sb.getConfigFloat(SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI,
SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI_DEFAULT));
prop.put("remoteSearchSolrMaxLoad", sb.getConfigFloat(SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR,
SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR_DEFAULT));
// parse initialization memory settings
final String Xmx = sb.getConfig("javastart_Xmx", "Xmx600m").substring(3);

12
source/net/yacy/peers/RemoteSearch.java
Unescape View File

@ -259,7 +259,11 @@ public class RemoteSearch extends Thread {
if (!sb.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_OFF, false)) {
final SolrQuery solrQuery = event.query.solrQuery(event.getQuery().contentdom, start == 0, event.excludeintext_image);
for (Seed s: robinsonPeers) {
if (MemoryControl.shortStatus() || Memory.load() > sb.getConfigFloat(SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR, 4.0f)) continue;
if (MemoryControl.shortStatus()
|| Memory.load() > sb.getConfigFloat(SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR,
SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR_DEFAULT)) {
continue;
}
Thread t = solrRemoteSearch(event, solrQuery, start, count, s, targets, blacklist);
event.nodeSearchThreads.add(t);
}
@ -269,7 +273,11 @@ public class RemoteSearch extends Thread {
if (!sb.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_OFF, false)) {
for (Seed dhtPeer: dhtPeers) {
if (dhtPeer == null || dhtPeer.hash == null) continue;
if (MemoryControl.shortStatus() || Memory.load() > sb.getConfigFloat(SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI, 8.0f)) continue;
if (MemoryControl.shortStatus()
|| Memory.load() > sb.getConfigFloat(SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI,
SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI_DEFAULT)) {
continue;
}
try {
RemoteSearch rs = new RemoteSearch(
event,

10
source/net/yacy/search/SwitchboardConstants.java
Unescape View File

@ -298,9 +298,19 @@ public final class SwitchboardConstants {
public static final String REMOTESEARCH_RESULT_STORE = "remotesearch.result.store"; // add remote results to local index
/** Maximum size allowed (in kbytes) for a remote document result to be stored to local index */
public static final String REMOTESEARCH_RESULT_STORE_MAXSIZE= "remotesearch.result.store.maxsize";
/** Setting key to configure the maximum system load allowing remote RWI searches */
public static final String REMOTESEARCH_MAXLOAD_RWI = "remotesearch.maxload.rwi";
/** Default maximum system load allowing remote RWI searches */
public static final float REMOTESEARCH_MAXLOAD_RWI_DEFAULT = 8.0f;
/** Setting key to configure the maximum system load allowing remote Solr searches */
public static final String REMOTESEARCH_MAXLOAD_SOLR = "remotesearch.maxload.solr";
/** Default maximum system load allowing remote Solr searches */
public static final float REMOTESEARCH_MAXLOAD_SOLR_DEFAULT = 4.0f;
/**
* Setting key to configure whether responses from remote Solr instances
* should be binary encoded :

Write Preview
Loading…
Cancel
Save