From 185765198887d3610258586ab22f0d0c6438eaee Mon Sep 17 00:00:00 2001 From: luccioman Date: Thu, 9 Feb 2017 11:05:06 +0100 Subject: [PATCH] Added a new Debug/Analysis advanced settings subsection. As discussed in PR #93 with @JeremyRand and @reger24 this new advanced settings page includes: - a new setting to control remote Solr responses encoding - some existing debug settings which could not be set through the admin user interface --- defaults/yacy.init | 3 + htroot/SettingsAck_p.html | 4 +- htroot/SettingsAck_p.java | 34 +++++ htroot/Settings_Debug.inc | 125 ++++++++++++++++++ htroot/Settings_p.html | 5 +- htroot/Settings_p.java | 16 +++ .../federate/SolrFederateSearchConnector.java | 10 +- .../solr/instance/InstanceMirror.java | 10 +- source/net/yacy/peers/Protocol.java | 8 +- source/net/yacy/search/AutoSearch.java | 5 +- .../net/yacy/search/SwitchboardConstants.java | 42 ++++-- source/net/yacy/search/index/Fulltext.java | 9 +- 12 files changed, 254 insertions(+), 17 deletions(-) create mode 100644 htroot/Settings_Debug.inc diff --git a/defaults/yacy.init b/defaults/yacy.init index cb9a95c04..6d978fc68 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -890,6 +890,9 @@ remotesearch.result.store.maxsize=-1 remotesearch.maxload.rwi=8.0 remotesearch.maxload.solr=4.0 +# Control whether remote Solr instances responses should be binary encoded. Responses are transferred as XML when set to false. +remote.solr.binaryResponse.enabled=true + # specifies if yacy should set it's own referer if no referer URL # was set by the client. useYacyReferer = false diff --git a/htroot/SettingsAck_p.html b/htroot/SettingsAck_p.html index e0e190029..bf4ca7759 100644 --- a/htroot/SettingsAck_p.html +++ b/htroot/SettingsAck_p.html @@ -190,7 +190,9 @@

the change will take effect after restart.

Note: the SSL option must be switched on, see Basic Configuration

:: -

URL Proxy settings have been saved.

+

URL Proxy settings have been saved.

+ :: +

Debug/Analysis settings have been saved.

#(/info)#

#(needsRestart)# diff --git a/htroot/SettingsAck_p.java b/htroot/SettingsAck_p.java index a377574cc..7facc08fc 100644 --- a/htroot/SettingsAck_p.java +++ b/htroot/SettingsAck_p.java @@ -42,6 +42,7 @@ import net.yacy.peers.Seed; import net.yacy.peers.operation.yacySeedUploader; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; +import net.yacy.search.query.SearchEventCache; import net.yacy.server.serverCore; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -512,6 +513,39 @@ public class SettingsAck_p { prop.put("info", "32"); return prop; } + + // Debug/Analysis settings + if (post.containsKey("debugAnalysisSettings")) { + boolean tickedCheckbox = post.containsKey("solrBinaryResponse"); + env.setConfig(SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED, tickedCheckbox); + + tickedCheckbox = post.containsKey("searchTestLocalDHT"); + env.setConfig(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_TESTLOCAL, tickedCheckbox); + + tickedCheckbox = post.containsKey("searchTestLocalSolr"); + env.setConfig(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL, tickedCheckbox); + + /* For easier user understanding, the following flags controlling data sources selection + * are rendered in the UI as checkboxes corresponding to enabled value when ticked */ + tickedCheckbox = post.containsKey("searchLocalDHT"); + env.setConfig(SwitchboardConstants.DEBUG_SEARCH_LOCAL_DHT_OFF, !tickedCheckbox); + + tickedCheckbox = post.containsKey("searchLocalSolr"); + env.setConfig(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, !tickedCheckbox); + + tickedCheckbox = post.containsKey("searchRemoteDHT"); + env.setConfig(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_OFF, !tickedCheckbox); + + tickedCheckbox = post.containsKey("searchRemoteSolr"); + env.setConfig(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_OFF, !tickedCheckbox); + + /* Let's clean up all search events as these settings affect how search is performed and we don't + * want cached results obtained with the previous settings */ + SearchEventCache.cleanupEvents(true); + + prop.put("info", "34"); + return prop; + } // nothing made prop.put("info", "1");//no information submitted diff --git a/htroot/Settings_Debug.inc b/htroot/Settings_Debug.inc new file mode 100644 index 000000000..b30b158bd --- /dev/null +++ b/htroot/Settings_Debug.inc @@ -0,0 +1,125 @@ +

Debug/Analysis Settings

+ + + +
+
+ Solr communication + +
+
+
+ +
+
+
+ When checked (default), responses from remote Solr index instances are transfered using an efficient binary data format. + When unchecked, responses are transferred as XML, + which can be captured and parsed by any external XML aware tool for debug/analysis. +
+
+
+ +
+ Search data sources +

By default all data sources are enabled to obtain search results, + but you can here disable one or more ones to check the behavior of the process.

+ +
+
+
+ +
+
+
+ +
+
+
+ +
+
+
+ +
+
+
+ +
+
+
+ +
+
+
+ +
+
+
+
+ +
+ Search testing tweaks + +
+
+
+ +
+
+
+ When checked, the remote DHT peers selection is overriden and only the local peer is selected to provide remote DHT search results. +
+
+ +
+
+
+ +
+
+
+ When checked, the remote Solr peers selection is overriden and only this peer is selected to provide remote Solr search results. +
+
+ +
+ +
+ + Changes will take effect immediately. +
+
\ No newline at end of file diff --git a/htroot/Settings_p.html b/htroot/Settings_p.html index 993f6f5ea..618eb4e49 100644 --- a/htroot/Settings_p.html +++ b/htroot/Settings_p.html @@ -1,5 +1,5 @@ - - + + YaCy '#[clientname]#': Advanced Settings #%env/templates/metas.template%# @@ -27,6 +27,7 @@
  • Transparent Proxy Access Settings
  • URL/Web Proxy Access Settings
  • Remote Proxy (optional)
    • +
  • Debug/Analysis Settings
    • diff --git a/htroot/Settings_p.java b/htroot/Settings_p.java index 028708a34..92816c86e 100644 --- a/htroot/Settings_p.java +++ b/htroot/Settings_p.java @@ -72,6 +72,8 @@ public final class Settings_p { } else if (page.equals("crawler")) { prop.put("settingsTables", "Settings_Crawler.inc"); + } else if (page.equals("debug")) { + prop.put("settingsTables", "Settings_Debug.inc"); } else { prop.put("settingsTables", ""); } @@ -196,6 +198,20 @@ public final class Settings_p { prop.put("server.https",sb.getConfigBool("server.https", false)); prop.put("server.https_port.ssl", sb.getConfig("port.ssl","8443")); + // debug/analysis + prop.put("solrBinaryResponseChecked", env.getConfigBool(SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED, + SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT) ? 1 : 0); + + /* For easier user understanding, the following flags controlling data sources selection + * are rendered in the UI as checkboxes corresponding to enabled value when ticked */ + prop.put("searchLocalDHTChecked", !env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_DHT_OFF, false) ? 1 : 0); + prop.put("searchLocalSolrChecked", !env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false) ? 1 : 0); + prop.put("searchRemoteDHTChecked", !env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_OFF, false) ? 1 : 0); + prop.put("searchRemoteSolrChecked", !env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_OFF, false) ? 1 : 0); + + prop.put("searchTestLocalDHTChecked", env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_TESTLOCAL, false) ? 1 : 0); + prop.put("searchTestLocalSolrChecked", env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL, false) ? 1 : 0); + // return rewrite properties return prop; } diff --git a/source/net/yacy/cora/federate/SolrFederateSearchConnector.java b/source/net/yacy/cora/federate/SolrFederateSearchConnector.java index 1a134a0fb..df57b5f62 100644 --- a/source/net/yacy/cora/federate/SolrFederateSearchConnector.java +++ b/source/net/yacy/cora/federate/SolrFederateSearchConnector.java @@ -29,6 +29,8 @@ import net.yacy.cora.federate.solr.connector.SolrConnector; import net.yacy.cora.federate.solr.instance.RemoteInstance; import net.yacy.cora.util.ConcurrentLog; import net.yacy.kelondro.data.meta.URIMetadataNode; +import net.yacy.search.Switchboard; +import net.yacy.search.SwitchboardConstants; import net.yacy.search.query.QueryParams; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.common.SolrDocument; @@ -96,7 +98,13 @@ public class SolrFederateSearchConnector extends AbstractFederateSearchConnector try { RemoteInstance instance = new RemoteInstance(baseurl, remotecorename, corename, 20000); try { - SolrConnector solrConnector = new RemoteSolrConnector(instance, false, null); + boolean useBinaryResponseWriter = SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT; + if (Switchboard.getSwitchboard() != null) { + useBinaryResponseWriter = Switchboard.getSwitchboard().getConfigBool( + SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED, + SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT); + } + SolrConnector solrConnector = new RemoteSolrConnector(instance, useBinaryResponseWriter); try { this.lastaccesstime = System.currentTimeMillis(); SolrDocumentList docList = solrConnector.getDocumentListByParams(msp); diff --git a/source/net/yacy/cora/federate/solr/instance/InstanceMirror.java b/source/net/yacy/cora/federate/solr/instance/InstanceMirror.java index 6e0be5c11..317382e96 100644 --- a/source/net/yacy/cora/federate/solr/instance/InstanceMirror.java +++ b/source/net/yacy/cora/federate/solr/instance/InstanceMirror.java @@ -33,6 +33,8 @@ import net.yacy.cora.federate.solr.connector.MirrorSolrConnector; import net.yacy.cora.federate.solr.connector.RemoteSolrConnector; import net.yacy.cora.federate.solr.connector.SolrConnector; import net.yacy.kelondro.util.MemoryControl; +import net.yacy.search.Switchboard; +import net.yacy.search.SwitchboardConstants; public class InstanceMirror { @@ -157,7 +159,13 @@ public class InstanceMirror { if (this.remoteSolrInstance == null) return null; RemoteSolrConnector rsc = this.remoteConnectorCache.get(corename); if (rsc != null) return rsc; - rsc = new RemoteSolrConnector(this.remoteSolrInstance, true, corename); + boolean useBinaryResponseWriter = SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT; + if (Switchboard.getSwitchboard() != null) { + useBinaryResponseWriter = Switchboard.getSwitchboard().getConfigBool( + SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED, + SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT); + } + rsc = new RemoteSolrConnector(this.remoteSolrInstance, useBinaryResponseWriter, corename); this.remoteConnectorCache.put(corename, rsc); return rsc; } diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java index d0e1b72fe..fbf7e35f4 100644 --- a/source/net/yacy/peers/Protocol.java +++ b/source/net/yacy/peers/Protocol.java @@ -1077,7 +1077,13 @@ public final class Protocol { try { this.instance = new RemoteInstance("http://" + this.address, null, "solr", this.timeout); // this is a 'patch configuration' which considers 'solr' as default collection try { - this.solrConnector = new RemoteSolrConnector(this.instance, this.mySeed ? true : this.target.getVersion() >= 1.63, "solr"); + boolean useBinaryResponseWriter = SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT; + if (Switchboard.getSwitchboard() != null) { + useBinaryResponseWriter = Switchboard.getSwitchboard().getConfigBool( + SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED, + SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT); + } + this.solrConnector = new RemoteSolrConnector(this.instance, useBinaryResponseWriter && (this.mySeed ? true : this.target.getVersion() >= 1.63), "solr"); if (!solrConnector.isClosed() && !this.closed) { try { this.rsp[0] = this.solrConnector.getResponseByParams(solrQuery); diff --git a/source/net/yacy/search/AutoSearch.java b/source/net/yacy/search/AutoSearch.java index 5e88f53c9..49c4d27b4 100644 --- a/source/net/yacy/search/AutoSearch.java +++ b/source/net/yacy/search/AutoSearch.java @@ -250,7 +250,10 @@ public class AutoSearch extends AbstractBusyThread { try { RemoteInstance instance = new RemoteInstance("http://" + seed.getPublicAddress(seed.getIP()) + "/solr/", null, null, 10000); // this is a 'patch configuration' which considers 'solr' as default collection try { - SolrConnector solrConnector = new RemoteSolrConnector(instance, true, null); + SolrConnector solrConnector = new RemoteSolrConnector(instance, + sb.getConfigBool(SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED, + SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT), + null); if (!solrConnector.isClosed()) { try { QueryResponse rsp = solrConnector.getResponseByParams(solrQuery); diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java index 3a20a4682..3a506479c 100644 --- a/source/net/yacy/search/SwitchboardConstants.java +++ b/source/net/yacy/search/SwitchboardConstants.java @@ -286,6 +286,20 @@ public final class SwitchboardConstants { public static final String REMOTESEARCH_RESULT_STORE_MAXSIZE= "remotesearch.result.store.maxsize"; public static final String REMOTESEARCH_MAXLOAD_RWI = "remotesearch.maxload.rwi"; public static final String REMOTESEARCH_MAXLOAD_SOLR = "remotesearch.maxload.solr"; + + /** + * Setting key to configure whether responses from remote Solr instances + * should be binary encoded : + * + */ + public static final String REMOTE_SOLR_BINARY_RESPONSE_ENABLED = "remote.solr.binaryResponse.enabled"; + + /** Default configuration setting for remote Solr responses binary encoding */ + public static final boolean REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT = true; public static final String FEDERATED_SERVICE_SOLR_INDEXING_ENABLED = "federated.service.solr.indexing.enabled"; public static final String FEDERATED_SERVICE_SOLR_INDEXING_URL = "federated.service.solr.indexing.url"; @@ -314,15 +328,25 @@ public final class SwitchboardConstants { public static final String CRAWLER_USER_AGENT_MINIMUMDELTA = "crawler.userAgent.minimumdelta"; public static final String CRAWLER_USER_AGENT_CLIENTTIMEOUT = "crawler.userAgent.clienttimeout"; - /** - * debug flags - */ - public static final String DEBUG_SEARCH_LOCAL_DHT_OFF = "debug.search.local.dht.off"; // =true: do not use the local dht/rwi index (which is not done if we do remote searches) - public static final String DEBUG_SEARCH_LOCAL_SOLR_OFF = "debug.search.local.solr.off"; // =true: do not use solr - public static final String DEBUG_SEARCH_REMOTE_DHT_OFF = "debug.search.remote.dht.off"; // =true: do not use dht/rwi - public static final String DEBUG_SEARCH_REMOTE_DHT_TESTLOCAL= "debug.search.remote.dht.testlocal"; // =true: do not use dht, search local peer in a shortcut to the own server - public static final String DEBUG_SEARCH_REMOTE_SOLR_OFF = "debug.search.remote.solr.off"; // =true: do not use solr - public static final String DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL= "debug.search.remote.solr.testlocal"; // =true: do not use dht, search local peer in a shortcut to the own server + /* --- debug flags --- */ + + /** when set to true : do not use the local dht/rwi index (which is not done if we do remote searches) */ + public static final String DEBUG_SEARCH_LOCAL_DHT_OFF = "debug.search.local.dht.off"; + + /** when set to true : do not use local solr index */ + public static final String DEBUG_SEARCH_LOCAL_SOLR_OFF = "debug.search.local.solr.off"; + + /** when set to true : do not use remote dht/rwi */ + public static final String DEBUG_SEARCH_REMOTE_DHT_OFF = "debug.search.remote.dht.off"; + + /** when set to true : do not use remote solr indexes */ + public static final String DEBUG_SEARCH_REMOTE_SOLR_OFF = "debug.search.remote.solr.off"; + + /** when set to true : do not use dht, search local peer in a shortcut to the own server */ + public static final String DEBUG_SEARCH_REMOTE_DHT_TESTLOCAL= "debug.search.remote.dht.testlocal"; + + /** when set to true : do not use dht, search local peer in a shortcut to the own server */ + public static final String DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL= "debug.search.remote.solr.testlocal"; /** *

    public static final String WORDCACHE_MAX_COUNT = "wordCacheMaxCount"

    diff --git a/source/net/yacy/search/index/Fulltext.java b/source/net/yacy/search/index/Fulltext.java index 4cf3d9d1c..3d6a79da1 100644 --- a/source/net/yacy/search/index/Fulltext.java +++ b/source/net/yacy/search/index/Fulltext.java @@ -71,6 +71,7 @@ import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.kelondro.data.word.WordReferenceVars; import net.yacy.kelondro.util.MemoryControl; import net.yacy.search.Switchboard; +import net.yacy.search.SwitchboardConstants; import net.yacy.search.schema.CollectionConfiguration; import net.yacy.search.schema.CollectionSchema; import net.yacy.search.schema.WebgraphConfiguration; @@ -178,7 +179,13 @@ public final class Fulltext { public RemoteSolrConnector getDefaultRemoteSolrConnector() { try { - return this.solrInstances.getDefaultRemoteConnector(true); + boolean useBinaryResponseWriter = SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT; + if (Switchboard.getSwitchboard() != null) { + useBinaryResponseWriter = Switchboard.getSwitchboard().getConfigBool( + SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED, + SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT); + } + return this.solrInstances.getDefaultRemoteConnector(useBinaryResponseWriter); } catch (IOException e) { return null; }