From b1140e3d827d1a2c2bf60debf787e2aeef82d52c Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 5 Mar 2013 12:19:32 +0100 Subject: [PATCH] added debug switches for detailed search testing --- defaults/yacy.init | 10 ++- source/net/yacy/peers/Protocol.java | 9 ++- source/net/yacy/peers/RemoteSearch.java | 65 ++++++++++--------- .../net/yacy/search/SwitchboardConstants.java | 13 +++- source/net/yacy/search/query/SearchEvent.java | 18 +++-- 5 files changed, 75 insertions(+), 40 deletions(-) diff --git a/defaults/yacy.init b/defaults/yacy.init index d9f5738b1..038ac0eeb 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -445,9 +445,13 @@ seedScpPath= # of the period here (minutes) peerCycle=2 -# Debug mode for YACY network: this will trigger that also local ip's are -# accepted as peer addresses -yacyDebugMode=false +# debug flags +debug.search.local.dht.on=false +debug.search.local.solr.off=false +debug.search.remote.dht.off=false +debug.search.remote.dht.testlocal=false +debug.search.remote.solr.off=false +debug.search.remote.solr.testlocal=false #staticIP if you have a static IP, you can use this setting staticIP= diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java index dc9ebcdf9..824aa8a10 100644 --- a/source/net/yacy/peers/Protocol.java +++ b/source/net/yacy/peers/Protocol.java @@ -1020,7 +1020,7 @@ public final class Protocol { final int offset, final int count, boolean getFacets, - final Seed target, + Seed target, final Blacklist blacklist) { if (event.query.getQueryGoal().getOriginalQueryString(false) == null || event.query.getQueryGoal().getOriginalQueryString(false).length() == 0) { @@ -1051,6 +1051,10 @@ public final class Protocol { for (CollectionSchema field: snippetFields) solrQuery.addHighlightField(field.getSolrFieldName()); boolean localsearch = target == null || target.equals(event.peers.mySeed()); + if (localsearch && Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL, false)) { + target = event.peers.mySeed(); + localsearch = false; + } SolrDocumentList docList = null; QueryResponse rsp = null; if (localsearch) { @@ -1064,7 +1068,8 @@ public final class Protocol { } } else { try { - RemoteInstance instance = new RemoteInstance("http://" + target.getPublicAddress(), null, "solr"); // this is a 'patch configuration' which considers 'solr' as default collection + String address = target == event.peers.mySeed() ? "localhost:" + target.getPort() : target.getPublicAddress(); + RemoteInstance instance = new RemoteInstance("http://" + address, null, "solr"); // this is a 'patch configuration' which considers 'solr' as default collection SolrConnector solrConnector = new RemoteSolrConnector(instance, "solr"); rsp = solrConnector.query(solrQuery); docList = rsp.getResults(); diff --git a/source/net/yacy/peers/RemoteSearch.java b/source/net/yacy/peers/RemoteSearch.java index 054b1f0f4..0aab7bf86 100644 --- a/source/net/yacy/peers/RemoteSearch.java +++ b/source/net/yacy/peers/RemoteSearch.java @@ -34,6 +34,8 @@ import net.yacy.cora.document.ASCII; import net.yacy.cora.storage.HandleSet; import net.yacy.kelondro.logging.Log; import net.yacy.repository.Blacklist; +import net.yacy.search.Switchboard; +import net.yacy.search.SwitchboardConstants; import net.yacy.search.query.QueryParams; import net.yacy.search.query.SearchEvent; import net.yacy.search.query.SecondarySearchSuperviser; @@ -162,39 +164,44 @@ public class RemoteSearch extends Thread { nodePeers.add(s); } } - - // for debugging: remove all dht peer to see if solr is working properly - //dhtPeers.clear(); // FOR DEBUGGING ONLY!!! // start solr searches - for (Seed s: nodePeers) { - solrRemoteSearch(event, start, count, s, blacklist); + if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_OFF, false)) { + for (Seed s: nodePeers) { + solrRemoteSearch(event, start, count, s, blacklist); + } + } + + if (Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_TESTLOCAL, false)) { + dhtPeers.clear(); + dhtPeers.add(event.peers.mySeed()); } - // start search to YaCy DHT peers - final int targets = dhtPeers.size(); - if (targets == 0) return; - for (int i = 0; i < targets; i++) { - if (dhtPeers.get(i) == null || dhtPeers.get(i).hash == null) continue; - try { - RemoteSearch rs = new RemoteSearch( - event, - QueryParams.hashSet2hashString(event.query.getQueryGoal().getIncludeHashes()), - QueryParams.hashSet2hashString(event.query.getQueryGoal().getExcludeHashes()), - event.query.targetlang == null ? "" : event.query.targetlang, - event.query.contentdom == null ? "all" : event.query.contentdom.toString(), - count, - time, - event.query.maxDistance, - targets, - dhtPeers.get(i), - event.secondarySearchSuperviser, - blacklist); - rs.start(); - event.primarySearchThreadsL.add(rs); - } catch (final OutOfMemoryError e) { - Log.logException(e); - break; + if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_OFF, false)) { + final int targets = dhtPeers.size(); + if (targets == 0) return; + for (int i = 0; i < targets; i++) { + if (dhtPeers.get(i) == null || dhtPeers.get(i).hash == null) continue; + try { + RemoteSearch rs = new RemoteSearch( + event, + QueryParams.hashSet2hashString(event.query.getQueryGoal().getIncludeHashes()), + QueryParams.hashSet2hashString(event.query.getQueryGoal().getExcludeHashes()), + event.query.targetlang == null ? "" : event.query.targetlang, + event.query.contentdom == null ? "all" : event.query.contentdom.toString(), + count, + time, + event.query.maxDistance, + targets, + dhtPeers.get(i), + event.secondarySearchSuperviser, + blacklist); + rs.start(); + event.primarySearchThreadsL.add(rs); + } catch (final OutOfMemoryError e) { + Log.logException(e); + break; + } } } } diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java index 042751a4d..ad43dec6b 100644 --- a/source/net/yacy/search/SwitchboardConstants.java +++ b/source/net/yacy/search/SwitchboardConstants.java @@ -324,8 +324,17 @@ public final class SwitchboardConstants { public static final String CRAWLER_THREADS_ACTIVE_MAX = "crawler.MaxActiveThreads"; public static final String CRAWLER_FOLLOW_REDIRECTS = "crawler.http.FollowRedirects"; // ignore the target url and follow to the redirect public static final String CRAWLER_RECORD_REDIRECTS = "crawler.http.RecordRedirects"; // record the ignored redirected page to the index store - public static final String YACY_MODE_DEBUG = "yacyDebugMode"; - + + /** + * debug flags + */ + public static final String DEBUG_SEARCH_LOCAL_DHT_ON = "debug.search.local.dht.on"; // =true: use the local dht/rwi index (which is not done if we do remote searches) + public static final String DEBUG_SEARCH_LOCAL_SOLR_OFF = "debug.search.local.solr.off"; // =true: do not use solr + public static final String DEBUG_SEARCH_REMOTE_DHT_OFF = "debug.search.remote.dht.off"; // =true: do not use dht/rwi + public static final String DEBUG_SEARCH_REMOTE_DHT_TESTLOCAL= "debug.search.remote.dht.testlocal"; // =true: do not use dht, search local peer in a shortcut to the own server + public static final String DEBUG_SEARCH_REMOTE_SOLR_OFF = "debug.search.remote.solr.off"; // =true: do not use solr + public static final String DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL= "debug.search.remote.solr.testlocal"; // =true: do not use dht, search local peer in a shortcut to the own server + /** *

public static final String WORDCACHE_MAX_COUNT = "wordCacheMaxCount"

*

Name of the setting how many words the word-cache (or DHT-Out cache) shall contain maximal. Indexing pages if the diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.java index 0ca62e5cc..3e82650de 100644 --- a/source/net/yacy/search/query/SearchEvent.java +++ b/source/net/yacy/search/query/SearchEvent.java @@ -86,6 +86,7 @@ import net.yacy.repository.LoaderDispatcher; import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.search.EventTracker; import net.yacy.search.Switchboard; +import net.yacy.search.SwitchboardConstants; import net.yacy.search.index.Segment; import net.yacy.search.ranking.ReferenceOrder; import net.yacy.search.schema.CollectionSchema; @@ -256,12 +257,17 @@ public final class SearchEvent { } // start a local solr search - this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, 0, this.query.itemsPerPage, null /*this peer*/, Switchboard.urlBlacklist); + if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) { + this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, 0, this.query.itemsPerPage, null /*this peer*/, Switchboard.urlBlacklist); + } this.localsolroffset = this.query.itemsPerPage; // start a local RWI search concurrently this.rwiProcess = null; - if (query.getSegment().connectedRWI() && (!this.remote || this.peers.mySeed().getBirthdate() < noRobinsonLocalRWISearch)) { + if (query.getSegment().connectedRWI() && ( + !this.remote || + this.peers.mySeed().getBirthdate() < noRobinsonLocalRWISearch) || + Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_DHT_ON, false)) { // we start the local search only if this peer is doing a remote search or when it is doing a local search and the peer is old rwiProcess = new RWIProcess(); rwiProcess.start(); @@ -1248,7 +1254,9 @@ public final class SearchEvent { // load remaining solr results now int nextitems = item - this.localsolroffset + this.query.itemsPerPage; // example: suddenly switch to item 60, just 10 had been shown, 20 loaded. if (this.localsolrsearch.isAlive()) {try {this.localsolrsearch.join();} catch (InterruptedException e) {}} - this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.localsolroffset, nextitems, null /*this peer*/, Switchboard.urlBlacklist); + if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) { + this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.localsolroffset, nextitems, null /*this peer*/, Switchboard.urlBlacklist); + } this.localsolroffset += nextitems; } @@ -1267,7 +1275,9 @@ public final class SearchEvent { if (!this.localsolrsearch.isAlive() && this.local_solr_stored.get() > this.localsolroffset && (item + 1) % this.query.itemsPerPage == 0) { // at the end of a list, trigger a next solr search - this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.localsolroffset, this.query.itemsPerPage, null /*this peer*/, Switchboard.urlBlacklist); + if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) { + this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.localsolroffset, this.query.itemsPerPage, null /*this peer*/, Switchboard.urlBlacklist); + } this.localsolroffset += this.query.itemsPerPage; } return re;