boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added debug switches for detailed search testing

Browse Source
pull/1/head
orbiter 12 years ago
parent cdbfddf091
commit b1140e3d82
5 changed files with 75 additions and 40 deletions
Show Stats Download Patch File Download Diff File

10
defaults/yacy.init
Unescape View File

@ -445,9 +445,13 @@ seedScpPath=
# of the period here (minutes)
peerCycle=2
# Debug mode for YACY network: this will trigger that also local ip's are
# accepted as peer addresses
yacyDebugMode=false
# debug flags
debug.search.local.dht.on=false
debug.search.local.solr.off=false
debug.search.remote.dht.off=false
debug.search.remote.dht.testlocal=false
debug.search.remote.solr.off=false
debug.search.remote.solr.testlocal=false
#staticIP if you have a static IP, you can use this setting
staticIP=

9
source/net/yacy/peers/Protocol.java
Unescape View File

@ -1020,7 +1020,7 @@ public final class Protocol {
final int offset,
final int count,
boolean getFacets,
final Seed target,
Seed target,
final Blacklist blacklist) {
if (event.query.getQueryGoal().getOriginalQueryString(false) == null || event.query.getQueryGoal().getOriginalQueryString(false).length() == 0) {
@ -1051,6 +1051,10 @@ public final class Protocol {
for (CollectionSchema field: snippetFields) solrQuery.addHighlightField(field.getSolrFieldName());
boolean localsearch = target == null || target.equals(event.peers.mySeed());
if (localsearch && Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL, false)) {
target = event.peers.mySeed();
localsearch = false;
}
SolrDocumentList docList = null;
QueryResponse rsp = null;
if (localsearch) {
@ -1064,7 +1068,8 @@ public final class Protocol {
}
} else {
try {
RemoteInstance instance = new RemoteInstance("http://" + target.getPublicAddress(), null, "solr"); // this is a 'patch configuration' which considers 'solr' as default collection
String address = target == event.peers.mySeed() ? "localhost:" + target.getPort() : target.getPublicAddress();
RemoteInstance instance = new RemoteInstance("http://" + address, null, "solr"); // this is a 'patch configuration' which considers 'solr' as default collection
SolrConnector solrConnector = new RemoteSolrConnector(instance, "solr");
rsp = solrConnector.query(solrQuery);
docList = rsp.getResults();

65
source/net/yacy/peers/RemoteSearch.java
Unescape View File

@ -34,6 +34,8 @@ import net.yacy.cora.document.ASCII;
import net.yacy.cora.storage.HandleSet;
import net.yacy.kelondro.logging.Log;
import net.yacy.repository.Blacklist;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.search.query.QueryParams;
import net.yacy.search.query.SearchEvent;
import net.yacy.search.query.SecondarySearchSuperviser;
@ -162,39 +164,44 @@ public class RemoteSearch extends Thread {
nodePeers.add(s);
}
}
// for debugging: remove all dht peer to see if solr is working properly
//dhtPeers.clear(); // FOR DEBUGGING ONLY!!!
// start solr searches
for (Seed s: nodePeers) {
solrRemoteSearch(event, start, count, s, blacklist);
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_OFF, false)) {
for (Seed s: nodePeers) {
solrRemoteSearch(event, start, count, s, blacklist);
}
}
if (Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_TESTLOCAL, false)) {
dhtPeers.clear();
dhtPeers.add(event.peers.mySeed());
}
// start search to YaCy DHT peers
final int targets = dhtPeers.size();
if (targets == 0) return;
for (int i = 0; i < targets; i++) {
if (dhtPeers.get(i) == null || dhtPeers.get(i).hash == null) continue;
try {
RemoteSearch rs = new RemoteSearch(
event,
QueryParams.hashSet2hashString(event.query.getQueryGoal().getIncludeHashes()),
QueryParams.hashSet2hashString(event.query.getQueryGoal().getExcludeHashes()),
event.query.targetlang == null ? "" : event.query.targetlang,
event.query.contentdom == null ? "all" : event.query.contentdom.toString(),
count,
time,
event.query.maxDistance,
targets,
dhtPeers.get(i),
event.secondarySearchSuperviser,
blacklist);
rs.start();
event.primarySearchThreadsL.add(rs);
} catch (final OutOfMemoryError e) {
Log.logException(e);
break;
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_OFF, false)) {
final int targets = dhtPeers.size();
if (targets == 0) return;
for (int i = 0; i < targets; i++) {
if (dhtPeers.get(i) == null || dhtPeers.get(i).hash == null) continue;
try {
RemoteSearch rs = new RemoteSearch(
event,
QueryParams.hashSet2hashString(event.query.getQueryGoal().getIncludeHashes()),
QueryParams.hashSet2hashString(event.query.getQueryGoal().getExcludeHashes()),
event.query.targetlang == null ? "" : event.query.targetlang,
event.query.contentdom == null ? "all" : event.query.contentdom.toString(),
count,
time,
event.query.maxDistance,
targets,
dhtPeers.get(i),
event.secondarySearchSuperviser,
blacklist);
rs.start();
event.primarySearchThreadsL.add(rs);
} catch (final OutOfMemoryError e) {
Log.logException(e);
break;
}
}
}
}

13
source/net/yacy/search/SwitchboardConstants.java
Unescape View File

@ -324,8 +324,17 @@ public final class SwitchboardConstants {
public static final String CRAWLER_THREADS_ACTIVE_MAX = "crawler.MaxActiveThreads";
public static final String CRAWLER_FOLLOW_REDIRECTS = "crawler.http.FollowRedirects"; // ignore the target url and follow to the redirect
public static final String CRAWLER_RECORD_REDIRECTS = "crawler.http.RecordRedirects"; // record the ignored redirected page to the index store
public static final String YACY_MODE_DEBUG = "yacyDebugMode";
/**
* debug flags
*/
public static final String DEBUG_SEARCH_LOCAL_DHT_ON = "debug.search.local.dht.on"; // =true: use the local dht/rwi index (which is not done if we do remote searches)
public static final String DEBUG_SEARCH_LOCAL_SOLR_OFF = "debug.search.local.solr.off"; // =true: do not use solr
public static final String DEBUG_SEARCH_REMOTE_DHT_OFF = "debug.search.remote.dht.off"; // =true: do not use dht/rwi
public static final String DEBUG_SEARCH_REMOTE_DHT_TESTLOCAL= "debug.search.remote.dht.testlocal"; // =true: do not use dht, search local peer in a shortcut to the own server
public static final String DEBUG_SEARCH_REMOTE_SOLR_OFF = "debug.search.remote.solr.off"; // =true: do not use solr
public static final String DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL= "debug.search.remote.solr.testlocal"; // =true: do not use dht, search local peer in a shortcut to the own server
/**
* <p><code>public static final String <strong>WORDCACHE_MAX_COUNT</strong> = "wordCacheMaxCount"</code></p>
* <p>Name of the setting how many words the word-cache (or DHT-Out cache) shall contain maximal. Indexing pages if the

18
source/net/yacy/search/query/SearchEvent.java
Unescape View File

@ -86,6 +86,7 @@ import net.yacy.repository.LoaderDispatcher;
import net.yacy.repository.Blacklist.BlacklistType;
import net.yacy.search.EventTracker;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.search.index.Segment;
import net.yacy.search.ranking.ReferenceOrder;
import net.yacy.search.schema.CollectionSchema;
@ -256,12 +257,17 @@ public final class SearchEvent {
}
// start a local solr search
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, 0, this.query.itemsPerPage, null /*this peer*/, Switchboard.urlBlacklist);
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, 0, this.query.itemsPerPage, null /*this peer*/, Switchboard.urlBlacklist);
}
this.localsolroffset = this.query.itemsPerPage;
// start a local RWI search concurrently
this.rwiProcess = null;
if (query.getSegment().connectedRWI() && (!this.remote || this.peers.mySeed().getBirthdate() < noRobinsonLocalRWISearch)) {
if (query.getSegment().connectedRWI() && (
!this.remote ||
this.peers.mySeed().getBirthdate() < noRobinsonLocalRWISearch) ||
Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_DHT_ON, false)) {
// we start the local search only if this peer is doing a remote search or when it is doing a local search and the peer is old
rwiProcess = new RWIProcess();
rwiProcess.start();
@ -1248,7 +1254,9 @@ public final class SearchEvent {
// load remaining solr results now
int nextitems = item - this.localsolroffset + this.query.itemsPerPage; // example: suddenly switch to item 60, just 10 had been shown, 20 loaded.
if (this.localsolrsearch.isAlive()) {try {this.localsolrsearch.join();} catch (InterruptedException e) {}}
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.localsolroffset, nextitems, null /*this peer*/, Switchboard.urlBlacklist);
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.localsolroffset, nextitems, null /*this peer*/, Switchboard.urlBlacklist);
}
this.localsolroffset += nextitems;
}
@ -1267,7 +1275,9 @@ public final class SearchEvent {
if (!this.localsolrsearch.isAlive() && this.local_solr_stored.get() > this.localsolroffset && (item + 1) % this.query.itemsPerPage == 0) {
// at the end of a list, trigger a next solr search
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.localsolroffset, this.query.itemsPerPage, null /*this peer*/, Switchboard.urlBlacklist);
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.localsolroffset, this.query.itemsPerPage, null /*this peer*/, Switchboard.urlBlacklist);
}
this.localsolroffset += this.query.itemsPerPage;
}
return re;

Write Preview
Loading…
Cancel
Save