From e8f7b85b98ab270678a80af7721a86c67356af7f Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Mon, 4 Feb 2013 17:11:02 +0100 Subject: [PATCH] fixes to internal RWI usage if RWI is switched off (NPE etc) --- source/net/yacy/peers/Protocol.java | 120 +++++++++--------- .../net/yacy/search/query/RankingProcess.java | 2 + source/net/yacy/search/query/SearchEvent.java | 8 +- 3 files changed, 66 insertions(+), 64 deletions(-) diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java index be26d99ae..17aec01d0 100644 --- a/source/net/yacy/peers/Protocol.java +++ b/source/net/yacy/peers/Protocol.java @@ -1122,77 +1122,77 @@ public final class Protocol // evaluate result List container = new ArrayList(); - if (docList.size() == 0) { + if (docList == null || docList.size() == 0) { Network.log.logInfo("SEARCH (solr), returned 0 out of " + docList.getNumFound() + " documents from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName())) + " query = " + solrQuery.toString()) ; - } else {// create containers - Network.log.logInfo("SEARCH (solr), returned " + docList.size() + " out of " + docList.getNumFound() + " documents from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName()))) ; + return 0; + } + + Network.log.logInfo("SEARCH (solr), returned " + docList.size() + " out of " + docList.getNumFound() + " documents from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName()))); + int term = count; + for (final SolrDocument doc: docList) { + if ( term-- <= 0 ) { + break; // do not process more that requested (in case that evil peers fill us up with rubbish) + } + // get one single search result + if ( doc == null ) { + continue; + } + URIMetadataNode urlEntry = new URIMetadataNode(doc); - int term = count; - for (final SolrDocument doc: docList) { - if ( term-- <= 0 ) { - break; // do not process more that requested (in case that evil peers fill us up with rubbish) - } - // get one single search result - if ( doc == null ) { - continue; - } - URIMetadataNode urlEntry = new URIMetadataNode(doc); - - if ( blacklist.isListed(BlacklistType.SEARCH, urlEntry) ) { - if ( Network.log.isInfo() ) { - if (localsearch) { - Network.log.logInfo("local search (solr): filtered blacklisted url " + urlEntry.url()); - } else { - Network.log.logInfo("remote search (solr): filtered blacklisted url " + urlEntry.url() + " from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName()))); - } + if ( blacklist.isListed(BlacklistType.SEARCH, urlEntry) ) { + if ( Network.log.isInfo() ) { + if (localsearch) { + Network.log.logInfo("local search (solr): filtered blacklisted url " + urlEntry.url()); + } else { + Network.log.logInfo("remote search (solr): filtered blacklisted url " + urlEntry.url() + " from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName()))); } - continue; // block with blacklist } + continue; // block with blacklist + } - final String urlRejectReason = Switchboard.getSwitchboard().crawlStacker.urlInAcceptedDomain(urlEntry.url()); - if ( urlRejectReason != null ) { - if ( Network.log.isInfo() ) { - if (localsearch) { - Network.log.logInfo("local search (solr): rejected url '" + urlEntry.url() + "' (" + urlRejectReason + ")"); - } else { - Network.log.logInfo("remote search (solr): rejected url '" + urlEntry.url() + "' (" + urlRejectReason + ") from peer " + target.getName()); - } + final String urlRejectReason = Switchboard.getSwitchboard().crawlStacker.urlInAcceptedDomain(urlEntry.url()); + if ( urlRejectReason != null ) { + if ( Network.log.isInfo() ) { + if (localsearch) { + Network.log.logInfo("local search (solr): rejected url '" + urlEntry.url() + "' (" + urlRejectReason + ")"); + } else { + Network.log.logInfo("remote search (solr): rejected url '" + urlEntry.url() + "' (" + urlRejectReason + ") from peer " + target.getName()); } - continue; // reject url outside of our domain } + continue; // reject url outside of our domain + } - // passed all checks, store url - if (!localsearch) { - try { - event.query.getSegment().fulltext().putDocument(ClientUtils.toSolrInputDocument(doc)); - ResultURLs.stack( - ASCII.String(urlEntry.url().hash()), - urlEntry.url().getHost(), - event.peers.mySeed().hash.getBytes(), - UTF8.getBytes(target.hash), - EventOrigin.QUERIES); - } catch ( final IOException e ) { - Network.log.logWarning("could not store search result", e); - continue; // db-error - } + // passed all checks, store url + if (!localsearch) { + try { + event.query.getSegment().fulltext().putDocument(ClientUtils.toSolrInputDocument(doc)); + ResultURLs.stack( + ASCII.String(urlEntry.url().hash()), + urlEntry.url().getHost(), + event.peers.mySeed().hash.getBytes(), + UTF8.getBytes(target.hash), + EventOrigin.QUERIES); + } catch ( final IOException e ) { + Network.log.logWarning("could not store search result", e); + continue; // db-error } - - // add the url entry to the word indexes - container.add(urlEntry); } - if (localsearch) { - event.add(container, facets, snippets, true, "localpeer", (int) docList.getNumFound()); - event.rankingProcess.addFinalize(); - event.addExpectedRemoteReferences(-count); - Network.log.logInfo("local search (solr): localpeer sent " + container.get(0).size() + "/" + docList.size() + " references"); - } else { - event.add(container, facets, snippets, false, target.getName() + "/" + target.hash, (int) docList.getNumFound()); - event.rankingProcess.addFinalize(); - event.addExpectedRemoteReferences(-count); - Network.log.logInfo("remote search (solr): peer " + target.getName() + " sent " + container.get(0).size() + "/" + docList.size() + " references"); - } - } + // add the url entry to the word indexes + container.add(urlEntry); + } + + if (localsearch) { + event.add(container, facets, snippets, true, "localpeer", (int) docList.getNumFound()); + event.rankingProcess.addFinalize(); + event.addExpectedRemoteReferences(-count); + Network.log.logInfo("local search (solr): localpeer sent " + container.get(0).size() + "/" + docList.size() + " references"); + } else { + event.add(container, facets, snippets, false, target.getName() + "/" + target.hash, (int) docList.getNumFound()); + event.rankingProcess.addFinalize(); + event.addExpectedRemoteReferences(-count); + Network.log.logInfo("remote search (solr): peer " + target.getName() + " sent " + container.get(0).size() + "/" + docList.size() + " references"); + } return docList.size(); } diff --git a/source/net/yacy/search/query/RankingProcess.java b/source/net/yacy/search/query/RankingProcess.java index 0e2d28327..be1ebf714 100644 --- a/source/net/yacy/search/query/RankingProcess.java +++ b/source/net/yacy/search/query/RankingProcess.java @@ -197,6 +197,8 @@ public final class RankingProcess extends Thread { @Override public void run() { + if (query.getSegment().termIndex() == null) return; // nothing to do; this index is not used + // do a search oneFeederStarted(); diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.java index 9afcd9b79..f9f6d2ead 100644 --- a/source/net/yacy/search/query/SearchEvent.java +++ b/source/net/yacy/search/query/SearchEvent.java @@ -224,7 +224,7 @@ public final class SearchEvent { // start a local RWI search concurrently if (this.remote || this.peers.mySeed().getBirthdate() < noRobinsonLocalRWISearch) { // we start the local search only if this peer is doing a remote search or when it is doing a local search and the peer is old - this.rankingProcess.start(); + if (query.getSegment().connectedRWI()) this.rankingProcess.start(); } if (this.remote) { @@ -270,7 +270,7 @@ public final class SearchEvent { if ( generateAbstracts ) { // we need the results now try { - this.rankingProcess.join(); + if (query.getSegment().connectedRWI()) this.rankingProcess.join(); } catch ( final Throwable e ) { } // compute index abstracts @@ -278,7 +278,7 @@ public final class SearchEvent { int maxcount = -1; long mindhtdistance = Long.MAX_VALUE, l; byte[] wordhash; - assert this.rankingProcess.searchContainerMap() != null; + assert !query.getSegment().connectedRWI() || this.rankingProcess.searchContainerMap() != null; if (this.rankingProcess.searchContainerMap() != null) { for (final Map.Entry> entry : this.rankingProcess.searchContainerMap().entrySet()) { wordhash = entry.getKey(); @@ -303,7 +303,7 @@ public final class SearchEvent { // give process time to accumulate a certain amount of data // before a reading process wants to get results from it try { - this.rankingProcess.join(100); + if (query.getSegment().connectedRWI()) this.rankingProcess.join(100); } catch ( final Throwable e ) { } // this will reduce the maximum waiting time until results are available to 100 milliseconds