boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fixes to internal RWI usage if RWI is switched off (NPE etc)

Browse Source
pull/1/head
Michael Peter Christen 12 years ago
parent 3834829b37
commit e8f7b85b98
3 changed files with 66 additions and 64 deletions
Show Stats Download Patch File Download Diff File

120
source/net/yacy/peers/Protocol.java
Unescape View File

@ -1122,77 +1122,77 @@ public final class Protocol
// evaluate result // evaluate result
List<URIMetadataNode> container = new ArrayList<URIMetadataNode>(); List<URIMetadataNode> container = new ArrayList<URIMetadataNode>();
if (docList.size() == 0) { if (docList == null || docList.size() == 0) {
Network.log.logInfo("SEARCH (solr), returned 0 out of " + docList.getNumFound() + " documents from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName())) + " query = " + solrQuery.toString()) ; Network.log.logInfo("SEARCH (solr), returned 0 out of " + docList.getNumFound() + " documents from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName())) + " query = " + solrQuery.toString()) ;
} else {// create containers return 0;
Network.log.logInfo("SEARCH (solr), returned " + docList.size() + " out of " + docList.getNumFound() + " documents from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName()))) ; }
Network.log.logInfo("SEARCH (solr), returned " + docList.size() + " out of " + docList.getNumFound() + " documents from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName())));
int term = count;
for (final SolrDocument doc: docList) {
if ( term-- <= 0 ) {
break; // do not process more that requested (in case that evil peers fill us up with rubbish)
}
// get one single search result
if ( doc == null ) {
continue;
}
URIMetadataNode urlEntry = new URIMetadataNode(doc);
int term = count; if ( blacklist.isListed(BlacklistType.SEARCH, urlEntry) ) {
for (final SolrDocument doc: docList) { if ( Network.log.isInfo() ) {
if ( term-- <= 0 ) { if (localsearch) {
break; // do not process more that requested (in case that evil peers fill us up with rubbish) Network.log.logInfo("local search (solr): filtered blacklisted url " + urlEntry.url());
} } else {
// get one single search result Network.log.logInfo("remote search (solr): filtered blacklisted url " + urlEntry.url() + " from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName())));
if ( doc == null ) {
continue;
}
URIMetadataNode urlEntry = new URIMetadataNode(doc);
if ( blacklist.isListed(BlacklistType.SEARCH, urlEntry) ) {
if ( Network.log.isInfo() ) {
if (localsearch) {
Network.log.logInfo("local search (solr): filtered blacklisted url " + urlEntry.url());
} else {
Network.log.logInfo("remote search (solr): filtered blacklisted url " + urlEntry.url() + " from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName())));
}
} }
continue; // block with blacklist
} }
continue; // block with blacklist
}
final String urlRejectReason = Switchboard.getSwitchboard().crawlStacker.urlInAcceptedDomain(urlEntry.url()); final String urlRejectReason = Switchboard.getSwitchboard().crawlStacker.urlInAcceptedDomain(urlEntry.url());
if ( urlRejectReason != null ) { if ( urlRejectReason != null ) {
if ( Network.log.isInfo() ) { if ( Network.log.isInfo() ) {
if (localsearch) { if (localsearch) {
Network.log.logInfo("local search (solr): rejected url '" + urlEntry.url() + "' (" + urlRejectReason + ")"); Network.log.logInfo("local search (solr): rejected url '" + urlEntry.url() + "' (" + urlRejectReason + ")");
} else { } else {
Network.log.logInfo("remote search (solr): rejected url '" + urlEntry.url() + "' (" + urlRejectReason + ") from peer " + target.getName()); Network.log.logInfo("remote search (solr): rejected url '" + urlEntry.url() + "' (" + urlRejectReason + ") from peer " + target.getName());
}
} }
continue; // reject url outside of our domain
} }
continue; // reject url outside of our domain
}
// passed all checks, store url // passed all checks, store url
if (!localsearch) { if (!localsearch) {
try { try {
event.query.getSegment().fulltext().putDocument(ClientUtils.toSolrInputDocument(doc)); event.query.getSegment().fulltext().putDocument(ClientUtils.toSolrInputDocument(doc));
ResultURLs.stack( ResultURLs.stack(
ASCII.String(urlEntry.url().hash()), ASCII.String(urlEntry.url().hash()),
urlEntry.url().getHost(), urlEntry.url().getHost(),
event.peers.mySeed().hash.getBytes(), event.peers.mySeed().hash.getBytes(),
UTF8.getBytes(target.hash), UTF8.getBytes(target.hash),
EventOrigin.QUERIES); EventOrigin.QUERIES);
} catch ( final IOException e ) { } catch ( final IOException e ) {
Network.log.logWarning("could not store search result", e); Network.log.logWarning("could not store search result", e);
continue; // db-error continue; // db-error
}
} }
// add the url entry to the word indexes
container.add(urlEntry);
} }
if (localsearch) { // add the url entry to the word indexes
event.add(container, facets, snippets, true, "localpeer", (int) docList.getNumFound()); container.add(urlEntry);
event.rankingProcess.addFinalize(); }
event.addExpectedRemoteReferences(-count);
Network.log.logInfo("local search (solr): localpeer sent " + container.get(0).size() + "/" + docList.size() + " references"); if (localsearch) {
} else { event.add(container, facets, snippets, true, "localpeer", (int) docList.getNumFound());
event.add(container, facets, snippets, false, target.getName() + "/" + target.hash, (int) docList.getNumFound()); event.rankingProcess.addFinalize();
event.rankingProcess.addFinalize(); event.addExpectedRemoteReferences(-count);
event.addExpectedRemoteReferences(-count); Network.log.logInfo("local search (solr): localpeer sent " + container.get(0).size() + "/" + docList.size() + " references");
Network.log.logInfo("remote search (solr): peer " + target.getName() + " sent " + container.get(0).size() + "/" + docList.size() + " references"); } else {
} event.add(container, facets, snippets, false, target.getName() + "/" + target.hash, (int) docList.getNumFound());
} event.rankingProcess.addFinalize();
event.addExpectedRemoteReferences(-count);
Network.log.logInfo("remote search (solr): peer " + target.getName() + " sent " + container.get(0).size() + "/" + docList.size() + " references");
}
return docList.size(); return docList.size();
} }

2
source/net/yacy/search/query/RankingProcess.java
Unescape View File

@ -197,6 +197,8 @@ public final class RankingProcess extends Thread {
@Override @Override
public void run() { public void run() {
if (query.getSegment().termIndex() == null) return; // nothing to do; this index is not used
// do a search // do a search
oneFeederStarted(); oneFeederStarted();

8
source/net/yacy/search/query/SearchEvent.java
Unescape View File

@ -224,7 +224,7 @@ public final class SearchEvent {
// start a local RWI search concurrently // start a local RWI search concurrently
if (this.remote || this.peers.mySeed().getBirthdate() < noRobinsonLocalRWISearch) { if (this.remote || this.peers.mySeed().getBirthdate() < noRobinsonLocalRWISearch) {
// we start the local search only if this peer is doing a remote search or when it is doing a local search and the peer is old // we start the local search only if this peer is doing a remote search or when it is doing a local search and the peer is old
this.rankingProcess.start(); if (query.getSegment().connectedRWI()) this.rankingProcess.start();
} }
if (this.remote) { if (this.remote) {
@ -270,7 +270,7 @@ public final class SearchEvent {
if ( generateAbstracts ) { if ( generateAbstracts ) {
// we need the results now // we need the results now
try { try {
this.rankingProcess.join(); if (query.getSegment().connectedRWI()) this.rankingProcess.join();
} catch ( final Throwable e ) { } catch ( final Throwable e ) {
} }
// compute index abstracts // compute index abstracts
@ -278,7 +278,7 @@ public final class SearchEvent {
int maxcount = -1; int maxcount = -1;
long mindhtdistance = Long.MAX_VALUE, l; long mindhtdistance = Long.MAX_VALUE, l;
byte[] wordhash; byte[] wordhash;
assert this.rankingProcess.searchContainerMap() != null; assert !query.getSegment().connectedRWI() || this.rankingProcess.searchContainerMap() != null;
if (this.rankingProcess.searchContainerMap() != null) { if (this.rankingProcess.searchContainerMap() != null) {
for (final Map.Entry<byte[], ReferenceContainer<WordReference>> entry : this.rankingProcess.searchContainerMap().entrySet()) { for (final Map.Entry<byte[], ReferenceContainer<WordReference>> entry : this.rankingProcess.searchContainerMap().entrySet()) {
wordhash = entry.getKey(); wordhash = entry.getKey();
@ -303,7 +303,7 @@ public final class SearchEvent {
// give process time to accumulate a certain amount of data // give process time to accumulate a certain amount of data
// before a reading process wants to get results from it // before a reading process wants to get results from it
try { try {
this.rankingProcess.join(100); if (query.getSegment().connectedRWI()) this.rankingProcess.join(100);
} catch ( final Throwable e ) { } catch ( final Throwable e ) {
} }
// this will reduce the maximum waiting time until results are available to 100 milliseconds // this will reduce the maximum waiting time until results are available to 100 milliseconds

Write Preview
Loading…
Cancel
Save