boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fixes to internal RWI usage if RWI is switched off (NPE etc)

Browse Source
pull/1/head
Michael Peter Christen 12 years ago
parent 3834829b37
commit e8f7b85b98
3 changed files with 66 additions and 64 deletions
Show Stats Download Patch File Download Diff File

120
source/net/yacy/peers/Protocol.java
Unescape View File

@ -1122,77 +1122,77 @@ public final class Protocol
// evaluate result
List<URIMetadataNode> container = new ArrayList<URIMetadataNode>();
if (docList.size() == 0) {
if (docList == null || docList.size() == 0) {
Network.log.logInfo("SEARCH (solr), returned 0 out of " + docList.getNumFound() + " documents from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName())) + " query = " + solrQuery.toString()) ;
} else {// create containers
Network.log.logInfo("SEARCH (solr), returned " + docList.size() + " out of " + docList.getNumFound() + " documents from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName()))) ;
return 0;
}
Network.log.logInfo("SEARCH (solr), returned " + docList.size() + " out of " + docList.getNumFound() + " documents from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName())));
int term = count;
for (final SolrDocument doc: docList) {
if ( term-- <= 0 ) {
break; // do not process more that requested (in case that evil peers fill us up with rubbish)
}
// get one single search result
if ( doc == null ) {
continue;
}
URIMetadataNode urlEntry = new URIMetadataNode(doc);
int term = count;
for (final SolrDocument doc: docList) {
if ( term-- <= 0 ) {
break; // do not process more that requested (in case that evil peers fill us up with rubbish)
}
// get one single search result
if ( doc == null ) {
continue;
}
URIMetadataNode urlEntry = new URIMetadataNode(doc);
if ( blacklist.isListed(BlacklistType.SEARCH, urlEntry) ) {
if ( Network.log.isInfo() ) {
if (localsearch) {
Network.log.logInfo("local search (solr): filtered blacklisted url " + urlEntry.url());
} else {
Network.log.logInfo("remote search (solr): filtered blacklisted url " + urlEntry.url() + " from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName())));
}
if ( blacklist.isListed(BlacklistType.SEARCH, urlEntry) ) {
if ( Network.log.isInfo() ) {
if (localsearch) {
Network.log.logInfo("local search (solr): filtered blacklisted url " + urlEntry.url());
} else {
Network.log.logInfo("remote search (solr): filtered blacklisted url " + urlEntry.url() + " from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName())));
}
continue; // block with blacklist
}
continue; // block with blacklist
}
final String urlRejectReason = Switchboard.getSwitchboard().crawlStacker.urlInAcceptedDomain(urlEntry.url());
if ( urlRejectReason != null ) {
if ( Network.log.isInfo() ) {
if (localsearch) {
Network.log.logInfo("local search (solr): rejected url '" + urlEntry.url() + "' (" + urlRejectReason + ")");
} else {
Network.log.logInfo("remote search (solr): rejected url '" + urlEntry.url() + "' (" + urlRejectReason + ") from peer " + target.getName());
}
final String urlRejectReason = Switchboard.getSwitchboard().crawlStacker.urlInAcceptedDomain(urlEntry.url());
if ( urlRejectReason != null ) {
if ( Network.log.isInfo() ) {
if (localsearch) {
Network.log.logInfo("local search (solr): rejected url '" + urlEntry.url() + "' (" + urlRejectReason + ")");
} else {
Network.log.logInfo("remote search (solr): rejected url '" + urlEntry.url() + "' (" + urlRejectReason + ") from peer " + target.getName());
}
continue; // reject url outside of our domain
}
continue; // reject url outside of our domain
}
// passed all checks, store url
if (!localsearch) {
try {
event.query.getSegment().fulltext().putDocument(ClientUtils.toSolrInputDocument(doc));
ResultURLs.stack(
ASCII.String(urlEntry.url().hash()),
urlEntry.url().getHost(),
event.peers.mySeed().hash.getBytes(),
UTF8.getBytes(target.hash),
EventOrigin.QUERIES);
} catch ( final IOException e ) {
Network.log.logWarning("could not store search result", e);
continue; // db-error
}
// passed all checks, store url
if (!localsearch) {
try {
event.query.getSegment().fulltext().putDocument(ClientUtils.toSolrInputDocument(doc));
ResultURLs.stack(
ASCII.String(urlEntry.url().hash()),
urlEntry.url().getHost(),
event.peers.mySeed().hash.getBytes(),
UTF8.getBytes(target.hash),
EventOrigin.QUERIES);
} catch ( final IOException e ) {
Network.log.logWarning("could not store search result", e);
continue; // db-error
}
// add the url entry to the word indexes
container.add(urlEntry);
}
if (localsearch) {
event.add(container, facets, snippets, true, "localpeer", (int) docList.getNumFound());
event.rankingProcess.addFinalize();
event.addExpectedRemoteReferences(-count);
Network.log.logInfo("local search (solr): localpeer sent " + container.get(0).size() + "/" + docList.size() + " references");
} else {
event.add(container, facets, snippets, false, target.getName() + "/" + target.hash, (int) docList.getNumFound());
event.rankingProcess.addFinalize();
event.addExpectedRemoteReferences(-count);
Network.log.logInfo("remote search (solr): peer " + target.getName() + " sent " + container.get(0).size() + "/" + docList.size() + " references");
}
}
// add the url entry to the word indexes
container.add(urlEntry);
}
if (localsearch) {
event.add(container, facets, snippets, true, "localpeer", (int) docList.getNumFound());
event.rankingProcess.addFinalize();
event.addExpectedRemoteReferences(-count);
Network.log.logInfo("local search (solr): localpeer sent " + container.get(0).size() + "/" + docList.size() + " references");
} else {
event.add(container, facets, snippets, false, target.getName() + "/" + target.hash, (int) docList.getNumFound());
event.rankingProcess.addFinalize();
event.addExpectedRemoteReferences(-count);
Network.log.logInfo("remote search (solr): peer " + target.getName() + " sent " + container.get(0).size() + "/" + docList.size() + " references");
}
return docList.size();
}

2
source/net/yacy/search/query/RankingProcess.java
Unescape View File

@ -197,6 +197,8 @@ public final class RankingProcess extends Thread {
@Override
public void run() {
if (query.getSegment().termIndex() == null) return; // nothing to do; this index is not used
// do a search
oneFeederStarted();

8
source/net/yacy/search/query/SearchEvent.java
Unescape View File

@ -224,7 +224,7 @@ public final class SearchEvent {
// start a local RWI search concurrently
if (this.remote || this.peers.mySeed().getBirthdate() < noRobinsonLocalRWISearch) {
// we start the local search only if this peer is doing a remote search or when it is doing a local search and the peer is old
this.rankingProcess.start();
if (query.getSegment().connectedRWI()) this.rankingProcess.start();
}
if (this.remote) {
@ -270,7 +270,7 @@ public final class SearchEvent {
if ( generateAbstracts ) {
// we need the results now
try {
this.rankingProcess.join();
if (query.getSegment().connectedRWI()) this.rankingProcess.join();
} catch ( final Throwable e ) {
}
// compute index abstracts
@ -278,7 +278,7 @@ public final class SearchEvent {
int maxcount = -1;
long mindhtdistance = Long.MAX_VALUE, l;
byte[] wordhash;
assert this.rankingProcess.searchContainerMap() != null;
assert !query.getSegment().connectedRWI() || this.rankingProcess.searchContainerMap() != null;
if (this.rankingProcess.searchContainerMap() != null) {
for (final Map.Entry<byte[], ReferenceContainer<WordReference>> entry : this.rankingProcess.searchContainerMap().entrySet()) {
wordhash = entry.getKey();
@ -303,7 +303,7 @@ public final class SearchEvent {
// give process time to accumulate a certain amount of data
// before a reading process wants to get results from it
try {
this.rankingProcess.join(100);
if (query.getSegment().connectedRWI()) this.rankingProcess.join(100);
} catch ( final Throwable e ) {
}
// this will reduce the maximum waiting time until results are available to 100 milliseconds

Write Preview
Loading…
Cancel
Save