boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fixed false multiple-generation of remote facet search which

Browse Source 
caused high cpu usage on remote side.
pull/1/head
Michael Peter Christen 12 years ago
parent 23fb458963
commit 8caaf6203a
4 changed files with 28 additions and 25 deletions
Show Stats Download Patch File Download Diff File

18
source/net/yacy/peers/Protocol.java
Unescape View File

@ -1005,7 +1005,6 @@ public final class Protocol {
final SolrQuery solrQuery,
final int offset,
final int count,
boolean getFacets,
Seed target,
final Blacklist blacklist) {
@ -1013,19 +1012,10 @@ public final class Protocol {
return -1; // we cannot query solr only with word hashes, there is no clear text string
}
event.addExpectedRemoteReferences(count);
solrQuery.setStart(offset);
solrQuery.setRows(count);
// set facet query attributes
if (getFacets && event.query.facetfields.size() > 0) {
solrQuery.setFacet(true);
solrQuery.setFacetLimit(event.query.maxfacets);
solrQuery.setFacetSort(FacetParams.FACET_SORT_COUNT);
for (String field: event.query.facetfields) solrQuery.addFacetField(field);
} else {
solrQuery.setFacet(false);
}
// set highlighting query attributes
solrQuery.setHighlight(true);
solrQuery.setHighlightFragsize(SearchEvent.SNIPPET_MAX_LENGTH);
@ -1035,8 +1025,6 @@ public final class Protocol {
solrQuery.setHighlightSnippets(1);
for (CollectionSchema field: snippetFields) solrQuery.addHighlightField(field.getSolrFieldName());
solrQuery.setFields("*", "score"); // we need the score for post-ranking
boolean localsearch = target == null || target.equals(event.peers.mySeed());
if (localsearch && Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL, false)) {
target = event.peers.mySeed();
@ -1083,7 +1071,7 @@ public final class Protocol {
if (c == 0) continue;
result.set(ff.getName(), c);
}
facets.put(field, result);
if (result.size() > 0) facets.put(field, result);
}
// evaluate snippets
@ -1112,7 +1100,7 @@ public final class Protocol {
return 0;
}
Network.log.logInfo("SEARCH (solr), returned " + docList.size() + " out of " + docList.getNumFound() + " documents from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName())));
Network.log.logInfo("SEARCH (solr), returned " + docList.size() + " out of " + docList.getNumFound() + " documents and " + facets.size() + " facets " + facets.keySet().toString() + " from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName())));
int term = count;
Collection<SolrInputDocument> docs = new ArrayList<SolrInputDocument>(docList.size());
for (final SolrDocument doc: docList) {

3
source/net/yacy/peers/RemoteSearch.java
Unescape View File

@ -172,7 +172,7 @@ public class RemoteSearch extends Thread {
nodePeers.add(event.peers.mySeed());
}
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_OFF, false)) {
final SolrQuery solrQuery = event.query.solrQuery();
final SolrQuery solrQuery = event.query.solrQuery(start == 0);
for (Seed s: nodePeers) {
Thread t = solrRemoteSearch(event, solrQuery, start, count, s, blacklist);
event.nodeSearchThreads.add(t);
@ -288,7 +288,6 @@ public class RemoteSearch extends Thread {
solrQuery,
start,
count,
start == 0,
targetPeer,
blacklist);
if (urls >= 0) {

26
source/net/yacy/search/query/QueryParams.java
Unescape View File

@ -30,7 +30,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
@ -38,6 +38,7 @@ import java.util.regex.PatternSyntaxException;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrQuery.SortClause;
import org.apache.solr.common.params.FacetParams;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.analysis.Classification;
@ -119,7 +120,7 @@ public final class QueryParams {
public final String userAgent;
protected boolean filterfailurls, filterscannerfail;
protected double lat, lon, radius;
public List<String> facetfields;
public LinkedHashSet<String> facetfields;
public int maxfacets;
private SolrQuery cachedQuery;
private CollectionConfiguration solrSchema;
@ -165,7 +166,7 @@ public final class QueryParams {
this.lat = 0.0d;
this.lon = 0.0d;
this.radius = 0.0d;
this.facetfields = new ArrayList<String>();
this.facetfields = new LinkedHashSet<String>();
this.solrSchema = indexSegment.fulltext().getDefaultConfiguration();
for (CollectionSchema f: defaultfacetfields) {
@ -266,7 +267,7 @@ public final class QueryParams {
this.lat = Math.floor(lat * this.kmNormal) / this.kmNormal;
this.lon = Math.floor(lon * this.kmNormal) / this.kmNormal;
this.radius = Math.floor(radius * this.kmNormal + 1) / this.kmNormal;
this.facetfields = new ArrayList<String>();
this.facetfields = new LinkedHashSet<String>();
this.solrSchema = indexSegment.fulltext().getDefaultConfiguration();
for (CollectionSchema f: defaultfacetfields) {
@ -376,7 +377,7 @@ public final class QueryParams {
return SetTools.anymatch(wordhashes, keyhashes);
}
public SolrQuery solrQuery() {
public SolrQuery solrQuery(boolean getFacets) {
if (this.cachedQuery != null) {
this.cachedQuery.setStart(this.offset);
return this.cachedQuery;
@ -512,6 +513,21 @@ public final class QueryParams {
params.setFilterQueries(fq.substring(5));
}
params.setStart(offset);
params.setRows(itemsPerPage);
// set facet query attributes
if (getFacets && this.facetfields.size() > 0) {
params.setFacet(true);
params.setFacetLimit(this.maxfacets);
params.setFacetSort(FacetParams.FACET_SORT_COUNT);
for (String field: this.facetfields) params.addFacetField(field);
} else {
params.setFacet(false);
}
params.setFields("*", "score"); // we need the score for post-ranking
// prepare result
Log.logInfo("Protocol", "SOLR QUERY: " + params.toString());
this.cachedQuery = params;

6
source/net/yacy/search/query/SearchEvent.java
Unescape View File

@ -276,7 +276,7 @@ public final class SearchEvent {
// start a local solr search
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(), 0, this.query.itemsPerPage, null /*this peer*/, Switchboard.urlBlacklist);
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(true), 0, this.query.itemsPerPage, null /*this peer*/, Switchboard.urlBlacklist);
}
this.localsolroffset = this.query.itemsPerPage;
@ -1317,7 +1317,7 @@ public final class SearchEvent {
int nextitems = item - this.localsolroffset + this.query.itemsPerPage; // example: suddenly switch to item 60, just 10 had been shown, 20 loaded.
if (this.localsolrsearch != null && this.localsolrsearch.isAlive()) {try {this.localsolrsearch.join();} catch (InterruptedException e) {}}
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(), this.localsolroffset, nextitems, null /*this peer*/, Switchboard.urlBlacklist);
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.localsolroffset == 0), this.localsolroffset, nextitems, null /*this peer*/, Switchboard.urlBlacklist);
}
this.localsolroffset += nextitems;
}
@ -1338,7 +1338,7 @@ public final class SearchEvent {
if (this.localsolrsearch == null || !this.localsolrsearch.isAlive() && this.local_solr_stored.get() > this.localsolroffset && (item + 1) % this.query.itemsPerPage == 0) {
// at the end of a list, trigger a next solr search
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(), this.localsolroffset, this.query.itemsPerPage, null /*this peer*/, Switchboard.urlBlacklist);
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.localsolroffset == 0), this.localsolroffset, this.query.itemsPerPage, null /*this peer*/, Switchboard.urlBlacklist);
}
this.localsolroffset += this.query.itemsPerPage;
}

Write Preview
Loading…
Cancel
Save