boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fixed false multiple-generation of remote facet search which

Browse Source 
caused high cpu usage on remote side.
pull/1/head
Michael Peter Christen 12 years ago
parent 23fb458963
commit 8caaf6203a
4 changed files with 28 additions and 25 deletions
Show Stats Download Patch File Download Diff File

18
source/net/yacy/peers/Protocol.java
Unescape View File

@ -1005,7 +1005,6 @@ public final class Protocol {
final SolrQuery solrQuery, final SolrQuery solrQuery,
final int offset, final int offset,
final int count, final int count,
boolean getFacets,
Seed target, Seed target,
final Blacklist blacklist) { final Blacklist blacklist) {
@ -1013,19 +1012,10 @@ public final class Protocol {
return -1; // we cannot query solr only with word hashes, there is no clear text string return -1; // we cannot query solr only with word hashes, there is no clear text string
} }
event.addExpectedRemoteReferences(count); event.addExpectedRemoteReferences(count);
solrQuery.setStart(offset); solrQuery.setStart(offset);
solrQuery.setRows(count); solrQuery.setRows(count);
// set facet query attributes
if (getFacets && event.query.facetfields.size() > 0) {
solrQuery.setFacet(true);
solrQuery.setFacetLimit(event.query.maxfacets);
solrQuery.setFacetSort(FacetParams.FACET_SORT_COUNT);
for (String field: event.query.facetfields) solrQuery.addFacetField(field);
} else {
solrQuery.setFacet(false);
}
// set highlighting query attributes // set highlighting query attributes
solrQuery.setHighlight(true); solrQuery.setHighlight(true);
solrQuery.setHighlightFragsize(SearchEvent.SNIPPET_MAX_LENGTH); solrQuery.setHighlightFragsize(SearchEvent.SNIPPET_MAX_LENGTH);
@ -1035,8 +1025,6 @@ public final class Protocol {
solrQuery.setHighlightSnippets(1); solrQuery.setHighlightSnippets(1);
for (CollectionSchema field: snippetFields) solrQuery.addHighlightField(field.getSolrFieldName()); for (CollectionSchema field: snippetFields) solrQuery.addHighlightField(field.getSolrFieldName());
solrQuery.setFields("*", "score"); // we need the score for post-ranking
boolean localsearch = target == null || target.equals(event.peers.mySeed()); boolean localsearch = target == null || target.equals(event.peers.mySeed());
if (localsearch && Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL, false)) { if (localsearch && Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL, false)) {
target = event.peers.mySeed(); target = event.peers.mySeed();
@ -1083,7 +1071,7 @@ public final class Protocol {
if (c == 0) continue; if (c == 0) continue;
result.set(ff.getName(), c); result.set(ff.getName(), c);
} }
facets.put(field, result); if (result.size() > 0) facets.put(field, result);
} }
// evaluate snippets // evaluate snippets
@ -1112,7 +1100,7 @@ public final class Protocol {
return 0; return 0;
} }
Network.log.logInfo("SEARCH (solr), returned " + docList.size() + " out of " + docList.getNumFound() + " documents from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName()))); Network.log.logInfo("SEARCH (solr), returned " + docList.size() + " out of " + docList.getNumFound() + " documents and " + facets.size() + " facets " + facets.keySet().toString() + " from " + (target == null ? "shard" : ("peer " + target.hash + ":" + target.getName())));
int term = count; int term = count;
Collection<SolrInputDocument> docs = new ArrayList<SolrInputDocument>(docList.size()); Collection<SolrInputDocument> docs = new ArrayList<SolrInputDocument>(docList.size());
for (final SolrDocument doc: docList) { for (final SolrDocument doc: docList) {

3
source/net/yacy/peers/RemoteSearch.java
Unescape View File

@ -172,7 +172,7 @@ public class RemoteSearch extends Thread {
nodePeers.add(event.peers.mySeed()); nodePeers.add(event.peers.mySeed());
} }
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_OFF, false)) { if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_OFF, false)) {
final SolrQuery solrQuery = event.query.solrQuery(); final SolrQuery solrQuery = event.query.solrQuery(start == 0);
for (Seed s: nodePeers) { for (Seed s: nodePeers) {
Thread t = solrRemoteSearch(event, solrQuery, start, count, s, blacklist); Thread t = solrRemoteSearch(event, solrQuery, start, count, s, blacklist);
event.nodeSearchThreads.add(t); event.nodeSearchThreads.add(t);
@ -288,7 +288,6 @@ public class RemoteSearch extends Thread {
solrQuery, solrQuery,
start, start,
count, count,
start == 0,
targetPeer, targetPeer,
blacklist); blacklist);
if (urls >= 0) { if (urls >= 0) {

26
source/net/yacy/search/query/QueryParams.java
Unescape View File

@ -30,7 +30,7 @@ import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.LinkedHashSet;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@ -38,6 +38,7 @@ import java.util.regex.PatternSyntaxException;
import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrQuery.SortClause; import org.apache.solr.client.solrj.SolrQuery.SortClause;
import org.apache.solr.common.params.FacetParams;
import net.yacy.cora.document.ASCII; import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.analysis.Classification; import net.yacy.cora.document.analysis.Classification;
@ -119,7 +120,7 @@ public final class QueryParams {
public final String userAgent; public final String userAgent;
protected boolean filterfailurls, filterscannerfail; protected boolean filterfailurls, filterscannerfail;
protected double lat, lon, radius; protected double lat, lon, radius;
public List<String> facetfields; public LinkedHashSet<String> facetfields;
public int maxfacets; public int maxfacets;
private SolrQuery cachedQuery; private SolrQuery cachedQuery;
private CollectionConfiguration solrSchema; private CollectionConfiguration solrSchema;
@ -165,7 +166,7 @@ public final class QueryParams {
this.lat = 0.0d; this.lat = 0.0d;
this.lon = 0.0d; this.lon = 0.0d;
this.radius = 0.0d; this.radius = 0.0d;
this.facetfields = new ArrayList<String>(); this.facetfields = new LinkedHashSet<String>();
this.solrSchema = indexSegment.fulltext().getDefaultConfiguration(); this.solrSchema = indexSegment.fulltext().getDefaultConfiguration();
for (CollectionSchema f: defaultfacetfields) { for (CollectionSchema f: defaultfacetfields) {
@ -266,7 +267,7 @@ public final class QueryParams {
this.lat = Math.floor(lat * this.kmNormal) / this.kmNormal; this.lat = Math.floor(lat * this.kmNormal) / this.kmNormal;
this.lon = Math.floor(lon * this.kmNormal) / this.kmNormal; this.lon = Math.floor(lon * this.kmNormal) / this.kmNormal;
this.radius = Math.floor(radius * this.kmNormal + 1) / this.kmNormal; this.radius = Math.floor(radius * this.kmNormal + 1) / this.kmNormal;
this.facetfields = new ArrayList<String>(); this.facetfields = new LinkedHashSet<String>();
this.solrSchema = indexSegment.fulltext().getDefaultConfiguration(); this.solrSchema = indexSegment.fulltext().getDefaultConfiguration();
for (CollectionSchema f: defaultfacetfields) { for (CollectionSchema f: defaultfacetfields) {
@ -376,7 +377,7 @@ public final class QueryParams {
return SetTools.anymatch(wordhashes, keyhashes); return SetTools.anymatch(wordhashes, keyhashes);
} }
public SolrQuery solrQuery() { public SolrQuery solrQuery(boolean getFacets) {
if (this.cachedQuery != null) { if (this.cachedQuery != null) {
this.cachedQuery.setStart(this.offset); this.cachedQuery.setStart(this.offset);
return this.cachedQuery; return this.cachedQuery;
@ -512,6 +513,21 @@ public final class QueryParams {
params.setFilterQueries(fq.substring(5)); params.setFilterQueries(fq.substring(5));
} }
params.setStart(offset);
params.setRows(itemsPerPage);
// set facet query attributes
if (getFacets && this.facetfields.size() > 0) {
params.setFacet(true);
params.setFacetLimit(this.maxfacets);
params.setFacetSort(FacetParams.FACET_SORT_COUNT);
for (String field: this.facetfields) params.addFacetField(field);
} else {
params.setFacet(false);
}
params.setFields("*", "score"); // we need the score for post-ranking
// prepare result // prepare result
Log.logInfo("Protocol", "SOLR QUERY: " + params.toString()); Log.logInfo("Protocol", "SOLR QUERY: " + params.toString());
this.cachedQuery = params; this.cachedQuery = params;

6
source/net/yacy/search/query/SearchEvent.java
Unescape View File

@ -276,7 +276,7 @@ public final class SearchEvent {
// start a local solr search // start a local solr search
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) { if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(), 0, this.query.itemsPerPage, null /*this peer*/, Switchboard.urlBlacklist); this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(true), 0, this.query.itemsPerPage, null /*this peer*/, Switchboard.urlBlacklist);
} }
this.localsolroffset = this.query.itemsPerPage; this.localsolroffset = this.query.itemsPerPage;
@ -1317,7 +1317,7 @@ public final class SearchEvent {
int nextitems = item - this.localsolroffset + this.query.itemsPerPage; // example: suddenly switch to item 60, just 10 had been shown, 20 loaded. int nextitems = item - this.localsolroffset + this.query.itemsPerPage; // example: suddenly switch to item 60, just 10 had been shown, 20 loaded.
if (this.localsolrsearch != null && this.localsolrsearch.isAlive()) {try {this.localsolrsearch.join();} catch (InterruptedException e) {}} if (this.localsolrsearch != null && this.localsolrsearch.isAlive()) {try {this.localsolrsearch.join();} catch (InterruptedException e) {}}
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) { if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(), this.localsolroffset, nextitems, null /*this peer*/, Switchboard.urlBlacklist); this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.localsolroffset == 0), this.localsolroffset, nextitems, null /*this peer*/, Switchboard.urlBlacklist);
} }
this.localsolroffset += nextitems; this.localsolroffset += nextitems;
} }
@ -1338,7 +1338,7 @@ public final class SearchEvent {
if (this.localsolrsearch == null || !this.localsolrsearch.isAlive() && this.local_solr_stored.get() > this.localsolroffset && (item + 1) % this.query.itemsPerPage == 0) { if (this.localsolrsearch == null || !this.localsolrsearch.isAlive() && this.local_solr_stored.get() > this.localsolroffset && (item + 1) % this.query.itemsPerPage == 0) {
// at the end of a list, trigger a next solr search // at the end of a list, trigger a next solr search
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) { if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(), this.localsolroffset, this.query.itemsPerPage, null /*this peer*/, Switchboard.urlBlacklist); this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.localsolroffset == 0), this.localsolroffset, this.query.itemsPerPage, null /*this peer*/, Switchboard.urlBlacklist);
} }
this.localsolroffset += this.query.itemsPerPage; this.localsolroffset += this.query.itemsPerPage;
} }

Write Preview
Loading…
Cancel
Save