boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Improved search navigators counters accuracy and consistency.

Browse Source 
- added some missing increments from RWI results
- decrement relevant navigator counts when solr or RWI results are
evicted because duplicates detection or constraints checked belatedly
- do not compute facets when unnecessary to avoid unwanted CPU load
- do not increment from facets when already done
- do not rely on facets on remote solr peers requests, as most of the
time only a limited part of their total results if fetched (thus also
preventing unnecessary load on remote peers)
- use a concurrency friendly score map for the dates navigators to
prevent unwanted ConcurrentModificationExceptions

This improves the situation for the most obvious inconsistencies in
search navigators counts, but more has to be done for a true accuracy
(notably when query modifiers constraints are applied belatedly - after
the solr or RWI retrieval request - such as the content domain
constraint)
pull/127/head
luccioman 8 years ago
parent ba0ba75998
commit 5d3ceb31b7
8 changed files with 580 additions and 170 deletions
Show Stats Download Patch File Download Diff File

5
htroot/yacysearchtrailer.java
Unescape View File

@ -246,7 +246,7 @@ public class yacysearchtrailer {
prop.put("nav-dates", 0);
} else {
prop.put("nav-dates", 1);
navigatorIterator = theSearch.dateNavigator.iterator(); // this iterator is different as it iterates by the key order (which is a date order)
navigatorIterator = theSearch.dateNavigator.keysByNaturalOrder(true); // this iterator is different as it iterates by the key order (which is a date order)
int i = 0, pos = 0, neg = 0;
long dx = -1;
Date fromconstraint = theSearch.getQuery().modifier.from == null ? null : DateDetection.parseLine(theSearch.getQuery().modifier.from, theSearch.getQuery().timezoneOffset);
@ -257,6 +257,9 @@ public class yacysearchtrailer {
name = navigatorIterator.next().trim();
if (name.length() < 10) continue;
count = theSearch.dateNavigator.get(name);
if(count == 0) {
continue;
}
String shortname = name.substring(0, 10);
long d = Instant.parse(name).toEpochMilli();
Date dd = new Date(d);

9
source/net/yacy/cora/federate/AbstractFederateSearchConnector.java
Unescape View File

@ -29,19 +29,21 @@ import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import javax.servlet.http.HttpServletResponse;
import org.apache.solr.common.SolrDocument;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.federate.solr.SchemaConfiguration;
import net.yacy.cora.federate.solr.SchemaDeclaration;
import net.yacy.cora.federate.solr.SolrType;
import net.yacy.cora.sorting.ReversibleScoreMap;
import net.yacy.cora.storage.Configuration;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.kelondro.data.meta.URIMetadataNode;
import net.yacy.search.Switchboard;
import net.yacy.search.query.SearchEvent;
import net.yacy.search.schema.CollectionSchema;
import org.apache.solr.common.SolrDocument;
/**
* Base implementation class for Federated Search Connectors providing the basic
@ -115,8 +117,7 @@ abstract public class AbstractFederateSearchConnector implements FederateSearchC
if (doclist != null) {
ConcurrentLog.info("YACY SEARCH (federated)", "Got " + doclist.size() + " documents from " + instancename);
Map<String, LinkedHashSet<String>> snippets = new HashMap<String, LinkedHashSet<String>>(); // add nodes doesn't allow null
Map<String, ReversibleScoreMap<String>> facets = new HashMap<String, ReversibleScoreMap<String>>(); // add nodes doesn't allow null
theSearch.addNodes(doclist, facets, snippets, false, instancename, doclist.size());
theSearch.addNodes(doclist, null, snippets, false, instancename, doclist.size(), true);
for (URIMetadataNode doc : doclist) {
theSearch.addHeuristic(doc.hash(), instancename, false);

20
source/net/yacy/cora/sorting/ConcurrentScoreMap.java
Unescape View File

@ -25,12 +25,14 @@
package net.yacy.cora.sorting;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
@ -246,6 +248,24 @@ public class ConcurrentScoreMap<E> extends AbstractScoreMap<E> implements ScoreM
for (int i = l.size() - 1; i >= 0; i--) r.add(l.get(i));
return r.iterator();
}
/**
* Creates and returns a sorted view of the keys, sorted by their own natural order.
* @param up true = asc order, false = reverse order
* @return iterator accessing the keys in natural order
*/
public Iterator<E> keysByNaturalOrder(final boolean up) {
TreeSet<E> sortedKeys;
if(up) {
sortedKeys = new TreeSet<>();
} else {
sortedKeys = new TreeSet<>(Collections.reverseOrder());
}
for(E key : this.map.keySet()) {
sortedKeys.add(key);
}
return sortedKeys.iterator();
}
public static void main(final String[] args) {
final ConcurrentScoreMap<String> a = new ConcurrentScoreMap<String>();

9
source/net/yacy/kelondro/data/word/WordReferenceVars.java
Unescape View File

@ -154,7 +154,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
/**
* initializer for special poison object
*/
public WordReferenceVars() {
private WordReferenceVars() {
this.flags = null;
this.lastModified = 0;
this.language = null;
@ -210,6 +210,13 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
public byte[] getLanguage() {
return ASCII.getBytes(this.language);
}
/**
* @return the ISO 639 language code of the reference
*/
public String getLanguageString() {
return this.language;
}
@Override
public char getType() {

9
source/net/yacy/kelondro/util/ISO639.java
Unescape View File

@ -257,11 +257,14 @@ public class ISO639 {
}
/**
* see if the given country in alpha-2 country code exists
* @param code, the mnemonic of the country in alpha-2
* @return true if the code exists
* Check if the given country in alpha-2 country code is supported.
* @param code, the mnemonic of the country in alpha-2 (ISO 639-1)
* @return true if the code is not null and is known by this YaCy server
*/
public static final boolean exists(String code) {
if(code == null) {
return false;
}
return mapping.containsKey(code.toLowerCase(Locale.ROOT));
}

47
source/net/yacy/peers/Protocol.java
Unescape View File

@ -746,8 +746,7 @@ public final class Protocol {
} else {
// feed results as nodes (SolrQuery results) which carry metadata,
// to prevent a call to getMetaData for RWI results, which would fail (if no metadata in index and no display of these results)
Map<String, ReversibleScoreMap<String>> facets = new HashMap<String, ReversibleScoreMap<String>>();
event.addNodes(storeDocs, facets, snip, false, target.getName() + "/" + target.hash, count);
event.addNodes(storeDocs, null, snip, false, target.getName() + "/" + target.hash, count, true);
}
event.addFinalize();
event.addExpectedRemoteReferences(-count);
@ -1110,6 +1109,8 @@ public final class Protocol {
* @param target target peer to query. May be null : in that case, local peer is queried.
* @param partitions
* @param blacklist url list to exclude from results
* @param useSolrFacets when true, use Solr computed facets when possible to update the event navigators counters
* @param incrementNavigators when true, increment event navigators either with facet counts or with individual results
* @return the size of results list
* @throws InterruptedException when interrupt status on calling thread is detected while processing
*/
@ -1120,7 +1121,9 @@ public final class Protocol {
final int count,
final Seed target,
final int partitions,
final Blacklist blacklist) throws InterruptedException {
final Blacklist blacklist,
final boolean useSolrFacets,
final boolean incrementNavigators) throws InterruptedException {
//try {System.out.println("*** debug-query *** " + URLDecoder.decode(solrQuery.toString(), "UTF-8"));} catch (UnsupportedEncodingException e) {}
@ -1205,18 +1208,28 @@ public final class Protocol {
}
// evaluate facets
for (String field: event.query.facetfields) {
FacetField facet = rsp[0].getFacetField(field);
ReversibleScoreMap<String> result = new ClusteredScoreMap<String>(UTF8.insensitiveUTF8Comparator);
List<Count> values = facet == null ? null : facet.getValues();
if (values == null) continue;
for (Count ff: values) {
int c = (int) ff.getCount();
if (c == 0) continue;
if (ff.getName().length() == 0) continue; // facet entry without text is not useful
result.set(ff.getName(), c);
}
if (result.size() > 0) facets.put(field, result);
if(useSolrFacets) {
for (String field: event.query.facetfields) {
FacetField facet = rsp[0].getFacetField(field);
ReversibleScoreMap<String> result = new ClusteredScoreMap<String>(UTF8.insensitiveUTF8Comparator);
List<Count> values = facet == null ? null : facet.getValues();
if (values == null) {
continue;
}
for (Count ff: values) {
int c = (int) ff.getCount();
if (c == 0) {
continue;
}
if (ff.getName().length() == 0) {
continue; // facet entry without text is not useful
}
result.set(ff.getName(), c);
}
if (result.size() > 0) {
facets.put(field, result);
}
}
}
// evaluate snippets
@ -1331,7 +1344,7 @@ public final class Protocol {
docList[0].clear();
docList[0] = null;
if (localsearch) {
event.addNodes(resultContainer, facets, snippets, true, "localpeer", numFound);
event.addNodes(resultContainer, facets, snippets, true, "localpeer", numFound, incrementNavigators);
event.addFinalize();
event.addExpectedRemoteReferences(-count);
Network.log.info("local search (solr): localpeer sent " + resultContainer.size() + "/" + numFound + " references");
@ -1347,7 +1360,7 @@ public final class Protocol {
docs); // will clear docs on return
writeToLocalIndexThread.start();
}
event.addNodes(resultContainer, facets, snippets, false, target.getName() + "/" + target.hash, numFound);
event.addNodes(resultContainer, facets, snippets, false, target.getName() + "/" + target.hash, numFound, incrementNavigators);
event.addFinalize();
event.addExpectedRemoteReferences(-count);
Network.log.info("remote search (solr): peer " + target.getName() + " sent " + (resultContainer.size()) + "/" + numFound + " references");

23
source/net/yacy/peers/RemoteSearch.java
Unescape View File

@ -252,19 +252,26 @@ public class RemoteSearch extends Thread {
log.info("preparing remote search: shortmem=" + (shortmem ? "true" : "false") + ", indexingQueueSize=" + indexingQueueSize +
", redundancy=" + redundancy + ", minage=" + minage + ", dhtPeers=" + dhtPeers.size() + ", robinsonpeers=" + robinsonPeers.size() + ", health: " + (healthMessage.length() > 0 ? healthMessage.substring(2) : "perfect"));
/* Computing Solr facets is not relevant for remote Solr results and adds unnecessary CPU load on remote peers :
* facets count the total number of matching results per facet field, but we only fetch here at most 'count' results. The remaining part
* is not to be retrieved from remote peers even if making a new request filtering on one of these fields,
* as there is no insurance the same remote peers would be selected. What's more, remote results can contain many
* duplicates that would be filtered when adding them to the event node stack.
*/
final boolean useFacets = false;
// start solr searches
final int targets = dhtPeers.size() + robinsonPeers.size();
if (!sb.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_OFF, false)) {
final SolrQuery solrQuery = event.query.solrQuery(event.getQuery().contentdom, start == 0, event.excludeintext_image);
final SolrQuery solrQuery = event.query.solrQuery(event.getQuery().contentdom, useFacets, event.excludeintext_image);
for (Seed s: robinsonPeers) {
if (MemoryControl.shortStatus()
|| Memory.load() > sb.getConfigFloat(SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR,
SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR_DEFAULT)) {
continue;
}
Thread t = solrRemoteSearch(event, solrQuery, start, count, s, targets, blacklist);
Thread t = solrRemoteSearch(event, solrQuery, start, count, s, targets, blacklist, useFacets, true);
event.nodeSearchThreads.add(t);
}
}
@ -365,6 +372,8 @@ public class RemoteSearch extends Thread {
* @param targetPeer the target of the Solr query. When null, the query will run on this local peer.
* @param partitions the Solr query "partitions" parameter. Ignored when set to zero.
* @param blacklist the blacklist to use. Can be empty but must not be null.
* @param useSolrFacets when true, use Solr computed facets when possible to update the event navigators counters
* @param incrementNavigators when true, increment event navigators either with facet counts or with individual results
* @return the created and running Thread instance
*/
public static Thread solrRemoteSearch(
@ -374,7 +383,9 @@ public class RemoteSearch extends Thread {
final int count,
final Seed targetPeer,
final int partitions,
final Blacklist blacklist) {
final Blacklist blacklist,
final boolean useSolrFacets,
final boolean incrementNavigators) {
//System.out.println("*** debug-remoteSearch ***:" + ConcurrentLog.stackTrace());
@ -396,7 +407,9 @@ public class RemoteSearch extends Thread {
count,
targetPeer == null ? event.peers.mySeed() : targetPeer,
partitions,
blacklist);
blacklist,
useSolrFacets,
incrementNavigators);
if (urls >= 0) {
// urls is an array of url hashes. this is only used for log output
event.peers.mySeed().incRI(urls);

628
source/net/yacy/search/query/SearchEvent.java
Unescape View File

@ -30,6 +30,7 @@ import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.ConcurrentModificationException;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
@ -50,6 +51,7 @@ import java.util.regex.Pattern;
import org.apache.solr.common.SolrDocument;
import net.yacy.contentcontrol.ContentControlFilterUpdateThread;
import net.yacy.cora.date.ISO8601Formatter;
import net.yacy.cora.document.analysis.Classification;
import net.yacy.cora.document.analysis.Classification.ContentDomain;
import net.yacy.cora.document.encoding.ASCII;
@ -61,7 +63,6 @@ import net.yacy.cora.federate.yacy.Distribution;
import net.yacy.cora.lod.vocabulary.Tagging;
import net.yacy.cora.order.Base64Order;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.sorting.ClusteredScoreMap;
import net.yacy.cora.sorting.ConcurrentScoreMap;
import net.yacy.cora.sorting.ReversibleScoreMap;
import net.yacy.cora.sorting.ScoreMap;
@ -108,6 +109,10 @@ import net.yacy.search.snippet.TextSnippet;
import net.yacy.search.snippet.TextSnippet.ResultClass;
public final class SearchEvent {
/** Supported protocols to be displayed in the protocol navigator.
* (Using here a single String constant is faster than a unmodifiable Set instance) */
private static final String PROTOCOL_NAVIGATOR_SUPPORTED_VALUES = "http,https,smb,ftp,file";
private static final int max_results_rwi = 3000;
private static final int max_results_node = 150;
@ -151,15 +156,15 @@ public final class SearchEvent {
/** counter for referenced that had been sorted out for other reasons */
private final AtomicInteger expectedRemoteReferences, maxExpectedRemoteReferences;
/** a counter for the appearance of location coordinates */
/** a counter for the appearance of location coordinates .*/
public final ScoreMap<String> locationNavigator;
/** a counter for protocol types */
public final ScoreMap<String> protocolNavigator;
/** a counter for file types */
public final ScoreMap<String> dateNavigator;
public final ConcurrentScoreMap<String> dateNavigator;
/** a counter for appearance of languages */
public final ScoreMap<String> languageNavigator;
@ -265,7 +270,7 @@ public final class SearchEvent {
public int getResultCount() {
return Math.max(
this.local_rwi_available.get() + this.remote_rwi_available.get() +
this.remote_solr_available.get() + this.local_solr_stored.get() - this.local_solr_evicted.get(),
this.remote_solr_available.get() + Math.max(0, this.local_solr_stored.get() - this.local_solr_evicted.get()),
imageViewed.size() + sizeSpare()
);
}
@ -332,7 +337,7 @@ public final class SearchEvent {
final String navcfg = Switchboard.getSwitchboard().getConfig("search.navigation", "");
this.locationNavigator = navcfg.contains("location") ? new ConcurrentScoreMap<String>() : null;
this.protocolNavigator = navcfg.contains("protocol") ? new ConcurrentScoreMap<String>() : null;
this.dateNavigator = navcfg.contains("date") ? new ClusteredScoreMap<String>(true) : null;
this.dateNavigator = navcfg.contains("date") ? new ConcurrentScoreMap<String>() : null;
this.topicNavigatorCount = navcfg.contains("topics") ? MAX_TOPWORDS : 0;
this.languageNavigator = navcfg.contains("language") ? new ConcurrentScoreMap<String>() : null;
this.vocabularyNavigator = new TreeMap<String, ScoreMap<String>>();
@ -395,7 +400,10 @@ public final class SearchEvent {
// start a local solr search
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.query.contentdom, true, this.excludeintext_image), this.query.offset, this.query.itemsPerPage, null /*this peer*/, 0, Switchboard.urlBlacklist);
final boolean useSolrFacets = true;
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this,
this.query.solrQuery(this.query.contentdom, useSolrFacets, this.excludeintext_image), this.query.offset,
this.query.itemsPerPage, null /* this peer */, 0, Switchboard.urlBlacklist, useSolrFacets, true);
}
this.localsolroffset = this.query.offset + this.query.itemsPerPage;
@ -708,7 +716,16 @@ public final class SearchEvent {
if (log.isFine()) log.fine("dropped RWI: contentdom fail");
continue pollloop;
}
// check language
if (this.query.modifier.language != null && !this.query.modifier.language.isEmpty()
&& !this.query.modifier.language.equals(iEntry.getLanguageString())) {
if (log.isFine()) {
log.fine("dropped RWI: language constraint = " + this.query.modifier.language);
}
continue pollloop;
}
// count domZones
//this.domZones[DigestURI.domDomain(iEntry.metadataHash())]++;
@ -741,6 +758,15 @@ public final class SearchEvent {
}
// increase counter for statistics
if (local) this.local_rwi_available.incrementAndGet(); else this.remote_rwi_available.incrementAndGet();
/* Fisrt filtering pass is now complete : update navigator counters that can already be updated */
if(this.languageNavigator != null) {
final String lang = iEntry.getLanguageString();
if(ISO639.exists(lang)) {
this.languageNavigator.inc(lang);
}
}
successcounter++;
}
if (System.currentTimeMillis() >= timeout) ConcurrentLog.warn("SearchEvent", "rwi normalization ended with timeout = " + maxtime);
@ -849,13 +875,24 @@ public final class SearchEvent {
}
}
/**
* Add result entries to this nodes stack and update eventual navigators counters.
* @param nodeList a list of entries from a Solr instance
* @param facets a map from a field name to scored values (aka Solr facet). May be null : in that case the navigators counters are updated one by one when inserting each result in the nodes stack.
* @param solrsnippets a map from urlhash to snippet text
* @param local true when the nodeList comes from the local Solr
* @param resourceName the name of the data source to use for monitoring in the event tracker
* @param fullResource the full results count
* @param incrementNavigators when true, increment event navigators either with facet counts or with individual results
*/
public void addNodes(
final List<URIMetadataNode> nodeList,
final Map<String, ReversibleScoreMap<String>> facets, // a map from a field name to scored values
final Map<String, LinkedHashSet<String>> solrsnippets, // a map from urlhash to snippet text
final Map<String, ReversibleScoreMap<String>> facets,
final Map<String, LinkedHashSet<String>> solrsnippets,
final boolean local,
final String resourceName,
final int fullResource) {
final int fullResource,
final boolean incrementNavigators) {
this.addBegin();
@ -891,92 +928,20 @@ public final class SearchEvent {
// iterate over normalized entries and select some that are better than currently stored
timer = System.currentTimeMillis();
// collect navigation information
// iterate over active navigator plugins to let them update the counters
for (String s : this.navigatorPlugins.keySet()) {
Navigator navi = this.navigatorPlugins.get(s);
if (navi != null) {
if (facets == null || facets.isEmpty() || !facets.containsKey(navi.getIndexFieldName())) { // just in case we got no solr facet
navi.incDocList(nodeList);
} else {
navi.incFacet(facets);
}
}
}
ReversibleScoreMap<String> fcts;
if (this.locationNavigator != null) {
fcts = facets.get(CollectionSchema.coordinate_p_0_coordinate.getSolrFieldName());
if (fcts != null) {
for (String coordinate: fcts) {
int hc = fcts.get(coordinate);
if (hc == 0) continue;
this.locationNavigator.inc(coordinate, hc);
}
}
}
if (this.dateNavigator != null) {
fcts = facets.get(CollectionSchema.dates_in_content_dts.getSolrFieldName());
if (fcts != null) this.dateNavigator.inc(fcts);
}
if (this.languageNavigator != null) {
fcts = facets.get(CollectionSchema.language_s.getSolrFieldName());
if (fcts != null) {
// remove unknown languages
Iterator<String> i = fcts.iterator();
while (i.hasNext()) {
String lang = i.next();
if (!ISO639.exists(lang)) {
i.remove();
}
}
this.languageNavigator.inc(fcts);
}
}
if (this.protocolNavigator != null) {
fcts = facets.get(CollectionSchema.url_protocol_s.getSolrFieldName());
if (fcts != null) {
// remove all protocols that we don't know
Iterator<String> i = fcts.iterator();
while (i.hasNext()) {
String protocol = i.next();
if ("http,https,smb,ftp,file".indexOf(protocol) < 0) i.remove();
}
this.protocolNavigator.inc(fcts);
}
}
// get the vocabulary navigation
Set<String> genericFacets = new LinkedHashSet<>();
for (Tagging v: LibraryProvider.autotagging.getVocabularies()) genericFacets.add(v.getName());
genericFacets.addAll(ProbabilisticClassifier.getContextNames());
for (String v: genericFacets) {
fcts = facets.get(CollectionSchema.VOCABULARY_PREFIX + v + CollectionSchema.VOCABULARY_TERMS_SUFFIX);
if (fcts != null) {
ScoreMap<String> vocNav = this.vocabularyNavigator.get(v);
if (vocNav == null) {
vocNav = new ConcurrentScoreMap<String>();
this.vocabularyNavigator.put(v, vocNav);
}
vocNav.inc(fcts);
}
// collect navigation information from Solr facets when available
if(incrementNavigators) {
incrNavigatorsFromSolrFacets(facets);
}
// apply all constraints
try {
pollloop: for (URIMetadataNode iEntry: nodeList) {
if ( !this.query.urlMask_isCatchall ) {
// check url mask
if (!iEntry.matches(this.query.urlMaskPattern)) {
if (log.isFine()) log.fine("dropped Node: url mask does not match");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
updateCountsOnSolrEntryToEvict(iEntry, facets, local, !incrementNavigators);
continue pollloop;
}
}
@ -984,9 +949,7 @@ public final class SearchEvent {
// doublecheck for urls
if (this.urlhashes.has(iEntry.hash())) {
if (log.isFine()) log.fine("dropped Node: double check");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
updateCountsOnSolrEntryToEvict(iEntry, facets, local, !incrementNavigators);
continue pollloop;
}
@ -999,9 +962,7 @@ public final class SearchEvent {
Bitfield flags = iEntry.flags();
if (!this.testFlags(flags)) {
if (log.isFine()) log.fine("dropped Node: flag test");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
updateCountsOnSolrEntryToEvict(iEntry, facets, local, !incrementNavigators);
continue pollloop;
}
@ -1012,9 +973,7 @@ public final class SearchEvent {
(this.query.contentdom == ContentDomain.IMAGE && !(flags.get(Tokenizer.flag_cat_hasimage))) ||
(this.query.contentdom == ContentDomain.APP && !(flags.get(Tokenizer.flag_cat_hasapp))))) {
if (log.isFine()) log.fine("dropped Node: content domain does not match");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
updateCountsOnSolrEntryToEvict(iEntry, facets, local, !incrementNavigators);
continue pollloop;
}
@ -1022,9 +981,7 @@ public final class SearchEvent {
String ext = MultiProtocolURL.getFileExtension(iEntry.url().getFileName());
if (this.query.contentdom == ContentDomain.TEXT && Classification.isImageExtension(ext) && this.excludeintext_image) {
if (log.isFine()) log.fine("dropped Node: file name domain does not match");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
updateCountsOnSolrEntryToEvict(iEntry, facets, local, !incrementNavigators);
continue pollloop;
}
@ -1033,18 +990,14 @@ public final class SearchEvent {
if ( this.query.modifier.sitehash == null ) {
if (this.query.siteexcludes != null && this.query.siteexcludes.contains(hosthash)) {
if (log.isFine()) log.fine("dropped Node: siteexclude");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
updateCountsOnSolrEntryToEvict(iEntry, facets, local, !incrementNavigators);
continue pollloop;
}
} else {
// filter out all domains that do not match with the site constraint
if (iEntry.url().getHost().indexOf(this.query.modifier.sitehost) < 0) {
if (log.isFine()) log.fine("dropped Node: sitehost");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
updateCountsOnSolrEntryToEvict(iEntry, facets, local, !incrementNavigators);
continue pollloop;
}
}
@ -1052,9 +1005,7 @@ public final class SearchEvent {
if (this.query.modifier.language != null) {
if (!this.query.modifier.language.equals(iEntry.language())) {
if (log.isFine()) log.fine("dropped Node: language");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
updateCountsOnSolrEntryToEvict(iEntry, facets, local, !incrementNavigators);
continue pollloop;
}
}
@ -1062,9 +1013,7 @@ public final class SearchEvent {
if (this.query.modifier.author != null) {
if (!this.query.modifier.author.equals(iEntry.dc_creator())) {
if (log.isFine()) log.fine ("dropped Node: author");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
updateCountsOnSolrEntryToEvict(iEntry, facets, local, !incrementNavigators);
continue pollloop;
}
}
@ -1072,9 +1021,7 @@ public final class SearchEvent {
if (this.query.modifier.keyword != null) {
if (iEntry.dc_subject().indexOf(this.query.modifier.keyword) < 0) {
if (log.isFine()) log.fine ("dropped Node: keyword");
if (local) {
this.local_solr_evicted.incrementAndGet();
}
updateCountsOnSolrEntryToEvict(iEntry, facets, local, !incrementNavigators);
continue pollloop;
}
}
@ -1102,11 +1049,187 @@ public final class SearchEvent {
if (!local) {
this.remote_solr_available.incrementAndGet();
}
// collect navigation information not available in facets
if(incrementNavigators) {
incrNavigatorsFromSingleDocument(iEntry, facets);
}
}
} catch (final SpaceExceededException e ) {
}
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEventType.PRESORT, resourceName, nodeList.size(), System.currentTimeMillis() - timer), false);
}
/**
* Increment this event eventual navigators with the given facets processed by a Solr instance
* @param facets facets counts from a Solr instance
*/
private void incrNavigatorsFromSolrFacets(final Map<String, ReversibleScoreMap<String>> facets) {
if(facets != null && !facets.isEmpty()) {
/* Iterate over active navigator plugins to let them update the counters */
for (String s : this.navigatorPlugins.keySet()) {
Navigator navi = this.navigatorPlugins.get(s);
if (navi != null) {
navi.incFacet(facets);
}
}
ReversibleScoreMap<String> fcts;
if (this.locationNavigator != null) {
/* Is is still relevant? It looks like this nav is currently never filled, as a constraint on coordinates
* is expressed as a spatial filter not producing facets counts (see QueryParams.getFacetsFilterQueries()). */
fcts = facets.get(CollectionSchema.coordinate_p_0_coordinate.getSolrFieldName());
if (fcts != null) {
for (String coordinate: fcts) {
int hc = fcts.get(coordinate);
if (hc == 0) continue;
this.locationNavigator.inc(coordinate, hc);
}
}
}
if (this.dateNavigator != null) {
fcts = facets.get(CollectionSchema.dates_in_content_dts.getSolrFieldName());
if (fcts != null) this.dateNavigator.inc(fcts);
}
if (this.languageNavigator != null) {
fcts = facets.get(CollectionSchema.language_s.getSolrFieldName());
if (fcts != null) {
// remove unknown languages
Iterator<String> i = fcts.iterator();
while (i.hasNext()) {
String lang = i.next();
if (!ISO639.exists(lang)) {
i.remove();
}
}
this.languageNavigator.inc(fcts);
}
}
if (this.protocolNavigator != null) {
fcts = facets.get(CollectionSchema.url_protocol_s.getSolrFieldName());
if (fcts != null) {
// remove all protocols that we don't know
Iterator<String> i = fcts.iterator();
while (i.hasNext()) {
String protocol = i.next();
if (PROTOCOL_NAVIGATOR_SUPPORTED_VALUES.indexOf(protocol) < 0) {
i.remove();
}
}
this.protocolNavigator.inc(fcts);
}
}
// get the vocabulary navigation
Set<String> genericFacets = new LinkedHashSet<>();
for (Tagging v: LibraryProvider.autotagging.getVocabularies()) genericFacets.add(v.getName());
genericFacets.addAll(ProbabilisticClassifier.getContextNames());
for (String vocName: genericFacets) {
fcts = facets.get(CollectionSchema.VOCABULARY_PREFIX + vocName + CollectionSchema.VOCABULARY_TERMS_SUFFIX);
if (fcts != null) {
ScoreMap<String> vocNav = this.vocabularyNavigator.get(vocName);
if (vocNav == null) {
vocNav = new ConcurrentScoreMap<String>();
this.vocabularyNavigator.put(vocName, vocNav);
}
vocNav.inc(fcts);
}
}
}
}
/**
* Increment this event eventual navigators with the given entry, only when the concerned field is not present in facets
* @param doc a document entry from a Solr source
* @param facets facets counts from a Solr instance
*/
private void incrNavigatorsFromSingleDocument(final URIMetadataNode doc,
final Map<String, ReversibleScoreMap<String>> facets) {
/* Iterate over active navigator plugins to let them update the counters */
for (String s : this.navigatorPlugins.keySet()) {
Navigator navi = this.navigatorPlugins.get(s);
if (navi != null && facets == null || !facets.containsKey(navi.getIndexFieldName())) {
navi.incDoc(doc);
}
}
/* Note : would it be relevant to update here this.locationNavigator ?
It looks like this nav is currently never filled */
if (this.dateNavigator != null) {
if (facets == null || !facets.containsKey(CollectionSchema.dates_in_content_dts.getSolrFieldName())) {
Date[] dates = doc.datesInContent();
if (dates != null) {
for (final Date date : dates) {
if (date != null) {
this.dateNavigator.inc(ISO8601Formatter.FORMATTER.format(date));
}
}
}
}
}
if (this.languageNavigator != null) {
if (facets == null || !facets.containsKey(CollectionSchema.language_s.getSolrFieldName())) {
final String lang = doc.language();
if (ISO639.exists(lang)) {
this.languageNavigator.inc(lang);
}
}
}
if (this.protocolNavigator != null) {
if (facets == null || !facets.containsKey(CollectionSchema.url_protocol_s.getSolrFieldName())) {
final String protocol = doc.url().getProtocol();
// include only protocols supported protocols
if (protocol != null && PROTOCOL_NAVIGATOR_SUPPORTED_VALUES.indexOf(protocol) >= 0) {
this.protocolNavigator.inc(protocol);
}
}
}
// get the vocabulary navigation
if(this.vocabularyNavigator != null) {
Set<String> genericFacets = new LinkedHashSet<>();
for (Tagging v : LibraryProvider.autotagging.getVocabularies()) {
genericFacets.add(v.getName());
}
genericFacets.addAll(ProbabilisticClassifier.getContextNames());
for (String vocName : genericFacets) {
final String fieldName = CollectionSchema.VOCABULARY_PREFIX + vocName + CollectionSchema.VOCABULARY_TERMS_SUFFIX;
if (facets == null || !facets.containsKey(fieldName)) {
Object docValue = doc.getFieldValue(fieldName);
if(docValue instanceof String) {
ScoreMap<String> vocNav = this.vocabularyNavigator.get(vocName);
if (vocNav == null) {
vocNav = new ConcurrentScoreMap<String>();
this.vocabularyNavigator.put(vocName, vocNav);
}
vocNav.inc((String)docValue);
} else if(docValue instanceof Collection) {
if (!((Collection<?>) docValue).isEmpty()) {
ScoreMap<String> vocNav = this.vocabularyNavigator.get(vocName);
if (vocNav == null) {
vocNav = new ConcurrentScoreMap<String>();
this.vocabularyNavigator.put(vocName, vocNav);
}
for (Object singleDocValue : (Collection<?>) docValue) {
if (singleDocValue instanceof String) {
vocNav.inc((String) singleDocValue);
}
}
}
}
}
}
}
}
public void addExpectedRemoteReferences(int x) {
if ( x > 0 ) {
@ -1136,7 +1259,10 @@ public final class SearchEvent {
if (rwi == null) return null;
if (!skipDoubleDom) {
URIMetadataNode node = this.query.getSegment().fulltext().getMetadata(rwi);
if (node == null) continue pollloop;
if (node == null) {
decrementCounts(rwi.getElement());
continue pollloop;
}
return node;
}
@ -1151,7 +1277,10 @@ public final class SearchEvent {
m = new WeakPriorityBlockingQueue<WordReferenceVars>(max_results_rwi, false);
this.doubleDomCache.put(hosthash, m);
URIMetadataNode node = this.query.getSegment().fulltext().getMetadata(rwi);
if (node == null) continue pollloop;
if (node == null) {
decrementCounts(rwi.getElement());
continue pollloop;
}
return node;
}
// second appearances of dom
@ -1217,7 +1346,7 @@ public final class SearchEvent {
ConcurrentLog.logException(e);
}
if (node == null) {
if (bestEntry.getElement().local()) this.local_rwi_available.decrementAndGet(); else this.remote_rwi_available.decrementAndGet();
decrementCounts(bestEntry.getElement());
if (log.isFine()) log.fine("dropped RWI: hash not in metadata");
continue mainloop;
}
@ -1236,20 +1365,19 @@ public final class SearchEvent {
*/
public URIMetadataNode pullOneFilteredFromRWI(final boolean skipDoubleDom) {
// returns from the current RWI list the best URL entry and removes this entry from the list
int p = -1;
URIMetadataNode page;
mainloop: while ((page = pullOneRWI(skipDoubleDom)) != null) {
if (!this.query.urlMask_isCatchall && !page.matches(this.query.urlMaskPattern)) {
if (log.isFine()) log.fine("dropped RWI: no match with urlMask");
if (page.word().local()) this.local_rwi_available.decrementAndGet(); else this.remote_rwi_available.decrementAndGet();
decrementCounts(page.word());
continue;
}
// check for more errors
if (page.url() == null) {
if (log.isFine()) log.fine("dropped RWI: url == null");
if (page.word().local()) this.local_rwi_available.decrementAndGet(); else this.remote_rwi_available.decrementAndGet();
decrementCounts(page.word());
continue; // rare case where the url is corrupted
}
@ -1261,7 +1389,7 @@ public final class SearchEvent {
(this.query.contentdom == Classification.ContentDomain.VIDEO && contentDomain != Classification.ContentDomain.VIDEO) ||
(this.query.contentdom == Classification.ContentDomain.APP && contentDomain != Classification.ContentDomain.APP)) && this.query.urlMask_isCatchall) {
if (log.isFine()) log.fine("dropped RWI: wrong contentdom = " + this.query.contentdom + ", domain = " + contentDomain);
if (page.word().local()) this.local_rwi_available.decrementAndGet(); else this.remote_rwi_available.decrementAndGet();
decrementCounts(page.word());
continue;
}
@ -1278,21 +1406,22 @@ public final class SearchEvent {
// check modifier constraint filetype (using fileextension)
if (this.query.modifier.filetype != null && !this.query.modifier.filetype.equals(ext)) {
if (log.isFine()) log.fine("dropped RWI: file type constraint = " + this.query.modifier.filetype);
if (page.word().local()) this.local_rwi_available.decrementAndGet(); else this.remote_rwi_available.decrementAndGet();
decrementCounts(page.word());
continue;
}
// check modifier constraint (language)
/* check again modifier constraint (language) with the language in the full metadata,
* that may differ from the one in the reverse word reference which is already checked in addRWIs()*/
if (this.query.modifier.language != null && !this.query.modifier.language.equals(page.language())) {
if (log.isFine()) log.fine("dropped RWI: language constraint = " + this.query.modifier.language);
if (page.word().local()) this.local_rwi_available.decrementAndGet(); else this.remote_rwi_available.decrementAndGet();
decrementCounts(page.word());
continue;
}
// check modifier constraint (author)
if (this.query.modifier.author != null && !page.dc_creator().toLowerCase().contains(this.query.modifier.author.toLowerCase()) /*!this.query.modifier.author.equalsIgnoreCase(page.dc_creator())*/) {
if (log.isFine()) log.fine("dropped RWI: author constraint = " + this.query.modifier.author);
if (page.word().local()) this.local_rwi_available.decrementAndGet(); else this.remote_rwi_available.decrementAndGet();
decrementCounts(page.word());
continue;
}
@ -1301,10 +1430,10 @@ public final class SearchEvent {
if (this.query.modifier.collection != null) {
Collection<Object> docCols = page.getFieldValues(CollectionSchema.collection_sxt.getSolrFieldName()); // get multivalued value
if (docCols == null) { // no collection info
if (page.word().local()) this.local_rwi_available.decrementAndGet(); else this.remote_rwi_available.decrementAndGet();
decrementCounts(page.word());
continue;
} else if (!docCols.contains(this.query.modifier.collection)) {
if (page.word().local()) this.local_rwi_available.decrementAndGet(); else this.remote_rwi_available.decrementAndGet();
decrementCounts(page.word());
continue;
}
}
@ -1312,14 +1441,14 @@ public final class SearchEvent {
// check modifier constraint (keyword)
if (this.query.modifier.keyword != null && !page.dc_subject().toLowerCase().contains(this.query.modifier.keyword.toLowerCase())) {
if (log.isFine()) log.fine("dropped RWI: keyword constraint = " + this.query.modifier.keyword);
if (page.word().local()) this.local_rwi_available.decrementAndGet(); else this.remote_rwi_available.decrementAndGet();
decrementCounts(page.word());
continue;
}
// Check for blacklist
if (Switchboard.urlBlacklist.isListed(BlacklistType.SEARCH, page.url())) {
if (log.isFine()) log.fine("dropped RWI: url is blacklisted in url blacklist");
if (page.word().local()) this.local_rwi_available.decrementAndGet(); else this.remote_rwi_available.decrementAndGet();
decrementCounts(page.word());
continue;
}
@ -1328,7 +1457,7 @@ public final class SearchEvent {
FilterEngine f = ContentControlFilterUpdateThread.getNetworkFilter();
if (f != null && !f.isListed(page.url(), null)) {
if (log.isFine()) log.fine("dropped RWI: url is blacklisted in contentcontrol");
if (page.word().local()) this.local_rwi_available.decrementAndGet(); else this.remote_rwi_available.decrementAndGet();
decrementCounts(page.word());
continue;
}
}
@ -1343,7 +1472,7 @@ public final class SearchEvent {
|| (QueryParams.anymatch(pageurl.toLowerCase(), this.query.getQueryGoal().getExcludeWords()))
|| (QueryParams.anymatch(pageauthor.toLowerCase(), this.query.getQueryGoal().getExcludeWords())))) {
if (log.isFine()) log.fine("dropped RWI: no match with query goal exclusion");
if (page.word().local()) this.local_rwi_available.decrementAndGet(); else this.remote_rwi_available.decrementAndGet();
decrementCounts(page.word());
continue;
}
@ -1356,14 +1485,14 @@ public final class SearchEvent {
}
}
if (log.isFine()) log.fine("dropped RWI: url does not match index-of constraint");
if (page.word().local()) this.local_rwi_available.decrementAndGet(); else this.remote_rwi_available.decrementAndGet();
decrementCounts(page.word());
continue;
}
// check location constraint
if ((this.query.constraint != null) && (this.query.constraint.get(Tokenizer.flag_cat_haslocation)) && (page.lat() == 0.0 || page.lon() == 0.0)) {
if (log.isFine()) log.fine("dropped RWI: location constraint");
if (page.word().local()) this.local_rwi_available.decrementAndGet(); else this.remote_rwi_available.decrementAndGet();
decrementCounts(page.word());
continue;
}
@ -1375,7 +1504,7 @@ public final class SearchEvent {
double distance = Math.sqrt(latDelta * latDelta + lonDelta * lonDelta); // pythagoras
if (distance > this.query.radius) {
if (log.isFine()) log.fine("dropped RWI: radius constraint");
if (page.word().local()) this.local_rwi_available.decrementAndGet(); else this.remote_rwi_available.decrementAndGet();
decrementCounts(page.word());
continue;
}
}
@ -1392,7 +1521,7 @@ public final class SearchEvent {
}
} // if we reach this point the metatag was not found (= drop entry)
if (log.isFine()) log.fine("dropped RWI: url not tagged with vocabulary " + tag.getVocabularyName());
if (page.word().local()) this.local_rwi_available.decrementAndGet(); else this.remote_rwi_available.decrementAndGet();
decrementCounts(page.word());
continue mainloop;
}
}
@ -1407,11 +1536,229 @@ public final class SearchEvent {
navi.incDoc(page);
}
}
if(this.languageNavigator != null) {
if(page.word() == null || page.word().getLanguageString() == null) {
/* Increase the language navigator here only if the word reference
* did not include information about language. Otherwise it should be done earlier in addRWIs() */
final String lang = page.language();
if(ISO639.exists(lang)) {
this.languageNavigator.inc(lang);
}
}
}
if(this.protocolNavigator != null && page.url() != null) {
final String protocol = page.url().getProtocol();
if(protocol != null) {
this.protocolNavigator.inc(protocol);
}
}
if(this.dateNavigator != null) {
Date[] dates = page.datesInContent();
if (dates != null) {
for (final Date date : dates) {
if (date != null) {
this.dateNavigator.inc(ISO8601Formatter.FORMATTER.format(date));
}
}
}
}
return page; // accept url
}
return null;
}
/**
* Decrement statistics counts for the given RWI entry.
* @param entry an RWI entry result
*/
private void decrementCounts(final WordReferenceVars entry) {
if(entry == null) {
return;
}
if (entry.local()) {
if(this.local_rwi_available.get() > 0) {
this.local_rwi_available.decrementAndGet();
}
} else {
if(this.remote_rwi_available.get() > 0) {
this.remote_rwi_available.decrementAndGet();
}
}
/* The language navigator may have been incremented in the addRWIs() function */
if (this.languageNavigator != null) {
String lang = entry.getLanguageString();
if (lang != null && this.languageNavigator.get(lang) > 0) {
this.languageNavigator.dec(lang);
}
}
}
/**
* Update counters when evicting a Solr entry from results.
* @param entry a Solr entry result to be evicted
* @param local when true the entry is coming from the local Solr
* @param facets facets counts from Solr
* @param navIncrementedEarlier when true, navigators have been incremented earlier with other facets or individual documents
*/
private void updateCountsOnSolrEntryToEvict(final URIMetadataNode entry,
final Map<String, ReversibleScoreMap<String>> facets, final boolean local,
final boolean navIncrementedEarlier) {
if (entry == null) {
return;
}
if (local) {
this.local_solr_evicted.incrementAndGet();
/*
* No need to decrement remote_solr_available as this counter is only
* incremented after all filterings have been applied
*/
}
/*
* Update eventual navigators counters when relevant
*/
final boolean navIncrementedWithFacets = facets != null && !facets.isEmpty() && !navIncrementedEarlier;
ReversibleScoreMap<String> fcts;
/* Iterate over active navigator plugins to let them update the counters */
for (String s : this.navigatorPlugins.keySet()) {
Navigator navi = this.navigatorPlugins.get(s);
if (navi != null) {
if (navIncrementedWithFacets) {
fcts = facets.get(navi.getIndexFieldName());
} else {
fcts = null;
}
final Object value = entry.getFieldValue(navi.getIndexFieldName());
if (value != null) {
if(value instanceof Collection) {
for (final Object singleVal : (Collection<?>) value) {
if (singleVal instanceof String) {
final String singleStringVal = (String)singleVal;
if (navIncrementedEarlier || (fcts != null && fcts.containsKey(singleStringVal))) {
if (navi.get(singleStringVal) > 0) {
navi.dec(singleStringVal);
}
}
}
}
} else if(value instanceof String){
final String stringValue = (String)value;
if (navIncrementedEarlier || (fcts != null && fcts.containsKey(stringValue))) {
if (navi.get(stringValue) > 0) {
navi.dec(stringValue);
}
}
}
}
}
}
/* Note : would it be relevant to update here this.locationNavigator ?
* It looks like this nav is currently never filled, as a constraint on coordinates
* is expressed as a spatial filter not producing facets counts (see QueryParams.getFacetsFilterQueries())
*/
if (this.dateNavigator != null) {
if (navIncrementedWithFacets) {
fcts = facets.get(CollectionSchema.dates_in_content_dts.getSolrFieldName());
} else {
fcts = null;
}
Date[] dates = entry.datesInContent();
if (dates != null) {
for (final Date date : dates) {
if (date != null) {
final String dateStr = ISO8601Formatter.FORMATTER.format(date);
if (navIncrementedEarlier || (fcts != null && fcts.containsKey(dateStr))) {
if (this.dateNavigator.get(dateStr) > 0) {
this.dateNavigator.dec(dateStr);
}
}
}
}
}
}
if (this.languageNavigator != null) {
if (navIncrementedWithFacets) {
fcts = facets.get(CollectionSchema.language_s.getSolrFieldName());
} else {
fcts = null;
}
String lang = entry.language();
if (lang != null) {
if (navIncrementedEarlier || (fcts != null && fcts.containsKey(lang))) {
if (this.languageNavigator.get(lang) > 0) {
this.languageNavigator.dec(lang);
}
}
}
}
if (this.protocolNavigator != null) {
if (navIncrementedWithFacets) {
fcts = facets.get(CollectionSchema.url_protocol_s.getSolrFieldName());
} else {
fcts = null;
}
final String protocol = entry.url().getProtocol();
if (protocol != null) {
if (navIncrementedEarlier || (fcts != null && fcts.containsKey(protocol))) {
if (this.protocolNavigator.get(protocol) > 0) {
this.protocolNavigator.dec(protocol);
}
}
}
}
// get the vocabulary navigation
if (this.vocabularyNavigator != null) {
Set<String> genericFacets = new LinkedHashSet<>();
for (Tagging v : LibraryProvider.autotagging.getVocabularies()) {
genericFacets.add(v.getName());
}
genericFacets.addAll(ProbabilisticClassifier.getContextNames());
for (String vocName : genericFacets) {
final String fieldName = CollectionSchema.VOCABULARY_PREFIX + vocName
+ CollectionSchema.VOCABULARY_TERMS_SUFFIX;
if (navIncrementedWithFacets) {
fcts = facets.get(fieldName);
} else {
fcts = null;
}
Object docValue = entry.getFieldValue(fieldName);
if (docValue instanceof String) {
if (navIncrementedEarlier || (fcts != null && fcts.containsKey((String) docValue))) {
ScoreMap<String> vocNav = this.vocabularyNavigator.get(vocName);
if (vocNav != null && vocNav.get((String) docValue) > 0) {
vocNav.dec((String) docValue);
}
}
} else if(docValue instanceof Collection) {
if (!((Collection<?>) docValue).isEmpty()) {
for (Object singleDocValue : (Collection<?>) docValue) {
if (singleDocValue instanceof String) {
if (navIncrementedEarlier || (fcts != null && fcts.containsKey((String) singleDocValue))) {
ScoreMap<String> vocNav = this.vocabularyNavigator.get(vocName);
if (vocNav != null && vocNav.get((String) singleDocValue) > 0) {
vocNav.dec((String) singleDocValue);
}
}
}
}
}
}
}
}
}
public long getURLRetrievalTime() {
return this.urlRetrievalAllTime;
@ -1721,7 +2068,7 @@ public final class SearchEvent {
log.warn("Wait for local solr search was interrupted.");
}
}
if (this.remote && item >= this.localsolroffset && this.local_solr_stored.get() >= item) {
if (this.remote && item >= this.localsolroffset && this.local_solr_stored.get() > item) {
/* Request mixing remote and local Solr results : load remaining local solr results now.
* For local only search, a new SearchEvent should be created, starting directly at the requested offset,
* thus allowing to handle last pages of large resultsets
@ -1729,9 +2076,12 @@ public final class SearchEvent {
int nextitems = item - this.localsolroffset + this.query.itemsPerPage; // example: suddenly switch to item 60, just 10 had been shown, 20 loaded.
if (this.localsolrsearch != null && this.localsolrsearch.isAlive()) {try {this.localsolrsearch.join();} catch (final InterruptedException e) {}}
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {
// Do not increment again navigators from the local Solr on next local pages retrieval, as facets counts scope is on the total results and should already have been added
final boolean useSolrFacets = (this.localsolrsearch == null);
final boolean incrementNavigators = false;
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this,
this.query.solrQuery(this.query.contentdom, false, this.excludeintext_image),
this.localsolroffset, nextitems, null /* this peer */, 0, Switchboard.urlBlacklist);
this.query.solrQuery(this.query.contentdom, useSolrFacets, this.excludeintext_image),
this.localsolroffset, nextitems, null /* this peer */, 0, Switchboard.urlBlacklist, useSolrFacets, incrementNavigators);
}
this.localsolroffset += nextitems;
}

Write Preview
Loading…
Cancel
Save