Reduced number of search navigators refresh requests in JS resort mode

The SearchEvent listen to changes on each of its navigators, and the
information about their overall state is sent with each fetched search
item (as a "data-nav-generation" attribute). Then the browser can
regularly fetch a fresh version of yacysearchtrailer.html only if
necessary (when that nav-generation value change).
pull/135/head
luccioman 7 years ago
parent 2ac78e2cca
commit 8303e15419

@ -26,8 +26,11 @@ var currentPageNumber = 0;
/* Set to true to enable browser console log traces */
var logEnabled = false;
/* Indicates if the results feeders are running on the server */
var feedRunning = true;
/* Indicates if the results fetching from server is running */
var fetchingResults = true;
/* Holds the last known navigators generation known from the server */
var lastNavGeneration = null;
/**
* Refresh the results page, checking and eventually updating each result CSS class depending on its position.
@ -255,35 +258,67 @@ var processSidebar = function(data) {
// TODO: nav-vocabulary
// TODO: nav-about
/* Store the new nav-generation data attribute if provided */
var navGenerationHolder = $("#rankingButtons");
if(navGenerationHolder.length > 0) {
var newNavGenerationHolder = newSidebar.find("#rankingButtons");
if(newNavGenerationHolder.length > 0) {
var navGeneration = newNavGenerationHolder.data("nav-generation");
if(navGeneration != null) {
navGenerationHolder.data("nav-generation", navGeneration);
lastNavGeneration = navGeneration;
}
}
}
}
// TODO: figure out if a better timeout strategy is feasible
if(feedRunning) {
if(fetchingResults) {
setTimeout(updateSidebar, 500);
}
};
/**
* Update the search navigators (facets) sidebar if necessary.
*/
var updateSidebar = function() {
var trailerParams = {
var navGenerationHolder = $("#rankingButtons");
var shouldUpdateSideBar = true;
if(navGenerationHolder.length > 0) {
var oldNavGeneration = navGenerationHolder.data("nav-generation");
if(oldNavGeneration != null && lastNavGeneration != null && oldNavGeneration == lastNavGeneration) {
/* nav-generation has not changed : no need to refresh the navigators side bar*/
shouldUpdateSideBar = false;
if(logEnabled) {
console.log("Prevented unnecessary sidebar update");
}
}
}
if(shouldUpdateSideBar) {
var trailerParams = {
"eventID": theEventID,
"resource": "global"
};
var searchForm = document.forms.searchform;
if(searchForm != null) {
if(searchForm.resource != null && searchForm.resource.value != null) {
trailerParams.resource = searchForm.resource.value;
}
if(searchForm.auth != null && searchForm.auth.value != null) {
trailerParams.auth = searchForm.auth.value;
}
}
$.get(
"yacysearchtrailer.html",
trailerParams,
processSidebar
);
//$("#sidebar").load("yacysearchtrailer.html", {"eventID": theEventID});
};
var searchForm = document.forms.searchform;
if(searchForm != null) {
if(searchForm.resource != null && searchForm.resource.value != null) {
trailerParams.resource = searchForm.resource.value;
}
if(searchForm.auth != null && searchForm.auth.value != null) {
trailerParams.auth = searchForm.auth.value;
}
}
$.get(
"yacysearchtrailer.html",
trailerParams,
processSidebar
);
} else if(fetchingResults) {
setTimeout(updateSidebar, 500);
}
};
/**
@ -296,7 +331,7 @@ var processLatestInfo = function(latestInfo) {
eventID : theEventID
}, processItem);
} else {
feedRunning = false;
fetchingResults = false;
}
}
@ -355,6 +390,12 @@ var processItem = function(data) {
}
});
}
var navGeneration = newItem.data("nav-generation");
if(navGeneration != null) {
/* Store the navigators generation new value when the item provided this info */
lastNavGeneration = navGeneration;
}
displayPage(false, currentPageNumber);

@ -1,5 +1,5 @@
#(content)#::
<div class="searchresults" data-ranking="#[ranking]#">
<div class="searchresults" data-ranking="#[ranking]#" data-nav-generation="#[nav-generation]#">
<h4 class="linktitle">
#(favicon)#::
<img width="16" height="16" src="#[faviconUrl]#" id="f#[urlhash]#" class="favicon" style="width:16px; height:16px;" alt="" />

@ -177,6 +177,10 @@ public class yacysearchitem {
prop.putXML("content_title-xml", result.title());
prop.putJSON("content_title-json", result.title());
prop.putHTML("content_showPictures_link", resultUrlstring);
/* Add information about the current search navigators to let browser refresh yacysearchtrailer only if needed */
prop.put("content_nav-generation", theSearch.getNavGeneration());
//prop.putHTML("content_link", resultUrlstring);
// START interaction
@ -391,6 +395,7 @@ public class yacysearchitem {
prop.put("content_loc_lat", result.lat());
prop.put("content_loc_lon", result.lon());
}
final boolean clustersearch = sb.isRobinsonMode() && sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "").equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER);
final boolean indexReceiveGranted = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW_SEARCH, true) || clustersearch;
boolean p2pmode = sb.peers != null && sb.peers.sizeConnected() > 0 && indexReceiveGranted;

@ -43,7 +43,8 @@
#(/resource-select)#
<p class="navbutton"></p>
<div class="btn-group btn-group-justified">
<div id="rankingButtons" class="btn-group btn-group-justified" data-nav-generation="#[nav-generation]#">
<!-- data-nav-generation attribute helps the browser know whether the search navigators have to be refreshed -->
<div class="btn-group btn-group-xs"><button type="button" id="sort_button_context" class="btn btn-default" onclick="document.getElementById('search').value=document.getElementById('search').value.replace(' /date','');document.searchform.submit();">Context Ranking</button></div>
<div class="btn-group btn-group-xs"><button type="button" id="sort_button_date" class="btn btn-default" onclick="document.getElementById('search').value=document.getElementById('search').value + ' /date';document.searchform.submit();">Sort by Date</button></div>
</div>

@ -100,6 +100,9 @@ public class yacysearchtrailer {
return prop;
}
final RequestHeader.FileType fileType = header.fileType();
/* Add information about the current navigators generation (number of updates since their initialization) */
prop.put("nav-generation", theSearch.getNavGeneration());
// compose search navigation
ContentDomain contentdom = theSearch.getQuery().contentdom;

@ -72,32 +72,30 @@ public final class ClusteredScoreMap<E> extends AbstractScoreMap<E> implements R
this.encnt = 0;
}
/**
* shrink the cluster to a demanded size
* @param maxsize
*/
@Override
public void shrinkToMaxSize(final int maxsize) {
if (maxsize < 0) return;
public int shrinkToMaxSize(final int maxsize) {
if (maxsize < 0) {
return 0;
}
Long key;
int deletedNb = 0;
synchronized (this) {
while (this.map.size() > maxsize) {
// find and remove smallest objects until cluster has demanded size
key = this.pam.firstKey();
if (key == null) break;
this.map.remove(this.pam.remove(key));
deletedNb++;
}
}
return deletedNb;
}
/**
* shrink the cluster in such a way that the smallest score is equal or greater than a given minScore
* @param minScore
*/
@Override
public void shrinkToMinScore(final int minScore) {
public int shrinkToMinScore(final int minScore) {
int score;
Long key;
int deletedNb = 0;
synchronized (this) {
while (!this.pam.isEmpty()) {
// find and remove objects where their score is smaller than the demanded minimum score
@ -106,8 +104,10 @@ public final class ClusteredScoreMap<E> extends AbstractScoreMap<E> implements R
score = (int) ((key.longValue() & 0xFFFFFFFF00000000L) >> 32);
if (score >= minScore) break;
this.map.remove(this.pam.remove(key));
deletedNb++;
}
}
return deletedNb;
}
private long scoreKey(final int elementNr, final int elementCount) {

@ -40,12 +40,35 @@ import java.util.concurrent.atomic.AtomicInteger;
public class ConcurrentScoreMap<E> extends AbstractScoreMap<E> implements ScoreMap<E> {
protected final ConcurrentHashMap<E, AtomicInteger> map; // a mapping from a reference to the cluster key
private long gcount; // sum of all scores
/** a mapping from a reference to the cluster key */
protected final ConcurrentHashMap<E, AtomicInteger> map;
/** sum of all scores */
private long gcount;
/** Eventual registered object listening on map updates */
private ScoreMapUpdatesListener updatesListener;
public ConcurrentScoreMap() {
this(null);
}
/**
* @param updatesListener an eventual object listening on score map updates
*/
public ConcurrentScoreMap(final ScoreMapUpdatesListener updatesListener) {
this.map = new ConcurrentHashMap<E, AtomicInteger>();
this.gcount = 0;
this.updatesListener = updatesListener;
}
/**
* Dispatch the update event to the eventually registered listener.
*/
private void dispatchUpdateToListener() {
if(this.updatesListener != null) {
this.updatesListener.updatedScoreMap();
}
}
@Override
@ -57,34 +80,40 @@ public class ConcurrentScoreMap<E> extends AbstractScoreMap<E> implements ScoreM
public synchronized void clear() {
this.map.clear();
this.gcount = 0;
dispatchUpdateToListener();
}
/**
* shrink the cluster to a demanded size
* @param maxsize
*/
@Override
public void shrinkToMaxSize(final int maxsize) {
if (this.map.size() <= maxsize) return;
public int shrinkToMaxSize(final int maxsize) {
if (this.map.size() <= maxsize) {
return 0;
}
int deletedNb = 0;
int minScore = getMinScore();
while (this.map.size() > maxsize) {
minScore++;
shrinkToMinScore(minScore);
deletedNb += shrinkToMinScore(minScore);
}
// No need to dispatch to listener, it is already done in shrinkToMinScore()
return deletedNb;
}
/**
* shrink the cluster in such a way that the smallest score is equal or greater than a given minScore
* @param minScore
*/
@Override
public void shrinkToMinScore(final int minScore) {
public int shrinkToMinScore(final int minScore) {
final Iterator<Map.Entry<E, AtomicInteger>> i = this.map.entrySet().iterator();
Map.Entry<E, AtomicInteger> entry;
int deletedNb = 0;
while (i.hasNext()) {
entry = i.next();
if (entry.getValue().intValue() < minScore) i.remove();
if (entry.getValue().intValue() < minScore) {
i.remove();
deletedNb++;
}
}
if(deletedNb > 0) {
dispatchUpdateToListener();
}
return deletedNb;
}
public long totalCount() {
@ -116,6 +145,8 @@ public class ConcurrentScoreMap<E> extends AbstractScoreMap<E> implements ScoreM
// increase overall counter
this.gcount++;
dispatchUpdateToListener();
}
@Override
@ -128,6 +159,8 @@ public class ConcurrentScoreMap<E> extends AbstractScoreMap<E> implements ScoreM
// increase overall counter
this.gcount--;
dispatchUpdateToListener();
}
@Override
@ -143,6 +176,8 @@ public class ConcurrentScoreMap<E> extends AbstractScoreMap<E> implements ScoreM
}
// increase overall counter
this.gcount += newScore;
dispatchUpdateToListener();
}
@Override
@ -155,6 +190,8 @@ public class ConcurrentScoreMap<E> extends AbstractScoreMap<E> implements ScoreM
// increase overall counter
this.gcount += incrementScore;
dispatchUpdateToListener();
}
@Override
@ -171,6 +208,9 @@ public class ConcurrentScoreMap<E> extends AbstractScoreMap<E> implements ScoreM
// decrease overall counter
this.gcount -= score.intValue();
dispatchUpdateToListener();
return score.intValue();
}
@ -266,6 +306,13 @@ public class ConcurrentScoreMap<E> extends AbstractScoreMap<E> implements ScoreM
}
return sortedKeys.iterator();
}
/**
* @param updatesListener an eventual object which wants to listen to successful updates on this score map
*/
public void setUpdatesListener(final ScoreMapUpdatesListener updatesListener) {
this.updatesListener = updatesListener;
}
public static void main(final String[] args) {
final ConcurrentScoreMap<String> a = new ConcurrentScoreMap<String>();

@ -62,34 +62,35 @@ public class OrderedScoreMap<E> extends AbstractScoreMap<E> implements ScoreMap<
this.map.clear();
}
/**
* shrink the cluster to a demanded size
* @param maxsize
*/
@Override
public void shrinkToMaxSize(final int maxsize) {
if (this.map.size() <= maxsize) return;
public int shrinkToMaxSize(final int maxsize) {
if (this.map.size() <= maxsize) {
return 0;
}
int deletedNb = 0;
int minScore = getMinScore();
while (this.map.size() > maxsize) {
minScore++;
shrinkToMinScore(minScore);
deletedNb += shrinkToMinScore(minScore);
}
return deletedNb;
}
/**
* shrink the cluster in such a way that the smallest score is equal or greater than a given minScore
* @param minScore
*/
@Override
public void shrinkToMinScore(final int minScore) {
public int shrinkToMinScore(final int minScore) {
int deletedNb = 0;
synchronized (this.map) {
final Iterator<Map.Entry<E, AtomicInteger>> i = this.map.entrySet().iterator();
Map.Entry<E, AtomicInteger> entry;
while (i.hasNext()) {
entry = i.next();
if (entry.getValue().intValue() < minScore) i.remove();
if (entry.getValue().intValue() < minScore) {
i.remove();
deletedNb++;
}
}
}
return deletedNb;
}
@Override

@ -34,15 +34,17 @@ public interface ScoreMap<E> extends Iterable<E> {
/**
* shrink the cluster to a demanded size
* @param maxsize
* @param maxsize the new expected size
* @return the number of deleted entries, zero when maxsize is greater than the current map size
*/
public void shrinkToMaxSize(int maxsize);
public int shrinkToMaxSize(int maxsize);
/**
* shrink the cluster in such a way that the smallest score is equal or greater than a given minScore
* @param minScore
* @param minScore the new expected minimum score
* @return the number of deleted entries or zero when minScore is lower than the currently minimum score of the map
*/
public void shrinkToMinScore(int minScore);
public int shrinkToMinScore(int minScore);
/**
@ -58,6 +60,11 @@ public interface ScoreMap<E> extends Iterable<E> {
public void set(final E obj, final int newScore);
/**
* Delete a given entry and return its previous score
* @param obj the entry to delete
* @return the score of the deleted entry, or zero when the map did not contain that entry
*/
public int delete(final E obj);
public boolean containsKey(final E obj);

@ -0,0 +1,38 @@
// ScoreMapUpdatesListener.java
// Copyright 2017 by luccioman; https://github.com/luccioman
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.cora.sorting;
import java.util.EventListener;
/**
* A listener interface to monitor successful write operation on a score map (inc(), dec(), delete(), shrinkToMaxSize(),
* shrinkToMinScore() and clear()).
*/
public interface ScoreMapUpdatesListener extends EventListener {
/**
* Called when a successful update (inc(), dec(), delete(), shrinkToMaxSize(),
* shrinkToMinScore() and clear()) has been performed on the watched score map.
*/
public void updatedScoreMap();
}

@ -24,8 +24,10 @@ package net.yacy.search.navigator;
import java.util.List;
import java.util.Map;
import net.yacy.cora.sorting.ReversibleScoreMap;
import net.yacy.cora.sorting.ScoreMap;
import net.yacy.cora.sorting.ScoreMapUpdatesListener;
import net.yacy.kelondro.data.meta.URIMetadataNode;
import net.yacy.search.query.QueryModifier;
@ -98,4 +100,9 @@ public interface Navigator extends ScoreMap<String> {
* @return the name of the index field, the fieldname counted in incDoc, incDoclList, incFacet
*/
public String getIndexFieldName();
/**
* @param listener an eventual object which wants to listen to successful updates on this navigator score map
*/
public void setUpdatesListener(final ScoreMapUpdatesListener listener);
}

@ -148,9 +148,9 @@ public class StringNavigator extends ConcurrentScoreMap<String> implements Navi
@Override
public String getIndexFieldName() {
if (field != null)
return field.getSolrFieldName();
else
return "";
if (this.field != null) {
return this.field.getSolrFieldName();
}
return "";
}
}

@ -46,6 +46,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Pattern;
import org.apache.solr.common.SolrDocument;
@ -66,6 +67,7 @@ import net.yacy.cora.protocol.Domains;
import net.yacy.cora.sorting.ConcurrentScoreMap;
import net.yacy.cora.sorting.ReversibleScoreMap;
import net.yacy.cora.sorting.ScoreMap;
import net.yacy.cora.sorting.ScoreMapUpdatesListener;
import net.yacy.cora.sorting.WeakPriorityBlockingQueue;
import net.yacy.cora.sorting.WeakPriorityBlockingQueue.Element;
import net.yacy.cora.sorting.WeakPriorityBlockingQueue.ReverseElement;
@ -108,7 +110,7 @@ import net.yacy.search.schema.CollectionSchema;
import net.yacy.search.snippet.TextSnippet;
import net.yacy.search.snippet.TextSnippet.ResultClass;
public final class SearchEvent {
public final class SearchEvent implements ScoreMapUpdatesListener {
/** Supported protocols to be displayed in the protocol navigator.
* (Using here a single String constant is faster than a unmodifiable Set instance) */
@ -178,6 +180,9 @@ public final class SearchEvent {
/** map of search custom/configured search navigators in addition to above standard navigators (which use special handling or display forms) */
public final Map<String, Navigator> navigatorPlugins;
/** Holds the total number of successful write operations performed on all the active navigators since their initialization. */
private final AtomicLong navGeneration = new AtomicLong();
private final LoaderDispatcher loader;
/** a set of word hashes that are used to match with the snippets */
@ -263,6 +268,15 @@ public final class SearchEvent {
/** Ensure only one {@link #resortCachedResults()} operation to be performed on this search event */
public final Semaphore resortCacheAllowed;
/**
* Called when a search navigator has been updated : update the overall
* navGeneration counter to help then tracking changes and eventually refresh the yacysearchtrailer.
*/
@Override
public void updatedScoreMap() {
this.navGeneration.incrementAndGet();
}
/**
* @return the total number of results currently available and filtered (checking doubles and eventual query constraints/modifiers) from the different data sources
@ -275,6 +289,13 @@ public final class SearchEvent {
);
}
/**
* @return the total number of successful write operations performed on all the active navigators since their initialization.
*/
public long getNavGeneration() {
return this.navGeneration.get();
}
/**
* Set maximum size allowed (in kbytes) for a remote document result to be stored to local index.
* @param maxSize document content max size in kbytes. Zero or negative value means no limit.
@ -335,14 +356,19 @@ public final class SearchEvent {
this.excludeintext_image = Switchboard.getSwitchboard().getConfigBool("search.excludeintext.image", true);
// prepare configured search navigation
final String navcfg = Switchboard.getSwitchboard().getConfig("search.navigation", "");
this.locationNavigator = navcfg.contains("location") ? new ConcurrentScoreMap<String>() : null;
this.protocolNavigator = navcfg.contains("protocol") ? new ConcurrentScoreMap<String>() : null;
this.dateNavigator = navcfg.contains("date") ? new ConcurrentScoreMap<String>() : null;
this.locationNavigator = navcfg.contains("location") ? new ConcurrentScoreMap<String>(this) : null;
this.protocolNavigator = navcfg.contains("protocol") ? new ConcurrentScoreMap<String>(this) : null;
this.dateNavigator = navcfg.contains("date") ? new ConcurrentScoreMap<String>(this) : null;
this.topicNavigatorCount = navcfg.contains("topics") ? MAX_TOPWORDS : 0;
this.languageNavigator = navcfg.contains("language") ? new ConcurrentScoreMap<String>() : null;
this.languageNavigator = navcfg.contains("language") ? new ConcurrentScoreMap<String>(this) : null;
this.vocabularyNavigator = new TreeMap<String, ScoreMap<String>>();
// prepare configured search navigation (plugins)
this.navigatorPlugins = NavigatorPlugins.initFromCfgString(navcfg);
if(this.navigatorPlugins != null) {
for(final Navigator nav : this.navigatorPlugins.values()) {
nav.setUpdatesListener(this);
}
}
this.snippets = new ConcurrentHashMap<String, LinkedHashSet<String>>();
this.secondarySearchSuperviser = (this.query.getQueryGoal().getIncludeHashes().size() > 1) ? new SecondarySearchSuperviser(this) : null; // generate abstracts only for combined searches
@ -378,7 +404,7 @@ public final class SearchEvent {
// attention: if minEntries is too high, this method will not terminate within the maxTime
// sortorder: 0 = hash, 1 = url, 2 = ranking
this.localSearchInclusion = null;
this.ref = new ConcurrentScoreMap<String>();
this.ref = new ConcurrentScoreMap<String>(this);
this.maxtime = query.maxtime;
this.rwiStack = new WeakPriorityBlockingQueue<WordReferenceVars>(max_results_rwi, false);
this.doubleDomCache = new ConcurrentHashMap<String, WeakPriorityBlockingQueue<WordReferenceVars>>();

Loading…
Cancel
Save