boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Enabled displaying results after 14th page for local search queries.

Browse Source 
Fixes issue #90 for local queries only: Stealth mode, Portal mode or
Intranet mode. 
For P2p mode, the issue would probably be difficult to solve with
reasonable performance. This is still to dig.

Also switched some InterreputedException catch log messages to warn
level as this is normal behavior when shutting down a peer.

Fixed yacysearch buttons navbar behavior to deal correctly with total
results count or offset over 1000. Also improved the buttons navbar to
be able to navigate over 10th page for local queries.
pull/97/head
luccioman 8 years ago
parent a3886c6adb
commit c25e48e969
7 changed files with 155 additions and 60 deletions
Show Stats Download Patch File Download Diff File

108
htroot/js/yacysearch.js
Unescape View File

@ -39,11 +39,72 @@ function fadeOutBar() {
document.getElementById("progressbar").setAttribute('style',"transition:transform 0s;-webkit-transition:-webkit-transform 0s;backgroundColor:transparent;");
}
function statistics(offset, itemscount, itemsperpage, totalcount, localResourceSize, remoteResourceSize, remoteIndexCount, remotePeerCount, navurlbase) {
if (totalcount == 0) return;
if (offset >= 0) document.getElementById("offset").innerHTML = offset;
if (offset >= 0) document.getElementById("startRecord").setAttribute('value', offset - 1);
if (itemscount >= 0) document.getElementById("itemscount").firstChild.nodeValue = itemscount;
/**
* @returns pagination buttons
*/
function renderPaginationButtons(offset, itemscount, itemsperpage, totalcount, localResourceSize, remoteResourceSize, remoteIndexCount, remotePeerCount, navurlbase, localQuery) {
var resnav = "<ul class=\"pagination\">";
var thispage = Math.floor(offset / itemsperpage);
var firstPage = thispage - (thispage % 10);
if (thispage == 0) {
resnav += "<li class=\"disabled\"><a href=\"#\">&laquo;</a></li>";
} else {
resnav += "<li><a id=\"prevpage\" href=\"";
resnav += (navurlbase + "&amp;startRecord=" + ((thispage - 1) * itemsperpage));
resnav += "\">&laquo;</a></li>";
}
var totalPagesNb = Math.floor(1 + ((totalcount - 1) / itemsperpage));
var numberofpages = Math.min(10, totalPagesNb - firstPage);
if (!numberofpages) numberofpages = 10;
for (i = firstPage; i < (firstPage + numberofpages); i++) {
if (i == thispage) {
resnav += "<li class=\"active\"><a href=\"#\">";
resnav += (i + 1);
resnav += "</a></li>";
} else {
resnav += "<li><a href=\"";
resnav += (navurlbase + "&amp;startRecord=" + (i * itemsperpage));
resnav += "\">" + (i + 1) + "</a></li>";
}
}
if ((localQuery && thispage >= (totalPagesNb - 1)) || (!localQuery && thispage >= (numberofpages - 1))) {
resnav += "<li class=\"disabled\"><a href=\"#\">&raquo;</a></li>";
} else {
resnav += "<li><a id=\"nextpage\" href=\"";
resnav += (navurlbase + "&amp;startRecord=" + ((thispage + 1) * itemsperpage));
resnav += "\">&raquo;</a>";
}
resnav += "</ul>";
return resnav;
}
/**
* Parses a string representing an integer value
* @param strIntValue formatted string
* @returns the number value or undefined when the string is undefined, or NaN when the string is not a number
*/
function parseFormattedInt(strIntValue) {
var inValue;
if(strIntValue != null && strIntValue.replace != null) {
/* Remove thousands separator and try to parse as integer */
intValue = parseInt(strIntValue.replace(/[\.\,]/g,''))
}
return intValue;
}
function statistics(offset, itemscount, itemsperpage, totalcount, localResourceSize, remoteResourceSize, remoteIndexCount, remotePeerCount, navurlbase, localQuery) {
var totalcountIntValue = parseFormattedInt(totalcount);
var offsetIntValue = parseFormattedInt(offset);
var itemscountIntValue = parseFormattedInt(itemscount);
var itemsperpageIntValue = parseFormattedInt(itemsperpage);
if (totalcountIntValue == 0) {
return;
}
if (offsetIntValue >= 0) document.getElementById("offset").innerHTML = offset;
if (offsetIntValue >= 0) document.getElementById("startRecord").setAttribute('value', offsetIntValue - 1);
if (itemscountIntValue >= 0) document.getElementById("itemscount").firstChild.nodeValue = itemscount;
document.getElementById("totalcount").firstChild.nodeValue = totalcount;
if (document.getElementById("localResourceSize") != null) document.getElementById("localResourceSize").firstChild.nodeValue = localResourceSize;
if (document.getElementById("remoteResourceSize") != null) document.getElementById("remoteResourceSize").firstChild.nodeValue = remoteResourceSize;
@ -52,7 +113,7 @@ function statistics(offset, itemscount, itemsperpage, totalcount, localResourceS
// compose page navigation
if (document.getElementById("progressbar").getAttribute('class') != "progress-bar progress-bar-success") {
var percent = 100 * (itemscount - offset + 1) / itemsperpage;
var percent = 100 * (itemscountIntValue - offsetIntValue + 1) / itemsperpageIntValue;
if (percent == 100) {
document.getElementById("progressbar").setAttribute('style',"transition:transform 0s;-webkit-transition:-webkit-transform 0s;");
document.getElementById("progressbar").setAttribute('class',"progress-bar progress-bar-success");
@ -62,38 +123,9 @@ function statistics(offset, itemscount, itemsperpage, totalcount, localResourceS
}
var resnavElement = document.getElementById("resNav");
if (resnavElement != null) {
var resnav = "<ul class=\"pagination\">";
thispage = Math.floor(offset / itemsperpage);
if (thispage == 0) {
resnav += "<li class=\"disabled\"><a href=\"#\">&laquo;</a></li>";
} else {
resnav += "<li><a id=\"prevpage\" href=\"";
resnav += (navurlbase + "&amp;startRecord=" + ((thispage - 1) * itemsperpage));
resnav += "\">&laquo;</a></li>";
}
numberofpages = Math.floor(Math.min(9, 1 + ((totalcount.replace(/\./g,'') - 1) / itemsperpage)));
if (!numberofpages) numberofpages = 9;
for (i = 0; i < numberofpages; i++) {
if (i == thispage) {
resnav += "<li class=\"active\"><a href=\"#\">";
resnav += (i + 1);
resnav += "</a></li>";
} else {
resnav += "<li><a href=\"";
resnav += (navurlbase + "&amp;startRecord=" + (i * itemsperpage));
resnav += "\">" + (i + 1) + "</a></li>";
}
}
if (thispage >= numberofpages) {
resnav += "<li><a href=\"#\">&raquo;</a></li>";
} else {
resnav += "<li><a id=\"nextpage\" href=\"";
resnav += (navurlbase + "&amp;startRecord=" + ((thispage + 1) * itemsperpage));
resnav += "\">&raquo;</a>";
}
resnav += "</ul>";
resnavElement.innerHTML = resnav;
resnavElement.innerHTML = renderPaginationButtons(offsetIntValue, itemscountIntValue, itemsperpageIntValue,
totalcountIntValue, parseFormattedInt(localResourceSize), parseFormattedInt(remoteResourceSize), parseFormattedInt(remoteIndexCount),
parseFormattedInt(remotePeerCount), navurlbase, localQuery);
}
}

3
htroot/yacysearch.html
Unescape View File

@ -222,7 +222,8 @@ function latestinfo() {
self.xmlHttpReq.onreadystatechange = function() {
if (self.xmlHttpReq.readyState == 4) {
var rsp = eval("(" + self.xmlHttpReq.responseText + ")");
statistics(rsp.offset, rsp.itemscount, rsp.itemsperpage, rsp.totalcount, rsp.localResourceSize, rsp.remoteResourceSize, rsp.remoteIndexCount, rsp.remotePeerCount, rsp.navurlBase);
statistics(rsp.offset, rsp.itemscount, rsp.itemsperpage, rsp.totalcount, rsp.localResourceSize, rsp.remoteResourceSize, rsp.remoteIndexCount, rsp.remotePeerCount, rsp.navurlBase, #[localQuery]#);
}
};
self.xmlHttpReq.send(null);

2
htroot/yacysearch.java
Unescape View File

@ -171,6 +171,7 @@ public class yacysearch {
prop.put("indexof", "off");
prop.put("constraint", "");
prop.put("depth", "0");
prop.put("localQuery", "0");
prop.put(
"search.verify",
(post == null) ? sb.getConfig("search.verify", "iffresh") : post.get("verify", "iffresh"));
@ -849,6 +850,7 @@ public class yacysearch {
}
prop.put("depth", "0");
prop.put("localQuery", theSearch.query.isLocal() ? "1" : "0");
}
prop.put("focus", focus ? 1 : 0); // focus search field

2
htroot/yacysearchitem.java
Unescape View File

@ -101,6 +101,7 @@ public class yacysearchitem {
prop.put("references", "0");
prop.put("rssreferences", "0");
prop.put("dynamic", "0");
prop.put("localQuery", "0");
// find search event
final SearchEvent theSearch = SearchEventCache.getEvent(eventID);
@ -119,6 +120,7 @@ public class yacysearchitem {
prop.put("remoteIndexCount", Formatter.number(theSearch.remote_rwi_available.get() + theSearch.remote_solr_available.get(), true));
prop.put("remotePeerCount", Formatter.number(theSearch.remote_rwi_peerCount.get() + theSearch.remote_solr_peerCount.get(), true));
prop.put("navurlBase", QueryParams.navurlBase(RequestHeader.FileType.HTML, theSearch.query, null, false).toString());
prop.put("localQuery", theSearch.query.isLocal() ? "1" : "0");
final String target_special_pattern = sb.getConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN, "");
long timeout = item == 0 ? 10000 : (theSearch.query.isLocal() ? 1000 : 3000);

8
source/net/yacy/search/query/QueryParams.java
Unescape View File

@ -187,7 +187,13 @@ public final class QueryParams {
this.contentdom = contentdom;
this.timezoneOffset = timezoneOffset;
this.itemsPerPage = Math.min((specialRights) ? 10000 : 1000, itemsPerPage);
this.offset = Math.max(0, Math.min((specialRights) ? 10000 - this.itemsPerPage : 1000 - this.itemsPerPage, offset));
if(domType == Searchdom.LOCAL) {
/* No offset restriction on local index only requests, as only itemsPerPage will be loaded */
this.offset = Math.max(0, offset);
} else {
/* Offset has to be limited on requests mixing local and remote results, because all results before offset are loaded */
this.offset = Math.max(0, Math.min((specialRights) ? 10000 - this.itemsPerPage : 1000 - this.itemsPerPage, offset));
}
try {
this.urlMaskString = urlMask;
// solr doesn't like slashes, backslashes or doublepoints; remove them // urlmask = ".*\\." + ft + "(\\?.*)?";

69
source/net/yacy/search/query/SearchEvent.java
Unescape View File

@ -142,6 +142,8 @@ public final class SearchEvent {
private byte[] IAmaxcounthash, IAneardhthash;
public Thread rwiProcess;
public Thread localsolrsearch;
/** Offset of the next local Solr index request
* Example : last local request with offset=10 and itemsPerPage=20, sets this attribute to 30. */
private int localsolroffset;
private final AtomicInteger expectedRemoteReferences, maxExpectedRemoteReferences; // counter for referenced that had been sorted out for other reasons
public final ScoreMap<String> locationNavigator; // a counter for the appearance of location coordinates
@ -323,9 +325,9 @@ public final class SearchEvent {
// start a local solr search
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.query.contentdom, true, this.excludeintext_image), 0, this.query.itemsPerPage, null /*this peer*/, 0, Switchboard.urlBlacklist);
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.query.contentdom, true, this.excludeintext_image), this.query.offset, this.query.itemsPerPage, null /*this peer*/, 0, Switchboard.urlBlacklist);
}
this.localsolroffset = this.query.itemsPerPage;
this.localsolroffset = this.query.offset + this.query.itemsPerPage;
// start a local RWI search concurrently
this.rwiProcess = null;
@ -1576,31 +1578,68 @@ public final class SearchEvent {
// (happens if a search pages is accessed a second time)
final long finishTime = timeout == Long.MAX_VALUE ? Long.MAX_VALUE : System.currentTimeMillis() + timeout;
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEventType.ONERESULT, "started, item = " + item + ", available = " + this.getResultCount(), 0, 0), false);
// wait until a local solr is finished, we must do that to be able to check if we need more
if (this.localsolrsearch != null && this.localsolrsearch.isAlive()) {try {this.localsolrsearch.join(100);} catch (final InterruptedException e) {}}
if (item >= this.localsolroffset && this.local_solr_stored.get() == 0 && this.localsolrsearch.isAlive()) {try {this.localsolrsearch.join();} catch (final InterruptedException e) {}}
if (item >= this.localsolroffset && this.local_solr_stored.get() >= item) {
// load remaining solr results now
if (this.localsolrsearch != null && this.localsolrsearch.isAlive()) {
try {
this.localsolrsearch.join(100);
} catch (final InterruptedException e) {
log.warn("Wait for local solr search was interrupted.");
}
}
if (item >= this.localsolroffset && this.local_solr_stored.get() == 0 && this.localsolrsearch.isAlive()) {
try {
this.localsolrsearch.join();
} catch (final InterruptedException e) {
log.warn("Wait for local solr search was interrupted.");
}
}
if (this.remote && item >= this.localsolroffset && this.local_solr_stored.get() >= item) {
/* Request mixing remote and local Solr results : load remaining local solr results now.
* For local only search, a new SearchEvent should be created, starting directly at the requested offset,
* thus allowing to handle last pages of large resultsets
*/
int nextitems = item - this.localsolroffset + this.query.itemsPerPage; // example: suddenly switch to item 60, just 10 had been shown, 20 loaded.
if (this.localsolrsearch != null && this.localsolrsearch.isAlive()) {try {this.localsolrsearch.join();} catch (final InterruptedException e) {}}
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.query.contentdom, false, this.excludeintext_image), this.localsolroffset, nextitems, null /*this peer*/, 0, Switchboard.urlBlacklist);
}
this.localsolroffset += nextitems;
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {
this.localsolrsearch = RemoteSearch.solrRemoteSearch(this,
this.query.solrQuery(this.query.contentdom, false, this.excludeintext_image),
this.localsolroffset, nextitems, null /* this peer */, 0, Switchboard.urlBlacklist);
}
this.localsolroffset += nextitems;
}
// now pull results as long as needed and as long as possible
if (this.remote && item < 10 && this.resultList.sizeAvailable() <= item) try {Thread.sleep(100);} catch (final InterruptedException e) {ConcurrentLog.logException(e);}
while ( this.resultList.sizeAvailable() <= item &&
if (this.remote && item < 10 && this.resultList.sizeAvailable() <= item) {
try {
Thread.sleep(100);
} catch (final InterruptedException e) {
log.warn("Remote search results wait was interrupted.");
}
}
final int resultListIndex;
if (this.remote) {
resultListIndex = item;
} else {
resultListIndex = item - (this.localsolroffset - this.query.itemsPerPage);
}
while ( this.resultList.sizeAvailable() <= resultListIndex &&
(this.rwiQueueSize() > 0 || this.nodeStack.sizeQueue() > 0 ||
(!this.feedingIsFinished() && System.currentTimeMillis() < finishTime))) {
if (!drainStacksToResult()) try {Thread.sleep(10);} catch (final InterruptedException e) {ConcurrentLog.logException(e);}
if (!drainStacksToResult()) {
try {
Thread.sleep(10);
} catch (final InterruptedException e) {
log.warn("Search results wait was interrupted.");
}
}
}
// check if we have a success
if (this.resultList.sizeAvailable() > item) {
if (this.resultList.sizeAvailable() > resultListIndex) {
// we have the wanted result already in the result array .. return that
final URIMetadataNode re = this.resultList.element(item).getElement();
final URIMetadataNode re = this.resultList.element(resultListIndex).getElement();
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEventType.ONERESULT, "fetched, item = " + item + ", available = " + this.getResultCount() + ": " + re.urlstring(), 0, 0), false);
/*

23
source/net/yacy/search/query/SearchEventCache.java
Unescape View File

@ -155,11 +155,24 @@ public class SearchEventCache {
event = null;
} else {
if (event != null) {
//re-new the event time for this event, so it is not deleted next time too early
event.resetEventTime();
// replace the current result offset
event.query.offset = query.offset;
event.query.itemsPerPage = query.itemsPerPage;
if(query.isLocal()) {
/* Searching the local index only : we do not reuse the cached event each time the page size or offset changes.
* This allow to request last result pages of large result sets (larger than SearchEvent.max_results_node)
* without the need to retrieve all the beginning pages */
if(event.query.offset != query.offset || event.query.itemsPerPage != query.itemsPerPage) {
synchronized (lastEvents) {
lastEvents.remove(id);
}
cacheDelete++;
event = null;
}
} else {
//re-new the event time for this event, so it is not deleted next time too early
event.resetEventTime();
// replace the current result offset
event.query.offset = query.offset;
event.query.itemsPerPage = query.itemsPerPage;
}
}
}
if (event == null) {

Write Preview
Loading…
Cancel
Save