boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

renovated the way how search results are count. should be correct now...

Browse Source
pull/1/head
Michael Peter Christen 13 years ago
parent 7bec253bb0
commit 8fb370d9f8
19 changed files with 228 additions and 311 deletions
Show Stats Download Patch File Download Diff File

6
htroot/AccessTracker_p.java
Unescape View File

@ -173,18 +173,18 @@ public class AccessTracker_p {
} }
prop.putNum("page_list_" + m + "_querycount", query.itemsPerPage); prop.putNum("page_list_" + m + "_querycount", query.itemsPerPage);
prop.putNum("page_list_" + m + "_transmitcount", query.transmitcount); prop.putNum("page_list_" + m + "_transmitcount", query.transmitcount);
prop.putNum("page_list_" + m + "_resultcount", query.resultcount); prop.putNum("page_list_" + m + "_resultcount", query.getResultCount());
prop.putNum("page_list_" + m + "_urltime", query.urlretrievaltime); prop.putNum("page_list_" + m + "_urltime", query.urlretrievaltime);
prop.putNum("page_list_" + m + "_snippettime", query.snippetcomputationtime); prop.putNum("page_list_" + m + "_snippettime", query.snippetcomputationtime);
prop.putNum("page_list_" + m + "_resulttime", query.searchtime); prop.putNum("page_list_" + m + "_resulttime", query.searchtime);
prop.putHTML("page_list_" + m + "_userAgent", query.userAgent); prop.putHTML("page_list_" + m + "_userAgent", query.userAgent);
qcountSum += query.itemsPerPage; qcountSum += query.itemsPerPage;
rcountSum += query.resultcount; rcountSum += query.getResultCount();
tcountSum += query.transmitcount; tcountSum += query.transmitcount;
utimeSum += query.urlretrievaltime; utimeSum += query.urlretrievaltime;
stimeSum += query.snippetcomputationtime; stimeSum += query.snippetcomputationtime;
rtimeSum += query.searchtime; rtimeSum += query.searchtime;
if (query.resultcount > 0){ if (query.getResultCount() > 0){
rcount++; rcount++;
utimeSum1 += query.urlretrievaltime; utimeSum1 += query.urlretrievaltime;
stimeSum1 += query.snippetcomputationtime; stimeSum1 += query.snippetcomputationtime;

5
htroot/HostBrowser.java
Unescape View File

@ -189,9 +189,12 @@ public class HostBrowser {
prop.putHTML("outbound_host", host); prop.putHTML("outbound_host", host);
prop.putHTML("inbound_host", host); prop.putHTML("inbound_host", host);
String hosthash = ASCII.String(uri.hash(), 6, 6); String hosthash = ASCII.String(uri.hash(), 6, 6);
String[] pathparts = uri.getPaths();
// get all files for a specific host from the index // get all files for a specific host from the index
BlockingQueue<SolrDocument> docs = fulltext.getSolr().concurrentQuery(YaCySchema.host_s.name() + ":" + host, 0, 100000, 3000, 100); String query = YaCySchema.host_s.name() + ":" + host;
for (String pe: pathparts) if (pe.length() > 0) query += " AND " + YaCySchema.url_paths_sxt.name() + ":" + pe;
BlockingQueue<SolrDocument> docs = fulltext.getSolr().concurrentQuery(query, 0, 100000, 3000, 100);
SolrDocument doc; SolrDocument doc;
Set<String> storedDocs = new HashSet<String>(); Set<String> storedDocs = new HashSet<String>();
Set<String> inboundLinks = new HashSet<String>(); Set<String> inboundLinks = new HashSet<String>();

56
htroot/IndexControlRWIs_p.java
Unescape View File

@ -126,7 +126,7 @@ public class IndexControlRWIs_p {
if ( post.containsKey("keystringsearch") ) { if ( post.containsKey("keystringsearch") ) {
prop.put("keyhash", keyhash); prop.put("keyhash", keyhash);
final SearchEvent theSearch = genSearchresult(prop, sb, segment, keyhash, null); final SearchEvent theSearch = genSearchresult(prop, sb, segment, keyhash, null);
if ( theSearch.rankingProcess.rwiAvailableCount() == 0 ) { if (theSearch.query.local_rwi_available.get() == 0) {
prop.put("searchresult", 1); prop.put("searchresult", 1);
prop.putHTML("searchresult_word", keystring); prop.putHTML("searchresult_word", keystring);
} }
@ -137,7 +137,7 @@ public class IndexControlRWIs_p {
prop.put("keystring", "&lt;" + errmsg + "&gt;"); prop.put("keystring", "&lt;" + errmsg + "&gt;");
} }
final SearchEvent theSearch = genSearchresult(prop, sb, segment, keyhash, null); final SearchEvent theSearch = genSearchresult(prop, sb, segment, keyhash, null);
if ( theSearch.rankingProcess.rwiAvailableCount() == 0 ) { if (theSearch.query.local_rwi_available.get() == 0) {
prop.put("searchresult", 2); prop.put("searchresult", 2);
prop.putHTML("searchresult_wordhash", ASCII.String(keyhash)); prop.putHTML("searchresult_wordhash", ASCII.String(keyhash));
} }
@ -466,7 +466,7 @@ public class IndexControlRWIs_p {
final String keyhashs = ASCII.String(keyhash); final String keyhashs = ASCII.String(keyhash);
prop.put("genUrlList_keyHash", keyhashs); prop.put("genUrlList_keyHash", keyhashs);
if ( theSearch.rankingProcess.rwiAvailableCount() == 0 ) { if (theSearch.query.local_rwi_stored.get() == 0) {
prop.put("genUrlList", 1); prop.put("genUrlList", 1);
prop.put("genUrlList_count", 0); prop.put("genUrlList_count", 0);
prop.put("searchresult", 2); prop.put("searchresult", 2);
@ -533,39 +533,17 @@ public class IndexControlRWIs_p {
prop prop
.put( .put(
"genUrlList_urlList_" + i + "_urlExists_props", "genUrlList_urlList_" + i + "_urlExists_props",
((entry.word().flags().get(Condenser.flag_cat_indexof)) ((entry.word().flags().get(Condenser.flag_cat_indexof)) ? "appears on index page, " : "")
? "appears on index page, " + ((entry.word().flags().get(Condenser.flag_cat_hasimage)) ? "contains images, " : "")
: "") + ((entry.word().flags().get(Condenser.flag_cat_hasaudio)) ? "contains audio, " : "")
+ ((entry.word().flags().get(Condenser.flag_cat_hasimage)) + ((entry.word().flags().get(Condenser.flag_cat_hasvideo)) ? "contains video, " : "")
? "contains images, " + ((entry.word().flags().get(Condenser.flag_cat_hasapp)) ? "contains applications, " : "")
: "") + ((entry.word().flags().get(WordReferenceRow.flag_app_dc_identifier)) ? "appears in url, " : "")
+ ((entry.word().flags().get(Condenser.flag_cat_hasaudio)) + ((entry.word().flags().get(WordReferenceRow.flag_app_dc_title)) ? "appears in title, " : "")
? "contains audio, " + ((entry.word().flags().get(WordReferenceRow.flag_app_dc_creator)) ? "appears in author, " : "")
: "") + ((entry.word().flags().get(WordReferenceRow.flag_app_dc_subject)) ? "appears in subject, " : "")
+ ((entry.word().flags().get(Condenser.flag_cat_hasvideo)) + ((entry.word().flags().get(WordReferenceRow.flag_app_dc_description)) ? "appears in description, " : "")
? "contains video, " + ((entry.word().flags().get(WordReferenceRow.flag_app_emphasized)) ? "appears emphasized, " : "")
: "")
+ ((entry.word().flags().get(Condenser.flag_cat_hasapp))
? "contains applications, "
: "")
+ ((entry.word().flags().get(WordReferenceRow.flag_app_dc_identifier))
? "appears in url, "
: "")
+ ((entry.word().flags().get(WordReferenceRow.flag_app_dc_title))
? "appears in title, "
: "")
+ ((entry.word().flags().get(WordReferenceRow.flag_app_dc_creator))
? "appears in author, "
: "")
+ ((entry.word().flags().get(WordReferenceRow.flag_app_dc_subject))
? "appears in subject, "
: "")
+ ((entry.word().flags().get(WordReferenceRow.flag_app_dc_description))
? "appears in description, "
: "")
+ ((entry.word().flags().get(WordReferenceRow.flag_app_emphasized))
? "appears emphasized, "
: "")
+ ((DigestURI.probablyRootURL(entry.word().urlhash())) ? "probably root url" : "")); + ((DigestURI.probablyRootURL(entry.word().urlhash())) ? "probably root url" : ""));
if ( Switchboard.urlBlacklist.isListed(BlacklistType.DHT, url) ) { if ( Switchboard.urlBlacklist.isListed(BlacklistType.DHT, url) ) {
prop.put("genUrlList_urlList_" + i + "_urlExists_urlhxChecked", "1"); prop.put("genUrlList_urlList_" + i + "_urlExists_urlhxChecked", "1");
@ -575,7 +553,7 @@ public class IndexControlRWIs_p {
break; break;
} }
} }
final Iterator<byte[]> iter = theSearch.rankingProcess.miss(); // iterates url hash strings final Iterator<byte[]> iter = theSearch.query.misses.iterator(); // iterates url hash strings
byte[] b; byte[] b;
while ( iter.hasNext() ) { while ( iter.hasNext() ) {
b = iter.next(); b = iter.next();
@ -677,12 +655,12 @@ public class IndexControlRWIs_p {
final SearchEvent theSearch = SearchEventCache.getEvent(query, sb.peers, sb.tables, null, false, sb.loader, Integer.MAX_VALUE, Long.MAX_VALUE, (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_ROBINSON, 0), (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_MULTIWORD, 0)); final SearchEvent theSearch = SearchEventCache.getEvent(query, sb.peers, sb.tables, null, false, sb.loader, Integer.MAX_VALUE, Long.MAX_VALUE, (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_ROBINSON, 0), (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_MULTIWORD, 0));
//theSearch.rankingProcess.run(); //theSearch.rankingProcess.run();
RankingProcess ranked = theSearch.rankingProcess; RankingProcess ranked = theSearch.rankingProcess;
if ( ranked.rwiAvailableCount() == 0 ) { if (theSearch.query.local_rwi_available.get() == 0) {
prop.put("searchresult", 2); prop.put("searchresult", 2);
prop.put("searchresult_wordhash", keyhash); prop.put("searchresult_wordhash", keyhash);
} else { } else {
prop.put("searchresult", 3); prop.put("searchresult", 3);
prop.put("searchresult_allurl", ranked.rwiAvailableCount()); prop.put("searchresult_allurl", theSearch.query.local_rwi_available.get());
prop prop
.put("searchresult_description", ranked.flagCount()[WordReferenceRow.flag_app_dc_description]); .put("searchresult_description", ranked.flagCount()[WordReferenceRow.flag_app_dc_description]);
prop.put("searchresult_title", ranked.flagCount()[WordReferenceRow.flag_app_dc_title]); prop.put("searchresult_title", ranked.flagCount()[WordReferenceRow.flag_app_dc_title]);

13
htroot/yacy/search.java
Unescape View File

@ -213,7 +213,6 @@ public final class search {
// prepare an abstract result // prepare an abstract result
final StringBuilder indexabstract = new StringBuilder(6000); final StringBuilder indexabstract = new StringBuilder(6000);
int indexabstractContainercount = 0; int indexabstractContainercount = 0;
int joincount = 0;
QueryParams theQuery = null; QueryParams theQuery = null;
SearchEvent theSearch = null; SearchEvent theSearch = null;
ArrayList<WeakPriorityBlockingQueue.Element<ResultEntry>> accu = null; ArrayList<WeakPriorityBlockingQueue.Element<ResultEntry>> accu = null;
@ -324,12 +323,11 @@ public final class search {
theSearch = SearchEventCache.getEvent(theQuery, sb.peers, sb.tables, null, abstracts.length() > 0, sb.loader, count, maxtime, (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_ROBINSON, 0), (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_MULTIWORD, 0)); theSearch = SearchEventCache.getEvent(theQuery, sb.peers, sb.tables, null, abstracts.length() > 0, sb.loader, count, maxtime, (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_ROBINSON, 0), (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_MULTIWORD, 0));
// set statistic details of search result and find best result index set // set statistic details of search result and find best result index set
joincount = theSearch.rankingProcess.rwiAvailableCount() - theSearch.rankingProcess.getMissCount() - theSearch.getSortOutCount(); prop.put("joincount", Integer.toString(theQuery.getResultCount()));
prop.put("joincount", Integer.toString(joincount)); if (theQuery.getResultCount() > 0) {
if (joincount != 0) {
accu = theSearch.completeResults(maxtime); accu = theSearch.completeResults(maxtime);
} }
if (joincount <= 0 || abstracts.isEmpty()) { if (theQuery.getResultCount() <= 0 || abstracts.isEmpty()) {
prop.put("indexcount", ""); prop.put("indexcount", "");
} else { } else {
// attach information about index abstracts // attach information about index abstracts
@ -391,7 +389,7 @@ public final class search {
prop.put("indexabstract", indexabstract.toString()); prop.put("indexabstract", indexabstract.toString());
// prepare result // prepare result
if (joincount == 0 || accu == null || accu.isEmpty()) { if (theQuery.getResultCount() == 0 || accu == null || accu.isEmpty()) {
// no results // no results
prop.put("links", ""); prop.put("links", "");
@ -419,7 +417,6 @@ public final class search {
// prepare search statistics // prepare search statistics
theQuery.remotepeer = client == null ? null : sb.peers.lookupByIP(Domains.dnsResolve(client), -1, true, false, false); theQuery.remotepeer = client == null ? null : sb.peers.lookupByIP(Domains.dnsResolve(client), -1, true, false, false);
theQuery.resultcount = (theSearch == null) ? 0 : joincount;
theQuery.searchtime = System.currentTimeMillis() - timestamp; theQuery.searchtime = System.currentTimeMillis() - timestamp;
theQuery.urlretrievaltime = (theSearch == null) ? 0 : theSearch.getURLRetrievalTime(); theQuery.urlretrievaltime = (theSearch == null) ? 0 : theSearch.getURLRetrievalTime();
theQuery.snippetcomputationtime = (theSearch == null) ? 0 : theSearch.getSnippetComputationTime(); theQuery.snippetcomputationtime = (theSearch == null) ? 0 : theSearch.getSnippetComputationTime();
@ -436,7 +433,7 @@ public final class search {
// log // log
Network.log.logInfo("EXIT HASH SEARCH: " + Network.log.logInfo("EXIT HASH SEARCH: " +
QueryParams.anonymizedQueryHashes(theQuery.query_include_hashes) + " - " + joincount + " links found, " + QueryParams.anonymizedQueryHashes(theQuery.query_include_hashes) + " - " + theQuery.getResultCount() + " links found, " +
prop.get("linkcount", "?") + " links selected, " + prop.get("linkcount", "?") + " links selected, " +
indexabstractContainercount + " index abstracts, " + indexabstractContainercount + " index abstracts, " +
(System.currentTimeMillis() - timestamp) + " milliseconds"); (System.currentTimeMillis() - timestamp) + " milliseconds");

68
htroot/yacysearch.java
Unescape View File

@ -831,27 +831,18 @@ public class yacysearch {
Log.logInfo("LOCAL_SEARCH", "EXIT WORD SEARCH: " Log.logInfo("LOCAL_SEARCH", "EXIT WORD SEARCH: "
+ theQuery.queryString + theQuery.queryString
+ " - " + " - "
+ "local-unfiltered(" + "local_rwi_available(" + theSearch.query.local_rwi_available.get() + "), "
+ theSearch.rankingProcess.rwiAvailableCount() + "local_rwi_stored(" + theSearch.query.local_rwi_stored.get() + "), "
+ "), " + "local_solr_available(" + theSearch.query.local_solr_available.get() + "), "
+ "local_miss(" + "local_solr_stored(" + theSearch.query.local_solr_stored.get() + "), "
+ theSearch.rankingProcess.getMissCount() + "remote_available(" + theSearch.query.remote_available.get() + "), "
+ "), " + "remote_stored(" + theSearch.query.remote_stored.get() + "), "
+ "local_sortout(" + "remote_peerCount(" + theSearch.query.remote_peerCount.get() + "), "
+ theSearch.getSortOutCount() + "local_sortout(" + theSearch.query.misses.size() + "), "
+ "), "
+ "remote("
+ theSearch.rankingProcess.getRemoteResourceSize()
+ ") links found, "
+ (System.currentTimeMillis() - timestamp) + (System.currentTimeMillis() - timestamp)
+ " ms"); + " ms");
// prepare search statistics // prepare search statistics
theQuery.resultcount =
theSearch.rankingProcess.rwiAvailableCount()
- theSearch.getSortOutCount()
- theSearch.rankingProcess.getMissCount()
+ theSearch.rankingProcess.getRemoteIndexCount();
theQuery.searchtime = System.currentTimeMillis() - timestamp; theQuery.searchtime = System.currentTimeMillis() - timestamp;
theQuery.urlretrievaltime = theSearch.getURLRetrievalTime(); theQuery.urlretrievaltime = theSearch.getURLRetrievalTime();
theQuery.snippetcomputationtime = theSearch.getSnippetComputationTime(); theQuery.snippetcomputationtime = theSearch.getSnippetComputationTime();
@ -932,37 +923,16 @@ public class yacysearch {
Log.logException(e); Log.logException(e);
} }
final int indexcount =
theSearch.rankingProcess.rwiAvailableCount()
- theSearch.getSortOutCount()
- theSearch.rankingProcess.getMissCount()
+ theSearch.rankingProcess.getRemoteIndexCount();
prop.put("num-results_offset", startRecord == 0 ? 0 : startRecord + 1); prop.put("num-results_offset", startRecord == 0 ? 0 : startRecord + 1);
prop.put("num-results_itemscount", Formatter.number( prop.put("num-results_itemscount", Formatter.number(startRecord + theSearch.query.itemsPerPage > theSearch.query.getResultCount() ? startRecord + theSearch.query.getResultCount() % theSearch.query.itemsPerPage : startRecord + theSearch.query.itemsPerPage, true));
startRecord + theSearch.getQuery().itemsPerPage > indexcount ? startRecord prop.put("num-results_itemsPerPage", Formatter.number(itemsPerPage));
+ indexcount prop.put("num-results_totalcount", Formatter.number(theSearch.query.getResultCount()));
% theSearch.getQuery().itemsPerPage : startRecord + theSearch.getQuery().itemsPerPage, prop.put("num-results_globalresults", global && (indexReceiveGranted || clustersearch) ? "1" : "0");
true)); prop.put("num-results_globalresults_localResourceSize", Formatter.number(theSearch.query.local_rwi_available.get() + theSearch.query.local_solr_available.get(), true));
prop.put("num-results_itemsPerPage", itemsPerPage); prop.put("num-results_globalresults_localMissCount", Formatter.number(theSearch.query.misses.size(), true));
prop.put("num-results_totalcount", indexcount); prop.put("num-results_globalresults_remoteResourceSize", Formatter.number(theSearch.query.remote_available.get(), true));
prop.put("num-results_globalresults", global && (indexReceiveGranted || clustersearch) prop.put("num-results_globalresults_remoteIndexCount", Formatter.number(theSearch.query.remote_stored.get(), true));
? "1" prop.put("num-results_globalresults_remotePeerCount", Formatter.number(theSearch.query.remote_peerCount.get(), true));
: "0");
prop.put(
"num-results_globalresults_localResourceSize",
Formatter.number(theSearch.rankingProcess.rwiAvailableCount(), true));
prop.put(
"num-results_globalresults_localMissCount",
Formatter.number(theSearch.rankingProcess.getMissCount(), true));
prop.put(
"num-results_globalresults_remoteResourceSize",
Formatter.number(theSearch.rankingProcess.getRemoteResourceSize(), true));
prop.put(
"num-results_globalresults_remoteIndexCount",
Formatter.number(theSearch.rankingProcess.getRemoteIndexCount(), true));
prop.put(
"num-results_globalresults_remotePeerCount",
Formatter.number(theSearch.rankingProcess.getRemotePeerCount(), true));
// compose page navigation // compose page navigation
final StringBuilder resnav = new StringBuilder(200); final StringBuilder resnav = new StringBuilder(200);
@ -982,7 +952,7 @@ public class yacysearch {
resnav resnav
.append("\"><img src=\"env/grafics/navdl.gif\" alt=\"arrowleft\" width=\"16\" height=\"16\" /></a>&nbsp;"); .append("\"><img src=\"env/grafics/navdl.gif\" alt=\"arrowleft\" width=\"16\" height=\"16\" /></a>&nbsp;");
} }
final int numberofpages = Math.min(10, 1 + ((indexcount - 1) / theQuery.itemsPerPage())); final int numberofpages = Math.min(10, 1 + ((theSearch.query.getResultCount() - 1) / theQuery.itemsPerPage()));
for ( int i = 0; i < numberofpages; i++ ) { for ( int i = 0; i < numberofpages; i++ ) {
if ( i == thispage ) { if ( i == thispage ) {
@ -1020,7 +990,7 @@ public class yacysearch {
} }
final String resnavs = resnav.toString(); final String resnavs = resnav.toString();
prop.put("num-results_resnav", resnavs); prop.put("num-results_resnav", resnavs);
prop.put("pageNavBottom", (indexcount - startRecord > 6) ? 1 : 0); // if there are more results than may fit on the page we add a navigation at the bottom prop.put("pageNavBottom", (theSearch.query.getResultCount() - startRecord > 6) ? 1 : 0); // if there are more results than may fit on the page we add a navigation at the bottom
prop.put("pageNavBottom_resnav", resnavs); prop.put("pageNavBottom_resnav", resnavs);
// generate the search result lines; the content will be produced by another servlet // generate the search result lines; the content will be produced by another servlet

73
htroot/yacysearchitem.java
Unescape View File

@ -89,28 +89,25 @@ public class yacysearchitem {
// the event does not exist, show empty page // the event does not exist, show empty page
return prop; return prop;
} }
final QueryParams theQuery = theSearch.getQuery();
// dynamically update count values // dynamically update count values
final int totalcount = theSearch.rankingProcess.rwiAvailableCount() - theSearch.rankingProcess.getMissCount() - theSearch.getSortOutCount() + theSearch.rankingProcess.getRemoteIndexCount(); prop.put("offset", theSearch.query.neededResults() - theSearch.query.itemsPerPage() + 1);
final int offset = theQuery.neededResults() - theQuery.itemsPerPage() + 1; prop.put("itemscount", Formatter.number(Math.min((item < 0) ? theSearch.query.neededResults() : item + 1, theSearch.query.getResultCount())));
prop.put("offset", offset); prop.put("itemsperpage", Formatter.number(theSearch.query.itemsPerPage));
prop.put("itemscount", Formatter.number(Math.min((item < 0) ? theQuery.neededResults() : item + 1, totalcount))); prop.put("totalcount", Formatter.number(theSearch.query.getResultCount(), true));
prop.put("itemsperpage", Formatter.number(theQuery.itemsPerPage)); prop.put("localResourceSize", Formatter.number(theSearch.query.local_rwi_available.get() + theSearch.query.local_solr_available.get(), true));
prop.put("totalcount", Formatter.number(totalcount, true)); prop.put("localMissCount", Formatter.number(theSearch.query.misses.size(), true));
prop.put("localResourceSize", Formatter.number(theSearch.rankingProcess.rwiAvailableCount(), true)); prop.put("remoteResourceSize", Formatter.number(theSearch.query.remote_available.get(), true));
prop.put("localMissCount", Formatter.number(theSearch.rankingProcess.getMissCount(), true)); prop.put("remoteIndexCount", Formatter.number(theSearch.query.remote_stored.get(), true));
prop.put("remoteResourceSize", Formatter.number(theSearch.rankingProcess.getRemoteResourceSize(), true)); prop.put("remotePeerCount", Formatter.number(theSearch.query.remote_peerCount.get(), true));
prop.put("remoteIndexCount", Formatter.number(theSearch.rankingProcess.getRemoteIndexCount(), true)); prop.put("navurlBase", QueryParams.navurlBase("html", theSearch.query, null, theSearch.query.urlMask.toString(), theSearch.query.navigators).toString());
prop.put("remotePeerCount", Formatter.number(theSearch.rankingProcess.getRemotePeerCount(), true));
prop.put("navurlBase", QueryParams.navurlBase("html", theQuery, null, theQuery.urlMask.toString(), theQuery.navigators).toString());
final String target_special_pattern = sb.getConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN, ""); final String target_special_pattern = sb.getConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN, "");
if (theQuery.contentdom == Classification.ContentDomain.TEXT || theQuery.contentdom == Classification.ContentDomain.ALL) { if (theSearch.query.contentdom == Classification.ContentDomain.TEXT || theSearch.query.contentdom == Classification.ContentDomain.ALL) {
// text search // text search
// generate result object // generate result object
final ResultEntry result = theSearch.oneResult(item, theQuery.isLocal() ? 1000 : 5000); final ResultEntry result = theSearch.oneResult(item, theSearch.query.isLocal() ? 1000 : 5000);
if (result == null) return prop; // no content if (result == null) return prop; // no content
final String resultUrlstring = result.urlstring(); final String resultUrlstring = result.urlstring();
final DigestURI resultURL = result.url(); final DigestURI resultURL = result.url();
@ -124,7 +121,7 @@ public class yacysearchitem {
Log.logException(e1); Log.logException(e1);
faviconURL = null; faviconURL = null;
} }
final String resource = theQuery.domType.toString(); final String resource = theSearch.query.domType.toString();
prop.put("content", 1); // switch on specific content prop.put("content", 1); // switch on specific content
prop.put("content_showDate", sb.getConfigBool("search.result.show.date", true) ? 1 : 0); prop.put("content_showDate", sb.getConfigBool("search.result.show.date", true) ? 1 : 0);
prop.put("content_showSize", sb.getConfigBool("search.result.show.size", true) ? 1 : 0); prop.put("content_showSize", sb.getConfigBool("search.result.show.size", true) ? 1 : 0);
@ -137,10 +134,10 @@ public class yacysearchitem {
prop.put("content_authorized", authenticated ? "1" : "0"); prop.put("content_authorized", authenticated ? "1" : "0");
final String urlhash = ASCII.String(result.hash()); final String urlhash = ASCII.String(result.hash());
prop.put("content_authorized_bookmark", sb.tables.bookmarks.hasBookmark("admin", urlhash) ? "0" : "1"); prop.put("content_authorized_bookmark", sb.tables.bookmarks.hasBookmark("admin", urlhash) ? "0" : "1");
prop.putHTML("content_authorized_bookmark_bookmarklink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.itemsPerPage() + "&offset=" + (theQuery.neededResults() - theQuery.itemsPerPage()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=" + resource + "&time=3&bookmarkref=" + urlhash + "&urlmaskfilter=.*"); prop.putHTML("content_authorized_bookmark_bookmarklink", "/yacysearch.html?query=" + theSearch.query.queryString.replace(' ', '+') + "&Enter=Search&count=" + theSearch.query.itemsPerPage() + "&offset=" + (theSearch.query.neededResults() - theSearch.query.itemsPerPage()) + "&order=" + crypt.simpleEncode(theSearch.query.ranking.toExternalString()) + "&resource=" + resource + "&time=3&bookmarkref=" + urlhash + "&urlmaskfilter=.*");
prop.put("content_authorized_recommend", (sb.peers.newsPool.getSpecific(NewsPool.OUTGOING_DB, NewsPool.CATEGORY_SURFTIPP_ADD, "url", resultUrlstring) == null) ? "1" : "0"); prop.put("content_authorized_recommend", (sb.peers.newsPool.getSpecific(NewsPool.OUTGOING_DB, NewsPool.CATEGORY_SURFTIPP_ADD, "url", resultUrlstring) == null) ? "1" : "0");
prop.putHTML("content_authorized_recommend_deletelink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.itemsPerPage() + "&offset=" + (theQuery.neededResults() - theQuery.itemsPerPage()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=" + resource + "&time=3&deleteref=" + urlhash + "&urlmaskfilter=.*"); prop.putHTML("content_authorized_recommend_deletelink", "/yacysearch.html?query=" + theSearch.query.queryString.replace(' ', '+') + "&Enter=Search&count=" + theSearch.query.itemsPerPage() + "&offset=" + (theSearch.query.neededResults() - theSearch.query.itemsPerPage()) + "&order=" + crypt.simpleEncode(theSearch.query.ranking.toExternalString()) + "&resource=" + resource + "&time=3&deleteref=" + urlhash + "&urlmaskfilter=.*");
prop.putHTML("content_authorized_recommend_recommendlink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.itemsPerPage() + "&offset=" + (theQuery.neededResults() - theQuery.itemsPerPage()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=" + resource + "&time=3&recommendref=" + urlhash + "&urlmaskfilter=.*"); prop.putHTML("content_authorized_recommend_recommendlink", "/yacysearch.html?query=" + theSearch.query.queryString.replace(' ', '+') + "&Enter=Search&count=" + theSearch.query.itemsPerPage() + "&offset=" + (theSearch.query.neededResults() - theSearch.query.itemsPerPage()) + "&order=" + crypt.simpleEncode(theSearch.query.ranking.toExternalString()) + "&resource=" + resource + "&time=3&recommendref=" + urlhash + "&urlmaskfilter=.*");
prop.put("content_authorized_urlhash", urlhash); prop.put("content_authorized_urlhash", urlhash);
final String resulthashString = urlhash; final String resulthashString = urlhash;
prop.putHTML("content_title", result.title()); prop.putHTML("content_title", result.title());
@ -205,11 +202,11 @@ public class yacysearchitem {
prop.putHTML("content_host", resultURL.getHost() == null ? "" : resultURL.getHost()); prop.putHTML("content_host", resultURL.getHost() == null ? "" : resultURL.getHost());
prop.putHTML("content_file", resultURL.getFileName()); prop.putHTML("content_file", resultURL.getFileName());
prop.putHTML("content_path", resultURL.getPath()); prop.putHTML("content_path", resultURL.getPath());
prop.put("content_nl", (item == theQuery.offset) ? 0 : 1); prop.put("content_nl", (item == theSearch.query.offset) ? 0 : 1);
prop.putHTML("content_publisher", result.publisher()); prop.putHTML("content_publisher", result.publisher());
prop.putHTML("content_creator", result.creator());// author prop.putHTML("content_creator", result.creator());// author
prop.putHTML("content_subject", result.subject()); prop.putHTML("content_subject", result.subject());
final Collection<String>[] query = theQuery.queryWords(); final Collection<String>[] query = theSearch.query.queryWords();
final StringBuilder s = new StringBuilder(query[0].size() * 20); final StringBuilder s = new StringBuilder(query[0].size() * 20);
for (final String t: query[0]) { for (final String t: query[0]) {
s.append('+').append(t); s.append('+').append(t);
@ -217,10 +214,10 @@ public class yacysearchitem {
final String words = (s.length() > 0) ? s.substring(1) : ""; final String words = (s.length() > 0) ? s.substring(1) : "";
prop.putHTML("content_words", words); prop.putHTML("content_words", words);
prop.putHTML("content_showParser_words", words); prop.putHTML("content_showParser_words", words);
prop.putHTML("content_former", theQuery.queryString); prop.putHTML("content_former", theSearch.query.queryString);
prop.putHTML("content_showPictures_former", theQuery.queryString); prop.putHTML("content_showPictures_former", theSearch.query.queryString);
final TextSnippet snippet = result.textSnippet(); final TextSnippet snippet = result.textSnippet();
final String desc = (snippet == null) ? "" : snippet.getLineMarked(theQuery.query_all_hashes); final String desc = (snippet == null) ? "" : snippet.getLineMarked(theSearch.query.query_all_hashes);
prop.put("content_description", desc); prop.put("content_description", desc);
prop.putXML("content_description-xml", desc); prop.putXML("content_description-xml", desc);
prop.putJSON("content_description-json", desc); prop.putJSON("content_description-json", desc);
@ -235,7 +232,7 @@ public class yacysearchitem {
} }
prop.put("content_heuristic_name", heuristic.heuristicName); prop.put("content_heuristic_name", heuristic.heuristicName);
} }
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEventType.FINALIZATION, "" + item, 0, 0), false); EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theSearch.query.id(true), SearchEventType.FINALIZATION, "" + item, 0, 0), false);
final String ext = resultURL.getFileExtension().toLowerCase(); final String ext = resultURL.getFileExtension().toLowerCase();
if (ext.equals("png") || ext.equals("jpg") || ext.equals("gif")) { if (ext.equals("png") || ext.equals("jpg") || ext.equals("gif")) {
final String license = sb.licensedURLs.aquireLicense(resultURL); final String license = sb.licensedURLs.aquireLicense(resultURL);
@ -251,16 +248,16 @@ public class yacysearchitem {
prop.put("content_loc_lon", result.lon()); prop.put("content_loc_lon", result.lon());
} }
if (sb.getConfigBool("heuristic.searchresults",false)) sb.heuristicSearchResults(resultUrlstring); if (sb.getConfigBool("heuristic.searchresults",false)) sb.heuristicSearchResults(resultUrlstring);
theQuery.transmitcount = item + 1; theSearch.query.transmitcount = item + 1;
return prop; return prop;
} }
if (theQuery.contentdom == Classification.ContentDomain.IMAGE) { if (theSearch.query.contentdom == Classification.ContentDomain.IMAGE) {
// image search; shows thumbnails // image search; shows thumbnails
prop.put("content", theQuery.contentdom.getCode() + 1); // switch on specific content prop.put("content", theSearch.query.contentdom.getCode() + 1); // switch on specific content
//final MediaSnippet ms = theSearch.result().oneImage(item); //final MediaSnippet ms = theSearch.result().oneImage(item);
final ResultEntry ms = theSearch.oneResult(item, theQuery.isLocal() ? 1000 : 5000); final ResultEntry ms = theSearch.oneResult(item, theSearch.query.isLocal() ? 1000 : 5000);
if (ms == null) { if (ms == null) {
prop.put("content_item", "0"); prop.put("content_item", "0");
} else { } else {
@ -283,21 +280,21 @@ public class yacysearchitem {
prop.put("content_item_source", ms.url().toNormalform(true)); prop.put("content_item_source", ms.url().toNormalform(true));
prop.putXML("content_item_source-xml", ms.url().toNormalform(true)); prop.putXML("content_item_source-xml", ms.url().toNormalform(true));
prop.put("content_item_sourcedom", ms.url().getHost()); prop.put("content_item_sourcedom", ms.url().getHost());
prop.put("content_item_nl", (item == theQuery.offset) ? 0 : 1); prop.put("content_item_nl", (item == theSearch.query.offset) ? 0 : 1);
prop.put("content_item", 1); prop.put("content_item", 1);
} }
theQuery.transmitcount = item + 1; theSearch.query.transmitcount = item + 1;
return prop; return prop;
} }
if ((theQuery.contentdom == ContentDomain.AUDIO) || if ((theSearch.query.contentdom == ContentDomain.AUDIO) ||
(theQuery.contentdom == ContentDomain.VIDEO) || (theSearch.query.contentdom == ContentDomain.VIDEO) ||
(theQuery.contentdom == ContentDomain.APP)) { (theSearch.query.contentdom == ContentDomain.APP)) {
// any other media content // any other media content
// generate result object // generate result object
final ResultEntry ms = theSearch.oneResult(item, theQuery.isLocal() ? 1000 : 5000); final ResultEntry ms = theSearch.oneResult(item, theSearch.query.isLocal() ? 1000 : 5000);
prop.put("content", theQuery.contentdom.getCode() + 1); // switch on specific content prop.put("content", theSearch.query.contentdom.getCode() + 1); // switch on specific content
if (ms == null) { if (ms == null) {
prop.put("content_item", "0"); prop.put("content_item", "0");
} else { } else {
@ -308,10 +305,10 @@ public class yacysearchitem {
prop.putHTML("content_item_target", target); prop.putHTML("content_item_target", target);
prop.putHTML("content_item_name", shorten(ms.title(), MAX_NAME_LENGTH)); prop.putHTML("content_item_name", shorten(ms.title(), MAX_NAME_LENGTH));
prop.put("content_item_col", (item % 2 == 0) ? "0" : "1"); prop.put("content_item_col", (item % 2 == 0) ? "0" : "1");
prop.put("content_item_nl", (item == theQuery.offset) ? 0 : 1); prop.put("content_item_nl", (item == theSearch.query.offset) ? 0 : 1);
prop.put("content_item", 1); prop.put("content_item", 1);
} }
theQuery.transmitcount = item + 1; theSearch.query.transmitcount = item + 1;
return prop; return prop;
} }

22
htroot/yacysearchlatestinfo.java
Unescape View File

@ -30,21 +30,19 @@ public class yacysearchlatestinfo {
prop.put("remotePeerCount", 0); prop.put("remotePeerCount", 0);
return prop; return prop;
} }
final QueryParams theQuery = theSearch.getQuery();
// dynamically update count values // dynamically update count values
final int totalcount = theSearch.rankingProcess.rwiAvailableCount() - theSearch.rankingProcess.getMissCount() - theSearch.getSortOutCount() + theSearch.rankingProcess.getRemoteIndexCount(); final int offset = theSearch.query.neededResults() - theSearch.query.itemsPerPage() + 1;
final int offset = theQuery.neededResults() - theQuery.itemsPerPage() + 1;
prop.put("offset", offset); prop.put("offset", offset);
prop.put("itemscount",Formatter.number(offset + theSearch.getQuery().itemsPerPage >= totalcount ? offset + totalcount % theSearch.getQuery().itemsPerPage - 1 : offset + theSearch.getQuery().itemsPerPage - 1)); prop.put("itemscount",Formatter.number(offset + theSearch.query.itemsPerPage >= theSearch.query.getResultCount() ? offset + theSearch.query.getResultCount() % theSearch.query.itemsPerPage - 1 : offset + theSearch.query.itemsPerPage - 1));
prop.put("itemsperpage", theSearch.getQuery().itemsPerPage); prop.put("itemsperpage", theSearch.query.itemsPerPage);
prop.put("totalcount", Formatter.number(totalcount, true)); prop.put("totalcount", Formatter.number(theSearch.query.getResultCount(), true));
prop.put("localResourceSize", Formatter.number(theSearch.rankingProcess.rwiAvailableCount(), true)); prop.put("localResourceSize", Formatter.number(theSearch.query.local_rwi_available.get() + theSearch.query.local_solr_available.get(), true));
prop.put("localMissCount", Formatter.number(theSearch.rankingProcess.getMissCount(), true)); prop.put("localMissCount", Formatter.number(theSearch.query.misses.size(), true));
prop.put("remoteResourceSize", Formatter.number(theSearch.rankingProcess.getRemoteResourceSize(), true)); prop.put("remoteResourceSize", Formatter.number(theSearch.query.remote_available.get(), true));
prop.put("remoteIndexCount", Formatter.number(theSearch.rankingProcess.getRemoteIndexCount(), true)); prop.put("remoteIndexCount", Formatter.number(theSearch.query.remote_stored.get(), true));
prop.put("remotePeerCount", Formatter.number(theSearch.rankingProcess.getRemotePeerCount(), true)); prop.put("remotePeerCount", Formatter.number(theSearch.query.remote_peerCount.get(), true));
prop.putJSON("navurlBase", QueryParams.navurlBase("html", theQuery, null, theQuery.urlMask.toString(), theQuery.navigators).toString()); prop.putJSON("navurlBase", QueryParams.navurlBase("html", theSearch.query, null, theSearch.query.urlMask.toString(), theSearch.query.navigators).toString());
return prop; return prop;
} }

49
htroot/yacysearchtrailer.java
Unescape View File

@ -59,7 +59,6 @@ public class yacysearchtrailer {
// the event does not exist, show empty page // the event does not exist, show empty page
return prop; return prop;
} }
final QueryParams theQuery = theSearch.getQuery();
final RequestHeader.FileType fileType = header.fileType(); final RequestHeader.FileType fileType = header.fileType();
// compose search navigation // compose search navigation
@ -80,7 +79,7 @@ public class yacysearchtrailer {
name = navigatorIterator.next(); name = navigatorIterator.next();
count = namespaceNavigator.get(name); count = namespaceNavigator.get(name);
nav = "inurl%3A" + name; nav = "inurl%3A" + name;
queryStringForUrl = theQuery.queryStringForUrl(); queryStringForUrl = theSearch.query.queryStringForUrl();
p = queryStringForUrl.indexOf(nav); p = queryStringForUrl.indexOf(nav);
if (p < 0) { if (p < 0) {
pos++; pos++;
@ -94,7 +93,7 @@ public class yacysearchtrailer {
queryStringForUrl = (queryStringForUrl.substring(0, p) + queryStringForUrl.substring(p + nav.length())).trim(); queryStringForUrl = (queryStringForUrl.substring(0, p) + queryStringForUrl.substring(p + nav.length())).trim();
} }
prop.put(fileType, "nav-namespace_element_" + i + "_name", name); prop.put(fileType, "nav-namespace_element_" + i + "_name", name);
prop.put(fileType, "nav-namespace_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theQuery, queryStringForUrl, theQuery.urlMask.toString(), theQuery.navigators).toString()); prop.put(fileType, "nav-namespace_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl, theSearch.query.urlMask.toString(), theSearch.query.navigators).toString());
prop.put("nav-namespace_element_" + i + "_count", count); prop.put("nav-namespace_element_" + i + "_count", count);
prop.put("nav-namespace_element_" + i + "_nl", 1); prop.put("nav-namespace_element_" + i + "_nl", 1);
i++; i++;
@ -119,7 +118,7 @@ public class yacysearchtrailer {
name = navigatorIterator.next(); name = navigatorIterator.next();
count = hostNavigator.get(name); count = hostNavigator.get(name);
nav = "site%3A" + name; nav = "site%3A" + name;
queryStringForUrl = theQuery.queryStringForUrl(); queryStringForUrl = theSearch.query.queryStringForUrl();
p = queryStringForUrl.indexOf(nav); p = queryStringForUrl.indexOf(nav);
if (p < 0) { if (p < 0) {
pos++; pos++;
@ -133,7 +132,7 @@ public class yacysearchtrailer {
prop.put(fileType, "nav-authors_element_" + i + "_modifier", "-" + nav); prop.put(fileType, "nav-authors_element_" + i + "_modifier", "-" + nav);
} }
prop.put(fileType, "nav-domains_element_" + i + "_name", name); prop.put(fileType, "nav-domains_element_" + i + "_name", name);
prop.put(fileType, "nav-domains_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theQuery, queryStringForUrl, theQuery.urlMask.toString(), theQuery.navigators).toString()); prop.put(fileType, "nav-domains_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl, theSearch.query.urlMask.toString(), theSearch.query.navigators).toString());
prop.put("nav-domains_element_" + i + "_count", count); prop.put("nav-domains_element_" + i + "_count", count);
prop.put("nav-domains_element_" + i + "_nl", 1); prop.put("nav-domains_element_" + i + "_nl", 1);
i++; i++;
@ -158,7 +157,7 @@ public class yacysearchtrailer {
name = navigatorIterator.next().trim(); name = navigatorIterator.next().trim();
count = authorNavigator.get(name); count = authorNavigator.get(name);
nav = (name.indexOf(' ', 0) < 0) ? "author%3A" + name : "author%3A%28" + name.replace(" ", "+") + "%29"; nav = (name.indexOf(' ', 0) < 0) ? "author%3A" + name : "author%3A%28" + name.replace(" ", "+") + "%29";
queryStringForUrl = theQuery.queryStringForUrl(); queryStringForUrl = theSearch.query.queryStringForUrl();
p = queryStringForUrl.indexOf(nav); p = queryStringForUrl.indexOf(nav);
if (p < 0) { if (p < 0) {
pos++; pos++;
@ -172,7 +171,7 @@ public class yacysearchtrailer {
prop.put(fileType, "nav-authors_element_" + i + "_modifier", "-" + nav); prop.put(fileType, "nav-authors_element_" + i + "_modifier", "-" + nav);
} }
prop.put(fileType, "nav-authors_element_" + i + "_name", name); prop.put(fileType, "nav-authors_element_" + i + "_name", name);
prop.put(fileType, "nav-authors_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theQuery, queryStringForUrl, theQuery.urlMask.toString(), theQuery.navigators).toString()); prop.put(fileType, "nav-authors_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl, theSearch.query.urlMask.toString(), theSearch.query.navigators).toString());
prop.put("nav-authors_element_" + i + "_count", count); prop.put("nav-authors_element_" + i + "_count", count);
prop.put("nav-authors_element_" + i + "_nl", 1); prop.put("nav-authors_element_" + i + "_nl", 1);
i++; i++;
@ -196,13 +195,13 @@ public class yacysearchtrailer {
while (i < MAX_TOPWORDS && navigatorIterator.hasNext()) { while (i < MAX_TOPWORDS && navigatorIterator.hasNext()) {
name = navigatorIterator.next(); name = navigatorIterator.next();
count = topicNavigator.get(name); count = topicNavigator.get(name);
if (theQuery.queryString == null) break; if (theSearch.query.queryString == null) break;
if (name != null) { if (name != null) {
queryStringForUrl = theQuery.queryStringForUrl(); queryStringForUrl = theSearch.query.queryStringForUrl();
prop.put("nav-topics_element_" + i + "_on", 1); prop.put("nav-topics_element_" + i + "_on", 1);
prop.put(fileType, "nav-topics_element_" + i + "_modifier", name); prop.put(fileType, "nav-topics_element_" + i + "_modifier", name);
prop.put(fileType, "nav-topics_element_" + i + "_name", name); prop.put(fileType, "nav-topics_element_" + i + "_name", name);
prop.put(fileType, "nav-topics_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theQuery, queryStringForUrl + "+" + name, theQuery.urlMask.toString(), theQuery.navigators).toString()); prop.put(fileType, "nav-topics_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl + "+" + name, theSearch.query.urlMask.toString(), theSearch.query.navigators).toString());
prop.put("nav-topics_element_" + i + "_count", count); prop.put("nav-topics_element_" + i + "_count", count);
prop.put("nav-topics_element_" + i + "_nl", 1); prop.put("nav-topics_element_" + i + "_nl", 1);
i++; i++;
@ -226,7 +225,7 @@ public class yacysearchtrailer {
name = navigatorIterator.next().trim(); name = navigatorIterator.next().trim();
count = protocolNavigator.get(name); count = protocolNavigator.get(name);
nav = "%2F" + name; nav = "%2F" + name;
queryStringForUrl = theQuery.queryStringForUrl(); queryStringForUrl = theSearch.query.queryStringForUrl();
p = queryStringForUrl.indexOf(nav); p = queryStringForUrl.indexOf(nav);
if (p < 0) { if (p < 0) {
pos++; pos++;
@ -240,7 +239,7 @@ public class yacysearchtrailer {
prop.put(fileType, "nav-protocols_element_" + i + "_modifier", "-" + nav); prop.put(fileType, "nav-protocols_element_" + i + "_modifier", "-" + nav);
} }
prop.put(fileType, "nav-protocols_element_" + i + "_name", name); prop.put(fileType, "nav-protocols_element_" + i + "_name", name);
prop.put(fileType, "nav-protocols_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theQuery, queryStringForUrl, (p >= 0 && theQuery.urlMask.toString().startsWith(name)) ? ".*" : theQuery.urlMask.toString(), theQuery.navigators).toString()); prop.put(fileType, "nav-protocols_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl, (p >= 0 && theSearch.query.urlMask.toString().startsWith(name)) ? ".*" : theSearch.query.urlMask.toString(), theSearch.query.navigators).toString());
prop.put("nav-protocols_element_" + i + "_count", count); prop.put("nav-protocols_element_" + i + "_count", count);
prop.put("nav-protocols_element_" + i + "_nl", 1); prop.put("nav-protocols_element_" + i + "_nl", 1);
i++; i++;
@ -265,7 +264,7 @@ public class yacysearchtrailer {
name = navigatorIterator.next().trim(); name = navigatorIterator.next().trim();
count = filetypeNavigator.get(name); count = filetypeNavigator.get(name);
nav = "filetype%3A" + name; nav = "filetype%3A" + name;
queryStringForUrl = theQuery.queryStringForUrl(); queryStringForUrl = theSearch.query.queryStringForUrl();
p = queryStringForUrl.indexOf(nav); p = queryStringForUrl.indexOf(nav);
if (p < 0) { if (p < 0) {
pos++; pos++;
@ -279,7 +278,7 @@ public class yacysearchtrailer {
prop.put(fileType, "nav-filetypes_element_" + i + "_modifier", "-" + nav); prop.put(fileType, "nav-filetypes_element_" + i + "_modifier", "-" + nav);
} }
prop.put(fileType, "nav-filetypes_element_" + i + "_name", name); prop.put(fileType, "nav-filetypes_element_" + i + "_name", name);
prop.put(fileType, "nav-filetypes_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theQuery, queryStringForUrl, (p >= 0 && theQuery.urlMask.toString().endsWith(name)) ? ".*" : theQuery.urlMask.toString(), theQuery.navigators).toString()); prop.put(fileType, "nav-filetypes_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl, (p >= 0 && theSearch.query.urlMask.toString().endsWith(name)) ? ".*" : theSearch.query.urlMask.toString(), theSearch.query.navigators).toString());
prop.put("nav-filetypes_element_" + i + "_count", count); prop.put("nav-filetypes_element_" + i + "_count", count);
prop.put("nav-filetypes_element_" + i + "_nl", 1); prop.put("nav-filetypes_element_" + i + "_nl", 1);
i++; i++;
@ -308,7 +307,7 @@ public class yacysearchtrailer {
name = navigatorIterator.next(); name = navigatorIterator.next();
count = ve.getValue().get(name); count = ve.getValue().get(name);
nav = "%2Fvocabulary%2F" + navname + "%2F" + MultiProtocolURI.escape(Tagging.encodePrintname(name)).toString(); nav = "%2Fvocabulary%2F" + navname + "%2F" + MultiProtocolURI.escape(Tagging.encodePrintname(name)).toString();
queryStringForUrl = theQuery.queryStringForUrl(); queryStringForUrl = theSearch.query.queryStringForUrl();
p = queryStringForUrl.indexOf(nav); p = queryStringForUrl.indexOf(nav);
if (p < 0) { if (p < 0) {
queryStringForUrl += "+" + nav; queryStringForUrl += "+" + nav;
@ -320,7 +319,7 @@ public class yacysearchtrailer {
prop.put(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_modifier", "-" + nav); prop.put(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_modifier", "-" + nav);
} }
prop.put(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_name", name); prop.put(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_name", name);
prop.put(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theQuery, queryStringForUrl, theQuery.urlMask.toString(), theQuery.navigators).toString()); prop.put(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl, theSearch.query.urlMask.toString(), theSearch.query.navigators).toString());
prop.put("nav-vocabulary_" + navvoccount + "_element_" + i + "_count", count); prop.put("nav-vocabulary_" + navvoccount + "_element_" + i + "_count", count);
prop.put("nav-vocabulary_" + navvoccount + "_element_" + i + "_nl", 1); prop.put("nav-vocabulary_" + navvoccount + "_element_" + i + "_nl", 1);
i++; i++;
@ -338,8 +337,7 @@ public class yacysearchtrailer {
// about box // about box
final String aboutBody = env.getConfig("about.body", ""); final String aboutBody = env.getConfig("about.body", "");
final String aboutHeadline = env.getConfig("about.headline", ""); final String aboutHeadline = env.getConfig("about.headline", "");
if ((aboutBody.isEmpty() && aboutHeadline.isEmpty()) || if ((aboutBody.isEmpty() && aboutHeadline.isEmpty()) || theSearch.query.getResultCount() == 0) {
theSearch.rankingProcess.rwiAvailableCount() - theSearch.rankingProcess.getMissCount() - theSearch.getSortOutCount() + theSearch.rankingProcess.getRemoteIndexCount() == 0) {
prop.put("nav-about", 0); prop.put("nav-about", 0);
} else { } else {
prop.put("nav-about", 1); prop.put("nav-about", 1);
@ -349,20 +347,15 @@ public class yacysearchtrailer {
// category: location search // category: location search
// show only if there is a location database present and if there had been any search results // show only if there is a location database present and if there had been any search results
if (LibraryProvider.geoLoc.isEmpty() || if (LibraryProvider.geoLoc.isEmpty() || theSearch.query.getResultCount() == 0) {
theSearch.rankingProcess.rwiAvailableCount() == 0) {
prop.put("cat-location", 0); prop.put("cat-location", 0);
} else { } else {
prop.put("cat-location", 1); prop.put("cat-location", 1);
prop.put(fileType, "cat-location_query", theQuery.queryString(true)); prop.put(fileType, "cat-location_query", theSearch.query.queryString(true));
prop.put(fileType, "cat-location_queryenc", theQuery.queryString(true).replace(' ', '+')); prop.put(fileType, "cat-location_queryenc", theSearch.query.queryString(true).replace(' ', '+'));
} }
prop.put("num-results_totalcount", theSearch.query.getResultCount());
final int indexcount = theSearch.rankingProcess.rwiAvailableCount() - theSearch.rankingProcess.getMissCount() - theSearch.getSortOutCount() + theSearch.rankingProcess.getRemoteIndexCount(); EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theSearch.query.id(true), SearchEventType.FINALIZATION, "bottomline", 0, 0), false);
prop.put("num-results_totalcount", indexcount);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEventType.FINALIZATION, "bottomline", 0, 0), false);
return prop; return prop;
} }

2
source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java
Unescape View File

@ -132,7 +132,9 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
@Override @Override
public synchronized void close() { public synchronized void close() {
this.commit();
super.close(); super.close();
this.defaultCore.close();
this.cores.shutdown(); this.cores.shutdown();
} }

37
source/net/yacy/peers/Protocol.java
Unescape View File

@ -117,6 +117,10 @@ import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.response.ResultContext;
import org.apache.solr.search.DocList;
public final class Protocol public final class Protocol
@ -590,7 +594,7 @@ public final class Protocol
null); null);
} }
public static int primarySearch( protected static int primarySearch(
final SearchEvent event, final SearchEvent event,
final String wordhashes, final String wordhashes,
final String excludehashes, final String excludehashes,
@ -689,7 +693,7 @@ public final class Protocol
return result.urlcount; return result.urlcount;
} }
public static int secondarySearch( protected static int secondarySearch(
final SearchEvent event, final SearchEvent event,
final String wordhashes, final String wordhashes,
final String urlhashes, final String urlhashes,
@ -810,7 +814,7 @@ public final class Protocol
// passed all checks, store url // passed all checks, store url
try { try {
event.getQuery().getSegment().fulltext().putMetadata(urlEntry); event.query.getSegment().fulltext().putMetadata(urlEntry);
ResultURLs.stack( ResultURLs.stack(
ASCII.String(urlEntry.url().hash()), ASCII.String(urlEntry.url().hash()),
urlEntry.url().getHost(), urlEntry.url().getHost(),
@ -853,7 +857,7 @@ public final class Protocol
// insert the containers to the index // insert the containers to the index
for ( final ReferenceContainer<WordReference> c : container ) { for ( final ReferenceContainer<WordReference> c : container ) {
try { try {
event.getQuery().getSegment().storeRWI(c); event.query.getSegment().storeRWI(c);
} catch ( final Exception e ) { } catch ( final Exception e ) {
Log.logException(e); Log.logException(e);
} }
@ -938,8 +942,8 @@ public final class Protocol
parts.put("exclude", UTF8.StringBody(excludehashes)); parts.put("exclude", UTF8.StringBody(excludehashes));
parts.put("duetime", UTF8.StringBody("1000")); parts.put("duetime", UTF8.StringBody("1000"));
parts.put("urls", UTF8.StringBody(urlhashes)); parts.put("urls", UTF8.StringBody(urlhashes));
parts.put("prefer", UTF8.StringBody(event.getQuery().prefer.pattern())); parts.put("prefer", UTF8.StringBody(event.query.prefer.pattern()));
parts.put("filter", UTF8.StringBody(event.getQuery().urlMask.pattern())); parts.put("filter", UTF8.StringBody(event.query.urlMask.pattern()));
parts.put("modifier", UTF8.StringBody(modifier)); parts.put("modifier", UTF8.StringBody(modifier));
parts.put("language", UTF8.StringBody(language)); parts.put("language", UTF8.StringBody(language));
parts.put("sitehash", UTF8.StringBody(sitehash)); parts.put("sitehash", UTF8.StringBody(sitehash));
@ -947,8 +951,8 @@ public final class Protocol
parts.put("contentdom", UTF8.StringBody(contentdom)); parts.put("contentdom", UTF8.StringBody(contentdom));
parts.put("ttl", UTF8.StringBody("0")); parts.put("ttl", UTF8.StringBody("0"));
parts.put("maxdist", UTF8.StringBody(Integer.toString(maxDistance))); parts.put("maxdist", UTF8.StringBody(Integer.toString(maxDistance)));
parts.put("profile", UTF8.StringBody(crypt.simpleEncode(event.getQuery().ranking.toExternalString()))); parts.put("profile", UTF8.StringBody(crypt.simpleEncode(event.query.ranking.toExternalString())));
parts.put("constraint", UTF8.StringBody((event.getQuery().constraint == null) ? "" : event.getQuery().constraint.exportB64())); parts.put("constraint", UTF8.StringBody((event.query.constraint == null) ? "" : event.query.constraint.exportB64()));
if ( secondarySearchSuperviser != null ) { if ( secondarySearchSuperviser != null ) {
parts.put("abstracts", UTF8.StringBody("auto")); parts.put("abstracts", UTF8.StringBody("auto"));
// resultMap = FileUtils.table(HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + hostaddress + "/yacy/search.html"), 60000, hostname, parts)); // resultMap = FileUtils.table(HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + hostaddress + "/yacy/search.html"), 60000, hostname, parts));
@ -1018,26 +1022,27 @@ public final class Protocol
} }
} }
public static int solrQuery( protected static int solrQuery(
final SearchEvent event, final SearchEvent event,
final int offset, final int offset,
final int count, final int count,
final Seed target, final Seed target,
final Blacklist blacklist) { final Blacklist blacklist) {
if (event.getQuery().queryString == null || event.getQuery().queryString.length() == 0) { if (event.query.queryString == null || event.query.queryString.length() == 0) {
return -1; // we cannot query solr only with word hashes, there is no clear text string return -1; // we cannot query solr only with word hashes, there is no clear text string
} }
event.addExpectedRemoteReferences(count); event.addExpectedRemoteReferences(count);
QueryResponse rsp = null;
SolrDocumentList docList = null; SolrDocumentList docList = null;
final SolrQuery solrQuery = event.getQuery().solrQuery(); final SolrQuery solrQuery = event.query.solrQuery();
solrQuery.setStart(offset); solrQuery.setStart(offset);
solrQuery.setRows(count); solrQuery.setRows(count);
boolean localsearch = target == null || target.equals(event.peers.mySeed()); boolean localsearch = target == null || target.equals(event.peers.mySeed());
if (localsearch) { if (localsearch) {
// search the local index // search the local index
try { try {
QueryResponse rsp = event.rankingProcess.getQuery().getSegment().fulltext().getSolr().query(solrQuery); rsp = event.rankingProcess.getQuery().getSegment().fulltext().getSolr().query(solrQuery);
docList = rsp.getResults(); docList = rsp.getResults();
} catch (SolrException e) { } catch (SolrException e) {
Network.log.logInfo("SEARCH failed (solr, 1), localpeer (" + e.getMessage() + ")", e); Network.log.logInfo("SEARCH failed (solr, 1), localpeer (" + e.getMessage() + ")", e);
@ -1050,7 +1055,7 @@ public final class Protocol
final String solrURL = "http://" + target.getPublicAddress() + "/solr"; final String solrURL = "http://" + target.getPublicAddress() + "/solr";
try { try {
SolrConnector solrConnector = new RemoteSolrConnector(solrURL); SolrConnector solrConnector = new RemoteSolrConnector(solrURL);
QueryResponse rsp = solrConnector.query(solrQuery); rsp = solrConnector.query(solrQuery);
docList = rsp.getResults(); docList = rsp.getResults();
// no need to close this here because that sends a commit to remote solr which is not wanted here // no need to close this here because that sends a commit to remote solr which is not wanted here
} catch (IOException e) { } catch (IOException e) {
@ -1103,7 +1108,7 @@ public final class Protocol
// passed all checks, store url // passed all checks, store url
if (!localsearch) { if (!localsearch) {
try { try {
event.getQuery().getSegment().fulltext().putDocument(ClientUtils.toSolrInputDocument(doc)); event.query.getSegment().fulltext().putDocument(ClientUtils.toSolrInputDocument(doc));
ResultURLs.stack( ResultURLs.stack(
ASCII.String(urlEntry.url().hash()), ASCII.String(urlEntry.url().hash()),
urlEntry.url().getHost(), urlEntry.url().getHost(),
@ -1121,12 +1126,12 @@ public final class Protocol
} }
if (localsearch) { if (localsearch) {
event.add(container, true, "localpeer", docList.size()); event.add(container, true, "localpeer", (int) docList.getNumFound());
event.rankingProcess.addFinalize(); event.rankingProcess.addFinalize();
event.addExpectedRemoteReferences(-count); event.addExpectedRemoteReferences(-count);
Network.log.logInfo("local search (solr): localpeer sent " + container.get(0).size() + "/" + docList.size() + " references"); Network.log.logInfo("local search (solr): localpeer sent " + container.get(0).size() + "/" + docList.size() + " references");
} else { } else {
event.add(container, false, target.getName() + "/" + target.hash, docList.size()); event.add(container, false, target.getName() + "/" + target.hash, (int) docList.getNumFound());
event.rankingProcess.addFinalize(); event.rankingProcess.addFinalize();
event.addExpectedRemoteReferences(-count); event.addExpectedRemoteReferences(-count);
Network.log.logInfo("remote search (solr): peer " + target.getName() + " sent " + container.get(0).size() + "/" + docList.size() + " references"); Network.log.logInfo("remote search (solr): peer " + target.getName() + " sent " + container.get(0).size() + "/" + docList.size() + " references");

18
source/net/yacy/peers/RemoteSearch.java
Unescape View File

@ -149,7 +149,7 @@ public class RemoteSearch extends Thread {
(clusterselection == null) ? (clusterselection == null) ?
DHTSelection.selectSearchTargets( DHTSelection.selectSearchTargets(
event.peers, event.peers,
event.getQuery().query_include_hashes, event.query.query_include_hashes,
event.peers.redundancy(), event.peers.redundancy(),
burstRobinsonPercent, burstRobinsonPercent,
burstMultiwordPercent) burstMultiwordPercent)
@ -172,16 +172,16 @@ public class RemoteSearch extends Thread {
try { try {
RemoteSearch rs = new RemoteSearch( RemoteSearch rs = new RemoteSearch(
event, event,
QueryParams.hashSet2hashString(event.getQuery().query_include_hashes), QueryParams.hashSet2hashString(event.query.query_include_hashes),
QueryParams.hashSet2hashString(event.getQuery().query_exclude_hashes), QueryParams.hashSet2hashString(event.query.query_exclude_hashes),
event.getQuery().modifier, event.query.modifier,
event.getQuery().targetlang == null ? "" : event.getQuery().targetlang, event.query.targetlang == null ? "" : event.query.targetlang,
event.getQuery().sitehash == null ? "" : event.getQuery().sitehash, event.query.sitehash == null ? "" : event.query.sitehash,
event.getQuery().authorhash == null ? "" : event.getQuery().authorhash, event.query.authorhash == null ? "" : event.query.authorhash,
event.getQuery().contentdom == null ? "all" : event.getQuery().contentdom.toString(), event.query.contentdom == null ? "all" : event.query.contentdom.toString(),
count, count,
time, time,
event.getQuery().maxDistance, event.query.maxDistance,
targets, targets,
targetPeers[i], targetPeers[i],
event.secondarySearchSuperviser, event.secondarySearchSuperviser,

3
source/net/yacy/peers/graphics/NetworkGraph.java
Unescape View File

@ -157,8 +157,7 @@ public class NetworkGraph {
*/ */
// draw in the search target // draw in the search target
final QueryParams query = event.getQuery(); final Iterator<byte[]> i = event.query.query_include_hashes.iterator();
final Iterator<byte[]> i = query.query_include_hashes.iterator();
eventPicture.setColor(RasterPlotter.GREY); eventPicture.setColor(RasterPlotter.GREY);
while (i.hasNext()) { while (i.hasNext()) {
byte[] wordHash = i.next(); byte[] wordHash = i.next();

5
source/net/yacy/search/Switchboard.java
Unescape View File

@ -2671,7 +2671,7 @@ public final class Switchboard extends serverSwitch {
} }
// check if some of the links match with the query // check if some of the links match with the query
final Map<MultiProtocolURI, String> matcher = searchEvent.getQuery().separateMatches(links); final Map<MultiProtocolURI, String> matcher = searchEvent.query.separateMatches(links);
// take the matcher and load them all // take the matcher and load them all
for ( final Map.Entry<MultiProtocolURI, String> entry : matcher.entrySet() ) { for ( final Map.Entry<MultiProtocolURI, String> entry : matcher.entrySet() ) {
@ -3319,8 +3319,7 @@ public final class Switchboard extends serverSwitch {
new Thread() { new Thread() {
@Override @Override
public void run() { public void run() {
QueryParams query = searchEvent.getQuery(); String queryString = searchEvent.query.queryString(true);
String queryString = query.queryString(true);
Thread.currentThread().setName("Switchboard.heuristicRSS:" + queryString); Thread.currentThread().setName("Switchboard.heuristicRSS:" + queryString);
final int meta = queryString.indexOf("heuristic:", 0); final int meta = queryString.indexOf("heuristic:", 0);
if ( meta >= 0 ) { if ( meta >= 0 ) {

2
source/net/yacy/search/index/Segment.java
Unescape View File

@ -413,7 +413,7 @@ public class Segment {
// data during search-time. To transfer indexed data directly to the search process // data during search-time. To transfer indexed data directly to the search process
// the following lines push the index data additionally to the search process // the following lines push the index data additionally to the search process
// this is done only for searched words // this is done only for searched words
if (searchEvent != null && !searchEvent.getQuery().query_exclude_hashes.has(wordhash) && searchEvent.getQuery().query_include_hashes.has(wordhash)) { if (searchEvent != null && !searchEvent.query.query_exclude_hashes.has(wordhash) && searchEvent.query.query_include_hashes.has(wordhash)) {
// if the page was added in the context of a heuristic this shall ensure that findings will fire directly into the search result // if the page was added in the context of a heuristic this shall ensure that findings will fire directly into the search result
ReferenceContainer<WordReference> container; ReferenceContainer<WordReference> container;
try { try {

10
source/net/yacy/search/query/AccessTracker.java
Unescape View File

@ -44,9 +44,9 @@ public class AccessTracker {
private final static long DUMP_PERIOD = 60000L; private final static long DUMP_PERIOD = 60000L;
public static final int minSize = 100; private static final int minSize = 100;
public static final int maxSize = 1000; private static final int maxSize = 1000;
public static final int maxAge = 24 * 60 * 60 * 1000; private static final int maxAge = 24 * 60 * 60 * 1000;
public enum Location {local, remote} public enum Location {local, remote}
@ -67,7 +67,7 @@ public class AccessTracker {
private static void add(final LinkedList<QueryParams> list, final QueryParams query) { private static void add(final LinkedList<QueryParams> list, final QueryParams query) {
// learn that this word can be a word completion for the DidYouMeanLibrary // learn that this word can be a word completion for the DidYouMeanLibrary
if (query.resultcount > 10 && query.queryString != null && query.queryString.length() > 0) { if (query.getResultCount() > 10 && query.queryString != null && query.queryString.length() > 0) {
final StringBuilder sb = new StringBuilder(query.queryString); final StringBuilder sb = new StringBuilder(query.queryString);
sb.append(query.queryString); sb.append(query.queryString);
WordCache.learn(sb); WordCache.learn(sb);
@ -109,7 +109,7 @@ public class AccessTracker {
private static void addToDump(final QueryParams query) { private static void addToDump(final QueryParams query) {
if (query.queryString == null || query.queryString.isEmpty()) return; if (query.queryString == null || query.queryString.isEmpty()) return;
addToDump(query.queryString, Integer.toString(query.resultcount), new Date(query.starttime)); addToDump(query.queryString, Integer.toString(query.getResultCount()), new Date(query.starttime));
} }
public static void addToDump(String querystring, String resultcount) { public static void addToDump(String querystring, String resultcount) {

50
source/net/yacy/search/query/QueryParams.java
Unescape View File

@ -30,12 +30,15 @@ import java.io.UnsupportedEncodingException;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.SortedSet; import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException; import java.util.regex.PatternSyntaxException;
@ -58,6 +61,7 @@ import net.yacy.document.Condenser;
import net.yacy.document.parser.html.AbstractScraper; import net.yacy.document.parser.html.AbstractScraper;
import net.yacy.document.parser.html.CharacterCoding; import net.yacy.document.parser.html.CharacterCoding;
import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.meta.URIMetadataRow;
import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.data.word.WordReferenceRow; import net.yacy.kelondro.data.word.WordReferenceRow;
import net.yacy.kelondro.index.RowHandleSet; import net.yacy.kelondro.index.RowHandleSet;
@ -103,18 +107,16 @@ public final class QueryParams {
public final String queryString; public final String queryString;
public final HandleSet query_include_hashes, query_exclude_hashes, query_all_hashes; public final HandleSet query_include_hashes, query_exclude_hashes, query_all_hashes;
private final Collection<String> query_include_words, query_exclude_words, query_all_words; private final Collection<String> query_include_words, query_exclude_words, query_all_words;
public final int itemsPerPage; public int itemsPerPage;
public int offset; public int offset;
public final Pattern urlMask, prefer; public final Pattern urlMask, prefer;
final boolean urlMask_isCatchall; final boolean urlMask_isCatchall;
private final boolean prefer_isMatchnothing;
public final Classification.ContentDomain contentdom; public final Classification.ContentDomain contentdom;
public final String targetlang; public final String targetlang;
protected final Collection<Tagging.Metatag> metatags; protected final Collection<Tagging.Metatag> metatags;
public final String navigators; public final String navigators;
public final Searchdom domType; public final Searchdom domType;
private final int zonecode; private final int zonecode;
private final int domMaxTargets;
public final int maxDistance; public final int maxDistance;
public final Bitfield constraint; public final Bitfield constraint;
final boolean allofconstraint; final boolean allofconstraint;
@ -131,14 +133,21 @@ public final class QueryParams {
protected final long maxtime; protected final long maxtime;
protected final long timeout; protected final long timeout;
// values that are set after a search: // values that are set after a search:
public int resultcount; // number of found results
public int transmitcount; // number of results that had been shown to the user public int transmitcount; // number of results that had been shown to the user
public long searchtime, urlretrievaltime, snippetcomputationtime; // time to perform the search, to get all the urls, and to compute the snippets public long searchtime, urlretrievaltime, snippetcomputationtime; // time to perform the search, to get all the urls, and to compute the snippets
private boolean specialRights; // is true if the user has a special authorization and my use more database-extensive options
public final String userAgent; public final String userAgent;
public boolean filterfailurls; protected boolean filterfailurls;
protected double lat, lon, radius; protected double lat, lon, radius;
// the following values are filled during the search process as statistics for the search
public final AtomicInteger local_rwi_available; // the number of hits generated/ranked by the local search in rwi index
public final AtomicInteger local_rwi_stored; // the number of existing hits by the local search in rwi index
public final AtomicInteger local_solr_available; // the number of hits generated/ranked by the local search in solr
public final AtomicInteger local_solr_stored; // the number of existing hits by the local search in solr
public final AtomicInteger remote_available; // the number of hits imported from remote peers (rwi/solr mixed)
public final AtomicInteger remote_stored; // the number of existing hits at remote site
public final AtomicInteger remote_peerCount; // the number of peers which contributed to the remote search result
public final SortedSet<byte[]> misses; // url hashes that had been sorted out because of constraints in postranking
public QueryParams( public QueryParams(
final String queryString, final String queryString,
@ -178,7 +187,6 @@ public final class QueryParams {
this.urlMask = catchall_pattern; this.urlMask = catchall_pattern;
this.urlMask_isCatchall = true; this.urlMask_isCatchall = true;
this.prefer = matchnothing_pattern; this.prefer = matchnothing_pattern;
this.prefer_isMatchnothing = true;
this.contentdom = ContentDomain.ALL; this.contentdom = ContentDomain.ALL;
this.itemsPerPage = itemsPerPage; this.itemsPerPage = itemsPerPage;
this.offset = 0; this.offset = 0;
@ -186,7 +194,6 @@ public final class QueryParams {
this.metatags = new ArrayList<Tagging.Metatag>(0); this.metatags = new ArrayList<Tagging.Metatag>(0);
this.domType = Searchdom.LOCAL; this.domType = Searchdom.LOCAL;
this.zonecode = DigestURI.TLD_any_zone_filter; this.zonecode = DigestURI.TLD_any_zone_filter;
this.domMaxTargets = 0;
this.constraint = constraint; this.constraint = constraint;
this.allofconstraint = false; this.allofconstraint = false;
this.snippetCacheStrategy = null; this.snippetCacheStrategy = null;
@ -198,7 +205,6 @@ public final class QueryParams {
this.starttime = Long.valueOf(System.currentTimeMillis()); this.starttime = Long.valueOf(System.currentTimeMillis());
this.maxtime = 10000; this.maxtime = 10000;
this.timeout = this.starttime + this.timeout; this.timeout = this.starttime + this.timeout;
this.specialRights = false;
this.navigators = "all"; this.navigators = "all";
this.indexSegment = indexSegment; this.indexSegment = indexSegment;
this.userAgent = userAgent; this.userAgent = userAgent;
@ -207,6 +213,14 @@ public final class QueryParams {
this.lat = 0.0d; this.lat = 0.0d;
this.lon = 0.0d; this.lon = 0.0d;
this.radius = 0.0d; this.radius = 0.0d;
this.local_rwi_available = new AtomicInteger(0); // the number of results in the local peer after filtering
this.local_rwi_stored = new AtomicInteger(0);
this.local_solr_available= new AtomicInteger(0);
this.local_solr_stored = new AtomicInteger(0);
this.remote_stored = new AtomicInteger(0);
this.remote_available = new AtomicInteger(0); // the number of result contributions from all the remote peers
this.remote_peerCount = new AtomicInteger(0); // the number of remote peers that have contributed
this.misses = Collections.synchronizedSortedSet(new TreeSet<byte[]>(URIMetadataRow.rowdef.objectOrder));
} }
public QueryParams( public QueryParams(
@ -262,14 +276,13 @@ public final class QueryParams {
} catch (final PatternSyntaxException ex) { } catch (final PatternSyntaxException ex) {
throw new IllegalArgumentException("Not a valid regular expression: " + prefer, ex); throw new IllegalArgumentException("Not a valid regular expression: " + prefer, ex);
} }
this.prefer_isMatchnothing = this.prefer.toString().equals(matchnothing_pattern.toString()); this.prefer.toString().equals(matchnothing_pattern.toString());
assert language != null; assert language != null;
this.targetlang = language; this.targetlang = language;
this.metatags = metatags; this.metatags = metatags;
this.navigators = navigators; this.navigators = navigators;
this.domType = domType; this.domType = domType;
this.zonecode = domainzone; this.zonecode = domainzone;
this.domMaxTargets = domMaxTargets;
this.constraint = constraint; this.constraint = constraint;
this.allofconstraint = allofconstraint; this.allofconstraint = allofconstraint;
this.sitehash = site; assert site == null || site.length() == 6; this.sitehash = site; assert site == null || site.length() == 6;
@ -281,7 +294,6 @@ public final class QueryParams {
this.starttime = Long.valueOf(System.currentTimeMillis()); this.starttime = Long.valueOf(System.currentTimeMillis());
this.maxtime = 10000; this.maxtime = 10000;
this.timeout = this.starttime + this.timeout; this.timeout = this.starttime + this.timeout;
this.specialRights = specialRights;
this.indexSegment = indexSegment; this.indexSegment = indexSegment;
this.userAgent = userAgent; this.userAgent = userAgent;
this.transmitcount = 0; this.transmitcount = 0;
@ -291,6 +303,14 @@ public final class QueryParams {
this.lat = Math.floor(lat * this.kmNormal) / this.kmNormal; this.lat = Math.floor(lat * this.kmNormal) / this.kmNormal;
this.lon = Math.floor(lon * this.kmNormal) / this.kmNormal; this.lon = Math.floor(lon * this.kmNormal) / this.kmNormal;
this.radius = Math.floor(radius * this.kmNormal + 1) / this.kmNormal; this.radius = Math.floor(radius * this.kmNormal + 1) / this.kmNormal;
this.local_rwi_available = new AtomicInteger(0); // the number of results in the local peer after filtering
this.local_rwi_stored = new AtomicInteger(0);
this.local_solr_available= new AtomicInteger(0);
this.local_solr_stored = new AtomicInteger(0);
this.remote_stored = new AtomicInteger(0);
this.remote_available = new AtomicInteger(0); // the number of result contributions from all the remote peers
this.remote_peerCount = new AtomicInteger(0); // the number of remote peers that have contributed
this.misses = Collections.synchronizedSortedSet(new TreeSet<byte[]>(URIMetadataRow.rowdef.objectOrder));
} }
private double kmNormal = 100.d; // 100 =ca 40000.d / 360.d == 111.11 - if lat/lon is multiplied with this, rounded and diveded by this, the location is normalized to a 1km grid private double kmNormal = 100.d; // 100 =ca 40000.d / 360.d == 111.11 - if lat/lon is multiplied with this, rounded and diveded by this, the location is normalized to a 1km grid
@ -309,6 +329,10 @@ public final class QueryParams {
return this.itemsPerPage; return this.itemsPerPage;
} }
public int getResultCount() {
return this.local_rwi_available.get() + this.local_solr_stored.get() - this.misses.size();
}
public void setOffset(final int newOffset) { public void setOffset(final int newOffset) {
this.offset = newOffset; this.offset = newOffset;
} }
@ -502,7 +526,7 @@ public final class QueryParams {
final SolrQuery params = new SolrQuery(); final SolrQuery params = new SolrQuery();
params.setQuery(q.toString()); params.setQuery(q.toString());
params.setStart(this.offset); params.setStart(this.offset);
params.setRows(this.resultcount); params.setRows(this.itemsPerPage);
params.setFacet(false); params.setFacet(false);
if (this.radius > 0.0d && this.lat != 0.0d && this.lon != 0.0d) { if (this.radius > 0.0d && this.lat != 0.0d && this.lon != 0.0d) {

60
source/net/yacy/search/query/RankingProcess.java
Unescape View File

@ -84,23 +84,18 @@ public final class RankingProcess extends Thread {
protected final WeakPriorityBlockingQueue<WordReferenceVars> rwiStack; protected final WeakPriorityBlockingQueue<WordReferenceVars> rwiStack;
protected final ConcurrentHashMap<String, WeakPriorityBlockingQueue<WordReferenceVars>> doubleDomCache; // key = domhash (6 bytes); value = like stack protected final ConcurrentHashMap<String, WeakPriorityBlockingQueue<WordReferenceVars>> doubleDomCache; // key = domhash (6 bytes); value = like stack
private final int[] flagcount; // flag counter private final int[] flagcount; // flag counter
private final HandleSet misses; // contains url-hashes that could not been found in the LURL-DB
private final AtomicInteger feedersAlive, feedersTerminated; private final AtomicInteger feedersAlive, feedersTerminated;
private boolean addRunning; private boolean addRunning;
protected final AtomicInteger receivedRemoteReferences; protected final AtomicInteger receivedRemoteReferences;
protected final ReferenceOrder order; protected final ReferenceOrder order;
int local_indexCount; protected final HandleSet urlhashes; // map for double-check; String/Long relation, addresses ranking number (backreference for deletion)
int remote_indexCount; protected final ScoreMap<String> hostNavigator; // a counter for the appearance of the host hash
int remote_resourceSize; protected final Map<String, byte[]> hostResolver; // a mapping from a host hash (6 bytes) to the full url hash of one of these urls that have the host hash
int remote_peerCount; protected final Map<String, String> taggingPredicates; // a map from tagging vocabulary names to tagging predicate uris
final HandleSet urlhashes; // map for double-check; String/Long relation, addresses ranking number (backreference for deletion) protected final Map<String, ScoreMap<String>> vocabularyNavigator; // counters for Vocabularies; key is metatag.getVocabularyName()
final ScoreMap<String> hostNavigator; // a counter for the appearance of the host hash
final Map<String, byte[]> hostResolver; // a mapping from a host hash (6 bytes) to the full url hash of one of these urls that have the host hash
final Map<String, String> taggingPredicates; // a map from tagging vocabulary names to tagging predicate uris
final Map<String, ScoreMap<String>> vocabularyNavigator; // counters for Vocabularies; key is metatag.getVocabularyName()
private boolean remote; private boolean remote;
public RankingProcess(final QueryParams query, boolean remote) { protected RankingProcess(final QueryParams query, boolean remote) {
// we collect the urlhashes and construct a list with urlEntry objects // we collect the urlhashes and construct a list with urlEntry objects
// attention: if minEntries is too high, this method will not terminate within the maxTime // attention: if minEntries is too high, this method will not terminate within the maxTime
// sortorder: 0 = hash, 1 = url, 2 = ranking // sortorder: 0 = hash, 1 = url, 2 = ranking
@ -112,7 +107,6 @@ public final class RankingProcess extends Thread {
int stackMaxsize = query.snippetCacheStrategy == null || query.snippetCacheStrategy == CacheStrategy.CACHEONLY ? max_results_preparation_special : max_results_preparation; int stackMaxsize = query.snippetCacheStrategy == null || query.snippetCacheStrategy == CacheStrategy.CACHEONLY ? max_results_preparation_special : max_results_preparation;
this.rwiStack = new WeakPriorityBlockingQueue<WordReferenceVars>(stackMaxsize, false); this.rwiStack = new WeakPriorityBlockingQueue<WordReferenceVars>(stackMaxsize, false);
this.doubleDomCache = new ConcurrentHashMap<String, WeakPriorityBlockingQueue<WordReferenceVars>>(); this.doubleDomCache = new ConcurrentHashMap<String, WeakPriorityBlockingQueue<WordReferenceVars>>();
this.misses = new RowHandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 100);
this.flagcount = new int[32]; this.flagcount = new int[32];
for ( int i = 0; i < 32; i++ ) { for ( int i = 0; i < 32; i++ ) {
this.flagcount[i] = 0; this.flagcount[i] = 0;
@ -122,10 +116,6 @@ public final class RankingProcess extends Thread {
this.addRunning = true; this.addRunning = true;
this.receivedRemoteReferences = new AtomicInteger(0); this.receivedRemoteReferences = new AtomicInteger(0);
this.order = new ReferenceOrder(this.query.ranking, UTF8.getBytes(this.query.targetlang)); this.order = new ReferenceOrder(this.query.ranking, UTF8.getBytes(this.query.targetlang));
this.remote_resourceSize = 0;
this.remote_indexCount = 0;
this.remote_peerCount = 0;
this.local_indexCount = 0;
this.urlhashes = new RowHandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 100); this.urlhashes = new RowHandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 100);
this.hostNavigator = new ConcurrentScoreMap<String>(); this.hostNavigator = new ConcurrentScoreMap<String>();
this.hostResolver = new ConcurrentHashMap<String, byte[]>(); this.hostResolver = new ConcurrentHashMap<String, byte[]>();
@ -160,11 +150,6 @@ public final class RankingProcess extends Thread {
this.feedersAlive.addAndGet(1); this.feedersAlive.addAndGet(1);
} }
public int rwiAvailableCount() {
// the number of results in the local peer after filtering
return this.rwiStack.sizeAvailable();
}
public QueryParams getQuery() { public QueryParams getQuery() {
return this.query; return this.query;
} }
@ -173,14 +158,6 @@ public final class RankingProcess extends Thread {
return this.flagcount; return this.flagcount;
} }
public Iterator<byte[]> miss() {
return this.misses.iterator();
}
public int getMissCount() {
return this.misses.size();
}
protected void addBegin() { protected void addBegin() {
this.addRunning = true; this.addRunning = true;
} }
@ -230,21 +207,6 @@ public final class RankingProcess extends Thread {
return false; return false;
} }
public int getRemoteIndexCount() {
// the number of result contributions from all the remote peers
return this.remote_indexCount;
}
public int getRemoteResourceSize() {
// the number of all hits in all the remote peers
return Math.max(this.remote_resourceSize, this.remote_indexCount);
}
public int getRemotePeerCount() {
// the number of remote peers that have contributed
return this.remote_peerCount;
}
@Override @Override
public void run() { public void run() {
@ -300,10 +262,12 @@ public final class RankingProcess extends Thread {
this.addRunning = true; this.addRunning = true;
assert (index != null); assert (index != null);
if (index.isEmpty()) return; if (index.isEmpty()) return;
if (!local) { if (local) {
this.query.local_rwi_stored.addAndGet(fullResource);
} else {
assert fullResource >= 0 : "fullResource = " + fullResource; assert fullResource >= 0 : "fullResource = " + fullResource;
this.remote_resourceSize += fullResource; this.query.remote_stored.addAndGet(fullResource);
this.remote_peerCount++; this.query.remote_peerCount.incrementAndGet();
} }
long timer = System.currentTimeMillis(); long timer = System.currentTimeMillis();
@ -433,7 +397,7 @@ public final class RankingProcess extends Thread {
} }
} }
// increase counter for statistics // increase counter for statistics
if (local) this.local_indexCount++; else this.remote_indexCount++; if (local) this.query.local_rwi_available.incrementAndGet(); else this.query.remote_available.incrementAndGet();
} }
if (System.currentTimeMillis() >= timeout) Log.logWarning("RWIProcess", "rwi normalization ended with timeout = " + maxtime); if (System.currentTimeMillis() >= timeout) Log.logWarning("RWIProcess", "rwi normalization ended with timeout = " + maxtime);

53
source/net/yacy/search/query/SearchEvent.java
Unescape View File

@ -90,7 +90,7 @@ public final class SearchEvent {
private final static int SNIPPET_WORKER_THREADS = Math.max(4, Runtime.getRuntime().availableProcessors() * 2); private final static int SNIPPET_WORKER_THREADS = Math.max(4, Runtime.getRuntime().availableProcessors() * 2);
private long eventTime; private long eventTime;
protected QueryParams query; public QueryParams query;
public final SeedDB peers; public final SeedDB peers;
final WorkTables workTables; final WorkTables workTables;
public final RankingProcess rankingProcess; // ordered search results, grows dynamically as all the query threads enrich this container public final RankingProcess rankingProcess; // ordered search results, grows dynamically as all the query threads enrich this container
@ -104,8 +104,7 @@ public final class SearchEvent {
private final SortedMap<byte[], HeuristicResult> heuristics; private final SortedMap<byte[], HeuristicResult> heuristics;
private byte[] IAmaxcounthash, IAneardhthash; private byte[] IAmaxcounthash, IAneardhthash;
private final Thread localsearch; private final Thread localsearch;
private final AtomicInteger expectedRemoteReferences, maxExpectedRemoteReferences; private final AtomicInteger expectedRemoteReferences, maxExpectedRemoteReferences; // counter for referenced that had been sorted out for other reasons
private int sortout; // counter for referenced that had been sorted out for other reasons
private final ScoreMap<String> authorNavigator; // a counter for the appearances of authors private final ScoreMap<String> authorNavigator; // a counter for the appearances of authors
private final ScoreMap<String> namespaceNavigator; // a counter for name spaces private final ScoreMap<String> namespaceNavigator; // a counter for name spaces
private final ScoreMap<String> protocolNavigator; // a counter for protocol types private final ScoreMap<String> protocolNavigator; // a counter for protocol types
@ -120,7 +119,6 @@ public final class SearchEvent {
protected long snippetComputationAllTime; protected long snippetComputationAllTime;
private final boolean remote; private final boolean remote;
private boolean cleanupState; private boolean cleanupState;
private int resultCounter = 0;
protected SearchEvent( protected SearchEvent(
final QueryParams query, final QueryParams query,
@ -144,7 +142,6 @@ public final class SearchEvent {
this.maxExpectedRemoteReferences = new AtomicInteger(0); this.maxExpectedRemoteReferences = new AtomicInteger(0);
this.expectedRemoteReferences = new AtomicInteger(0); this.expectedRemoteReferences = new AtomicInteger(0);
this.sortout = 0;
this.authorNavigator = new ConcurrentScoreMap<String>(); this.authorNavigator = new ConcurrentScoreMap<String>();
this.namespaceNavigator = new ConcurrentScoreMap<String>(); this.namespaceNavigator = new ConcurrentScoreMap<String>();
this.protocolNavigator = new ConcurrentScoreMap<String>(); this.protocolNavigator = new ConcurrentScoreMap<String>();
@ -315,14 +312,6 @@ public final class SearchEvent {
this.eventTime = System.currentTimeMillis(); this.eventTime = System.currentTimeMillis();
} }
public QueryParams getQuery() {
return this.query;
}
public void setQuery(final QueryParams query) {
this.query = query;
}
protected void cleanup() { protected void cleanup() {
this.cleanupState = true; this.cleanupState = true;
@ -486,10 +475,12 @@ public final class SearchEvent {
assert (index != null); assert (index != null);
if (index.isEmpty()) return; if (index.isEmpty()) return;
if (!local) { if (local) {
this.query.local_solr_stored.set(fullResource);
} else {
assert fullResource >= 0 : "fullResource = " + fullResource; assert fullResource >= 0 : "fullResource = " + fullResource;
this.rankingProcess.remote_resourceSize += fullResource; this.query.remote_stored.addAndGet(fullResource);
this.rankingProcess.remote_peerCount++; this.query.remote_peerCount.incrementAndGet();
} }
long timer = System.currentTimeMillis(); long timer = System.currentTimeMillis();
@ -604,7 +595,7 @@ public final class SearchEvent {
} }
} }
// increase counter for statistics // increase counter for statistics
if (local) this.rankingProcess.local_indexCount++; else this.rankingProcess.remote_indexCount++; if (local) this.query.local_solr_available.incrementAndGet(); else this.query.remote_available.incrementAndGet();
} }
} catch ( final SpaceExceededException e ) { } catch ( final SpaceExceededException e ) {
} }
@ -626,10 +617,6 @@ public final class SearchEvent {
this.expectedRemoteReferences.addAndGet(x); this.expectedRemoteReferences.addAndGet(x);
} }
public int getSortOutCount() {
return this.sortout;
}
private URIMetadataNode takeRWI(final boolean skipDoubleDom, final long waitingtime) { private URIMetadataNode takeRWI(final boolean skipDoubleDom, final long waitingtime) {
// returns from the current RWI list the best entry and removes this entry from the list // returns from the current RWI list the best entry and removes this entry from the list
@ -771,27 +758,27 @@ public final class SearchEvent {
if ( !this.query.urlMask_isCatchall ) { if ( !this.query.urlMask_isCatchall ) {
// check url mask // check url mask
if ( !page.matches(this.query.urlMask) ) { if ( !page.matches(this.query.urlMask) ) {
this.sortout++; this.query.misses.add(page.hash());
continue; continue;
} }
} }
// check for more errors // check for more errors
if ( page.url() == null ) { if ( page.url() == null ) {
this.sortout++; this.query.misses.add(page.hash());
continue; // rare case where the url is corrupted continue; // rare case where the url is corrupted
} }
// check content domain // check content domain
if ((this.query.contentdom.getCode() > 0 && page.url().getContentDomain() != this.query.contentdom) || if ((this.query.contentdom.getCode() > 0 && page.url().getContentDomain() != this.query.contentdom) ||
(this.query.contentdom == Classification.ContentDomain.TEXT && page.url().getContentDomain().getCode() > 0)) { (this.query.contentdom == Classification.ContentDomain.TEXT && page.url().getContentDomain().getCode() > 0)) {
this.sortout++; this.query.misses.add(page.hash());
continue; continue;
} }
// Check for blacklist // Check for blacklist
if (Switchboard.urlBlacklist.isListed(BlacklistType.SEARCH, page)) { if (Switchboard.urlBlacklist.isListed(BlacklistType.SEARCH, page)) {
this.sortout++; this.query.misses.add(page.hash());
continue; continue;
} }
@ -805,7 +792,7 @@ public final class SearchEvent {
FilterEngine f = ContentControlFilterUpdateThread.getNetworkFilter(); FilterEngine f = ContentControlFilterUpdateThread.getNetworkFilter();
if (f != null) { if (f != null) {
if (!f.isListed(page.url(), null)) { if (!f.isListed(page.url(), null)) {
this.sortout++; this.query.misses.add(page.hash());
continue; continue;
} }
} }
@ -821,7 +808,7 @@ public final class SearchEvent {
((QueryParams.anymatch(pagetitle, this.query.query_exclude_hashes)) ((QueryParams.anymatch(pagetitle, this.query.query_exclude_hashes))
|| (QueryParams.anymatch(pageurl.toLowerCase(), this.query.query_exclude_hashes)) || (QueryParams.anymatch(pageurl.toLowerCase(), this.query.query_exclude_hashes))
|| (QueryParams.anymatch(pageauthor.toLowerCase(), this.query.query_exclude_hashes)))) { || (QueryParams.anymatch(pageauthor.toLowerCase(), this.query.query_exclude_hashes)))) {
this.sortout++; this.query.misses.add(page.hash());
continue; continue;
} }
@ -831,13 +818,13 @@ public final class SearchEvent {
while ( wi.hasNext() ) { while ( wi.hasNext() ) {
this.query.getSegment().termIndex().removeDelayed(wi.next(), page.hash()); this.query.getSegment().termIndex().removeDelayed(wi.next(), page.hash());
} }
this.sortout++; this.query.misses.add(page.hash());
continue; continue;
} }
// check location constraint // check location constraint
if ((this.query.constraint != null) && (this.query.constraint.get(Condenser.flag_cat_haslocation)) && (page.lat() == 0.0f || page.lon() == 0.0f)) { if ((this.query.constraint != null) && (this.query.constraint.get(Condenser.flag_cat_haslocation)) && (page.lat() == 0.0f || page.lon() == 0.0f)) {
this.sortout++; this.query.misses.add(page.hash());
continue; continue;
} }
@ -848,7 +835,7 @@ public final class SearchEvent {
double lonDelta = this.query.lon - lon; double lonDelta = this.query.lon - lon;
double distance = Math.sqrt(latDelta * latDelta + lonDelta * lonDelta); // pythagoras double distance = Math.sqrt(latDelta * latDelta + lonDelta * lonDelta); // pythagoras
if (distance > this.query.radius) { if (distance > this.query.radius) {
this.sortout++; this.query.misses.add(page.hash());
continue; continue;
} }
} }
@ -861,20 +848,20 @@ public final class SearchEvent {
// check if we already are filtering for authors // check if we already are filtering for authors
if ( this.query.authorhash != null && !this.query.authorhash.equals(authorhash) ) { if ( this.query.authorhash != null && !this.query.authorhash.equals(authorhash) ) {
this.sortout++; this.query.misses.add(page.hash());
continue; continue;
} }
// add author to the author navigator // add author to the author navigator
this.authorNavigator.inc(pageauthor); this.authorNavigator.inc(pageauthor);
} else if ( this.query.authorhash != null ) { } else if ( this.query.authorhash != null ) {
this.sortout++; this.query.misses.add(page.hash());
continue; continue;
} }
// check Scanner // check Scanner
if ( !Scanner.acceptURL(page.url()) ) { if ( !Scanner.acceptURL(page.url()) ) {
this.sortout++; this.query.misses.add(page.hash());
continue; continue;
} }

5
source/net/yacy/search/query/SearchEventCache.java
Unescape View File

@ -138,8 +138,9 @@ public class SearchEventCache {
if (event != null) { if (event != null) {
//re-new the event time for this event, so it is not deleted next time too early //re-new the event time for this event, so it is not deleted next time too early
event.resetEventTime(); event.resetEventTime();
// replace the query, because this contains the current result offset // replace the current result offset
event.setQuery(query); event.query.offset = query.offset;
event.query.itemsPerPage = query.itemsPerPage;
} }
} }
if (event == null) { if (event == null) {

Write Preview
Loading…
Cancel
Save