From 7bf421b9ddeb282be6befe404dd3506d8fc9e95f Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Mon, 21 May 2012 01:58:29 +0200 Subject: [PATCH] - fixed image search page navigation - removed some deadlocks and ConcurrentModificationExceptions during DidYouMean collection --- htroot/yacy/search.java | 4 +- htroot/yacysearch.java | 65 ++++++++++--------- htroot/yacysearchitem.java | 8 +-- htroot/yacysearchlatestinfo.java | 2 +- source/net/yacy/search/query/QueryParams.java | 6 +- 5 files changed, 45 insertions(+), 40 deletions(-) diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java index 8c44f3260..38258a5b9 100644 --- a/htroot/yacy/search.java +++ b/htroot/yacy/search.java @@ -257,7 +257,7 @@ public final class search { header.get(RequestHeader.USER_AGENT, ""), false ); - Network.log.logInfo("INIT HASH SEARCH (abstracts only): " + QueryParams.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.displayResults() + " links"); + Network.log.logInfo("INIT HASH SEARCH (abstracts only): " + QueryParams.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.itemsPerPage() + " links"); final long timer = System.currentTimeMillis(); //final Map>[] containers = sb.indexSegment.index().searchTerm(theQuery.queryHashes, theQuery.excludeHashes, plasmaSearchQuery.hashes2StringSet(urls)); @@ -320,7 +320,7 @@ public final class search { header.get(RequestHeader.USER_AGENT, ""), false ); - Network.log.logInfo("INIT HASH SEARCH (query-" + abstracts + "): " + QueryParams.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.displayResults() + " links"); + Network.log.logInfo("INIT HASH SEARCH (query-" + abstracts + "): " + QueryParams.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.itemsPerPage() + " links"); EventChannel.channels(EventChannel.REMOTESEARCH).addMessage(new RSSMessage("Remote Search Request from " + ((remoteSeed == null) ? "unknown" : remoteSeed.getName()), QueryParams.anonymizedQueryHashes(theQuery.queryHashes), "")); // make event diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index ff82313b9..5e9f54b22 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -31,6 +31,7 @@ import java.io.IOException; import java.net.InetAddress; import java.util.ArrayList; import java.util.Collection; +import java.util.ConcurrentModificationException; import java.util.HashMap; import java.util.Iterator; import java.util.Map; @@ -122,7 +123,7 @@ public class yacysearch { //get focus option prop.put("focus", ((post == null) ? true : post.get("focus", "1").equals("1")) ? 1 : 0); - + // produce vocabulary navigation sidebars Collection vocabularies = LibraryProvider.autotagging.getVocabularies(); int j = 0; @@ -212,7 +213,7 @@ public class yacysearch { // collect search attributes - int maximumRecords = + int itemsPerPage = Math.min( (authenticated) ? (snippetFetchStrategy != null && snippetFetchStrategy.isAllowedToFetchOnline() @@ -275,10 +276,10 @@ public class yacysearch { ContentDomain.contentdomParser(post == null ? "all" : post.get("contentdom", "all")); // patch until better search profiles are available - if (contentdom == ContentDomain.IMAGE && (maximumRecords == 10 || maximumRecords == 100)) { - maximumRecords = 64; - } else if ( maximumRecords > 50 && maximumRecords < 100 ) { - maximumRecords = 10; + if (contentdom == ContentDomain.IMAGE && (itemsPerPage == 10 || itemsPerPage == 100)) { + itemsPerPage = 64; + } else if ( contentdom != ContentDomain.IMAGE && itemsPerPage > 50 && itemsPerPage < 100 ) { + itemsPerPage = 10; } // check the search tracker @@ -736,7 +737,7 @@ public class yacysearch { metatags, navigation, snippetFetchStrategy, - maximumRecords, + itemsPerPage, startRecord, urlmask, clustersearch && global ? QueryParams.Searchdom.CLUSTER : (global && indexReceiveGranted @@ -781,7 +782,7 @@ public class yacysearch { + " - " + theQuery.neededResults() + " links to be computed, " - + theQuery.displayResults() + + theQuery.itemsPerPage() + " lines to be displayed"); EventChannel.channels(EventChannel.LOCALSEARCH).addMessage( new RSSMessage("Local Search Request", theQuery.queryString, "")); @@ -858,21 +859,25 @@ public class yacysearch { final Iterator meanIt = didYouMean.getSuggestions(100, 5).iterator(); int meanCount = 0; String suggestion; - while ( meanCount < meanMax && meanIt.hasNext() ) { - suggestion = meanIt.next().toString(); - prop.put("didYouMean_suggestions_" + meanCount + "_word", suggestion); - prop.put( - "didYouMean_suggestions_" + meanCount + "_url", - QueryParams.navurl( - "html", - 0, - theQuery, - suggestion, - originalUrlMask.toString(), - theQuery.navigators).toString()); - prop.put("didYouMean_suggestions_" + meanCount + "_sep", "|"); - meanCount++; - } + try { + meanCollect: while ( meanCount < meanMax && meanIt.hasNext() ) { + try { + suggestion = meanIt.next().toString(); + prop.put("didYouMean_suggestions_" + meanCount + "_word", suggestion); + prop.put( + "didYouMean_suggestions_" + meanCount + "_url", + QueryParams.navurl( + "html", + 0, + theQuery, + suggestion, + originalUrlMask.toString(), + theQuery.navigators).toString()); + prop.put("didYouMean_suggestions_" + meanCount + "_sep", "|"); + meanCount++; + } catch (ConcurrentModificationException e) {break meanCollect;} + } + } catch (ConcurrentModificationException e) {} prop.put("didYouMean_suggestions_" + (meanCount - 1) + "_sep", ""); prop.put("didYouMean", meanCount > 0 ? 1 : 0); prop.put("didYouMean_suggestions", meanCount); @@ -931,7 +936,7 @@ public class yacysearch { + indexcount % theSearch.getQuery().itemsPerPage : startRecord + theSearch.getQuery().itemsPerPage, true)); - prop.put("num-results_itemsPerPage", maximumRecords); + prop.put("num-results_itemsPerPage", itemsPerPage); prop.put("num-results_totalcount", Formatter.number(indexcount, true)); prop.put("num-results_globalresults", global && (indexReceiveGranted || clustersearch) ? "1" @@ -954,7 +959,7 @@ public class yacysearch { // compose page navigation final StringBuilder resnav = new StringBuilder(200); - final int thispage = startRecord / theQuery.displayResults(); + final int thispage = startRecord / theQuery.itemsPerPage(); if ( thispage == 0 ) { resnav .append("\"arrowleft\" "); @@ -970,7 +975,7 @@ public class yacysearch { resnav .append("\">\"arrowleft\" "); } - final int numberofpages = Math.min(10, 1 + ((indexcount - 1) / theQuery.displayResults())); + final int numberofpages = Math.min(10, 1 + ((indexcount - 1) / theQuery.itemsPerPage())); for ( int i = 0; i < numberofpages; i++ ) { if ( i == thispage ) { @@ -1012,11 +1017,11 @@ public class yacysearch { prop.put("pageNavBottom_resnav", resnavs); // generate the search result lines; the content will be produced by another servlet - for ( int i = 0; i < theQuery.displayResults(); i++ ) { + for ( int i = 0; i < theQuery.itemsPerPage(); i++ ) { prop.put("results_" + i + "_item", startRecord + i); prop.put("results_" + i + "_eventID", theQuery.id(false)); } - prop.put("results", theQuery.displayResults()); + prop.put("results", theQuery.itemsPerPage()); prop .put( "resultTable", @@ -1050,7 +1055,7 @@ public class yacysearch { String hostName = header.get("Host", "localhost"); if ( hostName.indexOf(':', 0) == -1 ) { hostName += ":" + serverCore.getPortNr(env.getConfig("port", "8090")); - } + } prop.put("searchBaseURL", "http://" + hostName + "/yacysearch.html"); prop.put("rssYacyImageURL", "http://" + hostName + "/env/grafics/yacy.gif"); prop.put("thisaddress", hostName); @@ -1058,7 +1063,7 @@ public class yacysearch { prop.put("searchagain", global ? "1" : "0"); prop.putHTML("former", originalquerystring); - prop.put("count", maximumRecords); + prop.put("count", itemsPerPage); prop.put("offset", startRecord); prop.put("resource", global ? "global" : "local"); prop.putHTML("urlmaskfilter", originalUrlMask); diff --git a/htroot/yacysearchitem.java b/htroot/yacysearchitem.java index f6bdf3e55..c5f823473 100644 --- a/htroot/yacysearchitem.java +++ b/htroot/yacysearchitem.java @@ -90,7 +90,7 @@ public class yacysearchitem { // dynamically update count values final int totalcount = theSearch.getRankingResult().getLocalIndexCount() - theSearch.getRankingResult().getMissCount() - theSearch.getRankingResult().getSortOutCount() + theSearch.getRankingResult().getRemoteIndexCount(); - final int offset = theQuery.neededResults() - theQuery.displayResults() + 1; + final int offset = theQuery.neededResults() - theQuery.itemsPerPage() + 1; prop.put("offset", offset); prop.put("itemscount", Formatter.number(Math.min((item < 0) ? theQuery.neededResults() : item + 1, totalcount))); prop.put("itemsperpage", Formatter.number(theQuery.itemsPerPage)); @@ -131,10 +131,10 @@ public class yacysearchitem { prop.put("content_authorized", authenticated ? "1" : "0"); final String urlhash = ASCII.String(result.hash()); prop.put("content_authorized_bookmark", sb.tables.bookmarks.hasBookmark("admin", urlhash) ? "0" : "1"); - prop.putHTML("content_authorized_bookmark_bookmarklink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.displayResults() + "&offset=" + (theQuery.neededResults() - theQuery.displayResults()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=" + resource + "&time=3&bookmarkref=" + urlhash + "&urlmaskfilter=.*"); + prop.putHTML("content_authorized_bookmark_bookmarklink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.itemsPerPage() + "&offset=" + (theQuery.neededResults() - theQuery.itemsPerPage()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=" + resource + "&time=3&bookmarkref=" + urlhash + "&urlmaskfilter=.*"); prop.put("content_authorized_recommend", (sb.peers.newsPool.getSpecific(NewsPool.OUTGOING_DB, NewsPool.CATEGORY_SURFTIPP_ADD, "url", resultUrlstring) == null) ? "1" : "0"); - prop.putHTML("content_authorized_recommend_deletelink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.displayResults() + "&offset=" + (theQuery.neededResults() - theQuery.displayResults()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=" + resource + "&time=3&deleteref=" + urlhash + "&urlmaskfilter=.*"); - prop.putHTML("content_authorized_recommend_recommendlink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.displayResults() + "&offset=" + (theQuery.neededResults() - theQuery.displayResults()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=" + resource + "&time=3&recommendref=" + urlhash + "&urlmaskfilter=.*"); + prop.putHTML("content_authorized_recommend_deletelink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.itemsPerPage() + "&offset=" + (theQuery.neededResults() - theQuery.itemsPerPage()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=" + resource + "&time=3&deleteref=" + urlhash + "&urlmaskfilter=.*"); + prop.putHTML("content_authorized_recommend_recommendlink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.itemsPerPage() + "&offset=" + (theQuery.neededResults() - theQuery.itemsPerPage()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=" + resource + "&time=3&recommendref=" + urlhash + "&urlmaskfilter=.*"); prop.put("content_authorized_urlhash", urlhash); final String resulthashString = urlhash; prop.putHTML("content_title", result.title()); diff --git a/htroot/yacysearchlatestinfo.java b/htroot/yacysearchlatestinfo.java index 031f91892..662f3844f 100644 --- a/htroot/yacysearchlatestinfo.java +++ b/htroot/yacysearchlatestinfo.java @@ -34,7 +34,7 @@ public class yacysearchlatestinfo { // dynamically update count values final int totalcount = theSearch.getRankingResult().getLocalIndexCount() - theSearch.getRankingResult().getMissCount() - theSearch.getRankingResult().getSortOutCount() + theSearch.getRankingResult().getRemoteIndexCount(); - final int offset = theQuery.neededResults() - theQuery.displayResults() + 1; + final int offset = theQuery.neededResults() - theQuery.itemsPerPage() + 1; prop.put("offset", offset); prop.put("itemscount",Formatter.number(offset + theSearch.getQuery().itemsPerPage >= totalcount ? offset + totalcount % theSearch.getQuery().itemsPerPage - 1 : offset + theSearch.getQuery().itemsPerPage - 1)); prop.put("itemsperpage", theSearch.getQuery().itemsPerPage); diff --git a/source/net/yacy/search/query/QueryParams.java b/source/net/yacy/search/query/QueryParams.java index ec2dd8ccb..4c313365a 100644 --- a/source/net/yacy/search/query/QueryParams.java +++ b/source/net/yacy/search/query/QueryParams.java @@ -282,7 +282,7 @@ public final class QueryParams { return this.offset + this.itemsPerPage; } - public int displayResults() { + public int itemsPerPage() { // the number of result lines that are displayed at once (size of result page) return this.itemsPerPage; } @@ -537,7 +537,7 @@ public final class QueryParams { sb.append(ampersand); sb.append("startRecord="); - sb.append(page * theQuery.displayResults()); + sb.append(page * theQuery.itemsPerPage()); return sb; } @@ -554,7 +554,7 @@ public final class QueryParams { sb.append(ampersand); sb.append("maximumRecords="); - sb.append(theQuery.displayResults()); + sb.append(theQuery.itemsPerPage()); sb.append(ampersand); sb.append("resource=");