From 4b8ff84705b6e6e8aa634776cad016a05ad0c604 Mon Sep 17 00:00:00 2001 From: orbiter Date: Fri, 2 Dec 2011 09:16:24 +0000 Subject: [PATCH] - search bugfixes (page counter and number of results per page; recognition of new search) - experiments to speed-up the network image production (commented out) git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@8130 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/index.html | 8 +- htroot/index.java | 3 +- htroot/yacysearch.java | 41 +++--- .../net/yacy/peers/graphics/NetworkGraph.java | 122 ++++++++++++------ 4 files changed, 106 insertions(+), 68 deletions(-) diff --git a/htroot/index.html b/htroot/index.html index 16c8844dd..4039a881d 100644 --- a/htroot/index.html +++ b/htroot/index.html @@ -116,13 +116,7 @@ : - #(urlmaskoptions)# - - :: - restrict on - show all - #(/urlmaskoptions)# - + diff --git a/htroot/index.java b/htroot/index.java index ee034c075..4780a8be3 100644 --- a/htroot/index.java +++ b/htroot/index.java @@ -106,8 +106,7 @@ public class index { prop.put("searchoptions_resource-select_global", global ? "1" : "0"); prop.put("searchoptions_resource-select_global-disabled", indexReceiveGranted ? "0" : "1"); prop.put("searchoptions_resource-select_local", global ? "0" : "1"); - prop.put("searchoptions_urlmaskoptions", "0"); - prop.putHTML("searchoptions_urlmaskoptions_urlmaskfilter", urlmaskfilter); + prop.putHTML("searchoptions_urlmaskfilter", urlmaskfilter); prop.put("searchoptions_prefermaskoptions", "0"); prop.putHTML("searchoptions_prefermaskoptions_prefermaskfilter", prefermaskfilter); prop.put("searchoptions_indexofChecked", ""); diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index f01329ebc..7ae89fa42 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -179,18 +179,15 @@ public class yacysearch { } // collect search attributes - final boolean newsearch =post.hasValue("query") && post.hasValue("former") && !post.get("query","").equalsIgnoreCase(post.get("former","")); //new search term - int itemsPerPage = Math.min((authenticated) ? (snippetFetchStrategy != null && snippetFetchStrategy.isAllowedToFetchOnline() ? 100 : 5000) : (snippetFetchStrategy != null && snippetFetchStrategy.isAllowedToFetchOnline() ? 20 : 1000), post.getInt("maximumRecords", post.getInt("count", 10))); // SRU syntax with old property as alternative - int offset = (newsearch) ? 0 : post.getInt("startRecord", post.getInt("offset", 0)); + int maximumRecords = Math.min((authenticated) ? (snippetFetchStrategy != null && snippetFetchStrategy.isAllowedToFetchOnline() ? 100 : 5000) : (snippetFetchStrategy != null && snippetFetchStrategy.isAllowedToFetchOnline() ? 20 : 1000), post.getInt("maximumRecords", post.getInt("count", 10))); // SRU syntax with old property as alternative + int startRecord = post.getInt("startRecord", post.getInt("offset", 0)); boolean global = post.get("resource", "local").equals("global") && sb.peers.sizeConnected() > 0; final boolean indexof = (post != null && post.get("indexof","").equals("on")); final String originalUrlMask; - if (post.containsKey("urlmask") && post.get("urlmask").equals("no")) { // option search all - originalUrlMask = ".*"; - } else if (!newsearch && post.containsKey("urlmaskfilter")) { + if (post.containsKey("urlmaskfilter")) { originalUrlMask = post.get("urlmaskfilter", ".*"); } else { originalUrlMask = ".*"; @@ -231,8 +228,10 @@ public class yacysearch { final ContentDomain contentdom = ContentDomain.contentdomParser(post == null ? "text" : post.get("contentdom", "text")); // patch until better search profiles are available - if ((contentdom != ContentDomain.TEXT) && (itemsPerPage <= 32)) { - itemsPerPage = 64; + if (contentdom == ContentDomain.TEXT) { + if (maximumRecords > 50) maximumRecords = 10; + } else { + if (maximumRecords <= 32) maximumRecords = 64; } // check the search tracker @@ -610,8 +609,8 @@ public class yacysearch { language, navigation, snippetFetchStrategy, - itemsPerPage, - offset, + maximumRecords, + startRecord, urlmask, clustersearch && global ? QueryParams.Searchdom.CLUSTER : (global && indexReceiveGranted ? QueryParams.Searchdom.GLOBAL : QueryParams.Searchdom.LOCAL), @@ -644,7 +643,7 @@ public class yacysearch { // create a new search event if (SearchEventCache.getEvent(theQuery.id(false)) == null) { theQuery.setOffset(0); // in case that this is a new search, always start without a offset - offset = 0; + startRecord = 0; } final SearchEvent theSearch = SearchEventCache.getEvent( theQuery, sb.peers, sb.tables, (sb.isRobinsonMode()) ? sb.clusterhashes : null, false, sb.loader, @@ -653,7 +652,7 @@ public class yacysearch { (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_ROBINSON, 0), (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_MULTIWORD, 0)); - if (offset == 0) { + if (startRecord == 0) { if (sitehost != null && sb.getConfigBool("heuristic.site", false) && authenticated) { sb.heuristicSite(theSearch, sitehost); } @@ -707,7 +706,7 @@ public class yacysearch { // find geographic info final SortedSet coordinates = LibraryProvider.geoLoc.find(originalquerystring, false); - if (coordinates == null || coordinates.isEmpty() || offset > 0) { + if (coordinates == null || coordinates.isEmpty() || startRecord > 0) { prop.put("geoinfo", "0"); } else { int i = 0; @@ -740,9 +739,9 @@ public class yacysearch { } final int indexcount = theSearch.getRankingResult().getLocalIndexCount() - theSearch.getRankingResult().getMissCount() - theSearch.getRankingResult().getSortOutCount() + theSearch.getRankingResult().getRemoteIndexCount(); - prop.put("num-results_offset", offset == 0 ? 0 : offset + 1); - prop.put("num-results_itemscount", Formatter.number(offset + theSearch.getQuery().itemsPerPage > indexcount ? offset + indexcount % theSearch.getQuery().itemsPerPage : offset + theSearch.getQuery().itemsPerPage, true)); - prop.put("num-results_itemsPerPage", itemsPerPage); + prop.put("num-results_offset", startRecord == 0 ? 0 : startRecord + 1); + prop.put("num-results_itemscount", Formatter.number(startRecord + theSearch.getQuery().itemsPerPage > indexcount ? startRecord + indexcount % theSearch.getQuery().itemsPerPage : startRecord + theSearch.getQuery().itemsPerPage, true)); + prop.put("num-results_itemsPerPage", maximumRecords); prop.put("num-results_totalcount", Formatter.number(indexcount, true)); prop.put("num-results_globalresults", global && (indexReceiveGranted || clustersearch) ? "1" : "0"); prop.put("num-results_globalresults_localResourceSize", Formatter.number(theSearch.getRankingResult().getLocalIndexCount(), true)); @@ -753,7 +752,7 @@ public class yacysearch { // compose page navigation final StringBuilder resnav = new StringBuilder(200); - final int thispage = offset / theQuery.displayResults(); + final int thispage = startRecord / theQuery.displayResults(); if (thispage == 0) { resnav.append("\"arrowleft\" "); } else { @@ -789,12 +788,12 @@ public class yacysearch { } final String resnavs = resnav.toString(); prop.put("num-results_resnav", resnavs); - prop.put("pageNavBottom", (indexcount - offset > 6) ? 1 : 0); // if there are more results than may fit on the page we add a navigation at the bottom + prop.put("pageNavBottom", (indexcount - startRecord > 6) ? 1 : 0); // if there are more results than may fit on the page we add a navigation at the bottom prop.put("pageNavBottom_resnav", resnavs); // generate the search result lines; the content will be produced by another servlet for (int i = 0; i < theQuery.displayResults(); i++) { - prop.put("results_" + i + "_item", offset + i); + prop.put("results_" + i + "_item", startRecord + i); prop.put("results_" + i + "_eventID", theQuery.id(false)); } prop.put("results", theQuery.displayResults()); @@ -833,8 +832,8 @@ public class yacysearch { prop.put("searchagain", global ? "1" : "0"); prop.putHTML("former", originalquerystring); - prop.put("count", itemsPerPage); - prop.put("offset", offset); + prop.put("count", maximumRecords); + prop.put("offset", startRecord); prop.put("resource", global ? "global" : "local"); prop.putHTML("urlmaskfilter", originalUrlMask); prop.putHTML("prefermaskfilter", prefermask); diff --git a/source/net/yacy/peers/graphics/NetworkGraph.java b/source/net/yacy/peers/graphics/NetworkGraph.java index 7e332efce..d16869636 100644 --- a/source/net/yacy/peers/graphics/NetworkGraph.java +++ b/source/net/yacy/peers/graphics/NetworkGraph.java @@ -198,6 +198,24 @@ public class NetworkGraph { Seed seed; long lastseen; + // start processes that actually draw the peers + //final BlockingQueue drawQueue = new LinkedBlockingDeque(); + //final drawNetworkPicturePeerJob poison = new drawNetworkPicturePeerJob(); + /* + final Thread[] drawThreads = new Thread[Runtime.getRuntime().availableProcessors()]; + for (int i = 0; i < drawThreads.length; i++) { + drawThreads[i] = new Thread() { + public void run() { + try { + drawNetworkPicturePeerJob job; + while ((job = drawQueue.take()) != poison) job.draw(); + } catch (final InterruptedException e) { + } + } + }; + drawThreads[i].start(); + } + */ // draw connected senior and principals int count = 0; @@ -213,7 +231,7 @@ public class NetworkGraph { continue; } //Log.logInfo("NetworkGraph", "drawing peer " + seed.getName()); - drawNetworkPicturePeer(networkPicture, width / 2, height / 2, innerradius, outerradius, seed, COL_ACTIVE_DOT, COL_ACTIVE_LINE, COL_ACTIVE_TEXT, coronaangle, cyc); + new drawNetworkPicturePeerJob(networkPicture, width / 2, height / 2, innerradius, outerradius, seed, COL_ACTIVE_DOT, COL_ACTIVE_LINE, COL_ACTIVE_TEXT, coronaangle, cyc).draw(); count++; } totalCount += count; @@ -231,7 +249,7 @@ public class NetworkGraph { if (lastseen > passiveLimit) { break; // we have enough, this list is sorted so we don't miss anything } - drawNetworkPicturePeer(networkPicture, width / 2, height / 2, innerradius, outerradius, seed, COL_PASSIVE_DOT, COL_PASSIVE_LINE, COL_PASSIVE_TEXT, coronaangle, cyc); + new drawNetworkPicturePeerJob(networkPicture, width / 2, height / 2, innerradius, outerradius, seed, COL_PASSIVE_DOT, COL_PASSIVE_LINE, COL_PASSIVE_TEXT, coronaangle, cyc).draw(); count++; } totalCount += count; @@ -249,13 +267,16 @@ public class NetworkGraph { if (lastseen > potentialLimit) { break; // we have enough, this list is sorted so we don't miss anything } - drawNetworkPicturePeer(networkPicture, width / 2, height / 2, innerradius, outerradius, seed, COL_POTENTIAL_DOT, COL_POTENTIAL_LINE, COL_POTENTIAL_TEXT, coronaangle, cyc); + new drawNetworkPicturePeerJob(networkPicture, width / 2, height / 2, innerradius, outerradius, seed, COL_POTENTIAL_DOT, COL_POTENTIAL_LINE, COL_POTENTIAL_TEXT, coronaangle, cyc).draw(); count++; } totalCount += count; // draw my own peer - drawNetworkPicturePeer(networkPicture, width / 2, height / 2, innerradius, outerradius, seedDB.mySeed(), COL_MYPEER_DOT, COL_MYPEER_LINE, COL_MYPEER_TEXT, coronaangle, cyc); + new drawNetworkPicturePeerJob(networkPicture, width / 2, height / 2, innerradius, outerradius, seedDB.mySeed(), COL_MYPEER_DOT, COL_MYPEER_LINE, COL_MYPEER_TEXT, coronaangle, cyc).draw(); + + // signal termination + //for (@SuppressWarnings("unused") final Thread t: drawThreads) try { drawQueue.put(poison); } catch (final InterruptedException ee) {} // draw DHT activity if (communicationTimeout >= 0) { @@ -283,6 +304,9 @@ public class NetworkGraph { PrintTool.print(networkPicture, width - 2, 6, 0, "SNAPSHOT FROM " + new Date().toString().toUpperCase(), 1); PrintTool.print(networkPicture, width - 2, 14, 0, "DRAWING OF " + totalCount + " SELECTED PEERS", 1); + // wait for draw termination + //for (final Thread t: drawThreads) try { t.join(); } catch (final InterruptedException ee) {} + return networkPicture; } @@ -298,42 +322,64 @@ public class NetworkGraph { colorLine, 100, null, 100, 12, (coronaangle < 0) ? -1 : coronaangle / 30, 2, true); } - private static void drawNetworkPicturePeer( - final RasterPlotter img, final int centerX, final int centerY, - final int innerradius, final int outerradius, - final Seed seed, - final String colorDot, final String colorLine, final String colorText, - final int coronaangle, - final double cyc) { - final String name = seed.getName().toUpperCase() /*+ ":" + seed.hash + ":" + (((double) ((int) (100 * (((double) yacySeed.dhtPosition(seed.hash)) / ((double) yacySeed.maxDHTDistance))))) / 100.0)*/; - if (name.length() < shortestName) shortestName = name.length(); - if (name.length() > longestName) longestName = name.length(); - final double angle = cyc + (360.0d * FlatWordPartitionScheme.std.dhtPosition(ASCII.getBytes(seed.hash), null) / DOUBLE_LONG_MAX_VALUE); - //System.out.println("Seed " + seed.hash + " has distance " + seed.dhtDistance() + ", angle = " + angle); - int linelength = 20 + outerradius * (20 * (name.length() - shortestName) / (longestName - shortestName) + Math.abs(seed.hash.hashCode() % 20)) / 80; - if (linelength > outerradius) linelength = outerradius; - int dotsize = 2 + (int) (seed.getLinkCount() / 2000000L); - if (colorDot.equals(COL_MYPEER_DOT)) dotsize = dotsize + 4; - if (dotsize > 18) dotsize = 18; - // draw dot - img.setColor(colorDot); - img.arcDot(centerX, centerY, innerradius, angle, dotsize); - // draw line to text - img.arcLine(centerX, centerY, innerradius + 18, innerradius + linelength, angle, true, colorLine, "444444", 12, coronaangle / 30, 0, true); - // draw text - img.setColor(colorText); - PrintTool.arcPrint(img, centerX, centerY, innerradius + linelength, angle, name); - - // draw corona around dot for crawling activity - final int ppmx = seed.getPPM() / 40; - if (coronaangle >= 0 && ppmx > 0) { - drawCorona(img, centerX, centerY, innerradius, angle, dotsize, ppmx, coronaangle, true, false, 2, 2, 2); // color = 0..63 + private static class drawNetworkPicturePeerJob { + + private RasterPlotter img; + private int centerX, centerY, innerradius, outerradius, coronaangle; + private Seed seed; + private String colorDot, colorLine, colorText; + private double cyc; + public drawNetworkPicturePeerJob() {} // used to produce a poison pill + public drawNetworkPicturePeerJob( + final RasterPlotter img, final int centerX, final int centerY, + final int innerradius, final int outerradius, + final Seed seed, + final String colorDot, final String colorLine, final String colorText, + final int coronaangle, + final double cyc) { + this.img = img; + this.centerX = centerX; + this.centerY = centerY; + this.innerradius = innerradius; + this.outerradius = outerradius; + this.coronaangle = coronaangle; + this.seed = seed; + this.colorDot = colorDot; + this.colorLine = colorLine; + this.colorText = colorText; + this.cyc = cyc; } + public void draw() { + final String name = this.seed.getName().toUpperCase() /*+ ":" + seed.hash + ":" + (((double) ((int) (100 * (((double) yacySeed.dhtPosition(seed.hash)) / ((double) yacySeed.maxDHTDistance))))) / 100.0)*/; + if (name.length() < shortestName) shortestName = name.length(); + if (name.length() > longestName) longestName = name.length(); + final double angle = this.cyc + (360.0d * FlatWordPartitionScheme.std.dhtPosition(ASCII.getBytes(this.seed.hash), null) / DOUBLE_LONG_MAX_VALUE); + //System.out.println("Seed " + seed.hash + " has distance " + seed.dhtDistance() + ", angle = " + angle); + int linelength = 20 + this.outerradius * (20 * (name.length() - shortestName) / (longestName - shortestName) + Math.abs(this.seed.hash.hashCode() % 20)) / 80; + if (linelength > this.outerradius) linelength = this.outerradius; + int dotsize = 2 + (int) (this.seed.getLinkCount() / 2000000L); + if (this.colorDot.equals(COL_MYPEER_DOT)) dotsize = dotsize + 4; + if (dotsize > 18) dotsize = 18; + // draw dot + this.img.setColor(this.colorDot); + this.img.arcDot(this.centerX, this.centerY, this.innerradius, angle, dotsize); + // draw line to text + this.img.arcLine(this.centerX, this.centerY, this.innerradius + 18, this.innerradius + linelength, angle, true, this.colorLine, "444444", 12, this.coronaangle / 30, 0, true); + // draw text + this.img.setColor(this.colorText); + PrintTool.arcPrint(this.img, this.centerX, this.centerY, this.innerradius + linelength, angle, name); + + // draw corona around dot for crawling activity + final int ppmx = this.seed.getPPM() / 40; + if (this.coronaangle >= 0 && ppmx > 0) { + drawCorona(this.img, this.centerX, this.centerY, this.innerradius, angle, dotsize, ppmx, this.coronaangle, true, false, 2, 2, 2); // color = 0..63 + } - // draw corona around dot for query activity - final int qphx = ((int) (seed.getQPM() * 4.0)); - if (coronaangle >= 0 && qphx > 0) { - drawCorona(img, centerX, centerY, innerradius, angle, dotsize, qphx, coronaangle, false, true, 10, 40, 10); // color = 0..63 + // draw corona around dot for query activity + final int qphx = ((int) (this.seed.getQPM() * 4.0)); + if (this.coronaangle >= 0 && qphx > 0) { + drawCorona(this.img, this.centerX, this.centerY, this.innerradius, angle, dotsize, qphx, this.coronaangle, false, true, 10, 40, 10); // color = 0..63 + } } }