- better caclculation of search result size

- predefined search recommendations

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7361 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/js/yacyinteractive.js

@@ -84,7 +84,13 @@ function resultStart() {
       html += "<div id=\"downloadbutton\" style=\"float:left\"></div></form>";
   } else {
       if (query == "") {
-         html += "please enter some search words";
+         html += "please enter some search words<br\>or use the following predefined search queries:<br\>";
+         html += "find images: ";
+         html += "(<a style=\"text-decoration:underline\" href=\"/yacyinteractive.html?query=png+filetype:png\">png</a>),";
+         html += "(<a style=\"text-decoration:underline\" href=\"/yacyinteractive.html?query=gif+filetype:gif\">gif</a>),";
+         html += "(<a style=\"text-decoration:underline\" href=\"/yacyinteractive.html?query=jpg+filetype:jpg\">jpg</a>)<br>";
+         html += "list: ";
+         html += "<a style=\"text-decoration:underline\" href=\"/yacyinteractive.html?query=pdf+/date+filetype:pdf\">recent pdf</a>,";
       } else {
          html += "no results";
       }
@@ -125,7 +131,7 @@ function resultList() {
   var html = "";
   if (searchresult.length > 0) {
     html += "<table class=\"sortable\" id=\"sortable\" border=\"0\" cellpadding=\"0\" cellspacing=\"1\" width=\"99%\">";
-    html += "<tr class=\"TableHeader\" valign=\"bottom\"><td width=\"40\">Protocol</td><td width=\"60\">Host</td><td width=\"260\">Path</td><td width=\"360\">Name</td><td width=\"60\">Size</td><td width=\"70\">Date</td></tr>";
+    html += "<tr class=\"TableHeader\" valign=\"bottom\"><td width=\"40\">Protocol</td><td width=\"60\">Host</td><td width=\"260\">Path</td><td width=\"360\">Name</td><td width=\"60\">Size</td><td width=\"75\">Date</td></tr>";
     for (var i = 0; i < searchresult.length; i++) { html += resultLine("row", searchresult[i]); }
     html += "</table>";
   }

htroot/yacy/search.java

@@ -294,12 +294,12 @@ public final class search {
             theSearch = SearchEventCache.getEvent(theQuery, sb.peers, sb.crawlResults, null, abstracts.length() > 0, sb.loader);
             
             // set statistic details of search result and find best result index set
-            joincount = theSearch.getRankingResult().getLocalIndexCount();
+            joincount = theSearch.getRankingResult().getLocalIndexCount() - theSearch.getRankingResult().getMissCount();
             prop.put("joincount", Integer.toString(joincount));
             if (joincount != 0) {
                 accu = theSearch.result().completeResults(3000);
             }
-            if (theSearch.getRankingResult().getLocalIndexCount() == 0 || abstracts.length() == 0) {
+            if (joincount <= 0 || abstracts.length() == 0) {
                 prop.put("indexcount", "");
                 prop.put("joincount", "0");
             } else {
@@ -394,7 +394,7 @@ public final class search {
 
         // prepare search statistics
         theQuery.remotepeer = client == null ? null : sb.peers.lookupByIP(Domains.dnsResolve(client), true, false, false);
-        theQuery.resultcount = (theSearch == null) ? 0 : theSearch.getRankingResult().getLocalIndexCount() + theSearch.getRankingResult().getRemoteResourceSize();
+        theQuery.resultcount = (theSearch == null) ? 0 : joincount;
         theQuery.searchtime = System.currentTimeMillis() - timestamp;
         theQuery.urlretrievaltime = (theSearch == null) ? 0 : theSearch.result().getURLRetrievalTime();
         theQuery.snippetcomputationtime = (theSearch == null) ? 0 : theSearch.result().getSnippetComputationTime();

htroot/yacysearch.java

@@ -176,7 +176,7 @@ public class yacysearch {
         // collect search attributes
         boolean newsearch = post.hasValue("query") && post.hasValue("former") && !post.get("query","").equalsIgnoreCase(post.get("former","")); //new search term
         
-        int itemsPerPage = Math.min((authenticated) ? (snippetFetchStrategy != null && snippetFetchStrategy.isAllowedToFetchOnline() ? 100 : 1000) : (snippetFetchStrategy != null && snippetFetchStrategy.isAllowedToFetchOnline() ? 10 : 100), post.getInt("maximumRecords", post.getInt("count", 10))); // SRU syntax with old property as alternative
+        int itemsPerPage = Math.min((authenticated) ? (snippetFetchStrategy != null && snippetFetchStrategy.isAllowedToFetchOnline() ? 100 : 1000) : (snippetFetchStrategy != null && snippetFetchStrategy.isAllowedToFetchOnline() ? 20 : 500), post.getInt("maximumRecords", post.getInt("count", 10))); // SRU syntax with old property as alternative
         int offset = (newsearch) ? 0 : post.getInt("startRecord", post.getInt("offset", 0));
         
         int newcount;
@@ -545,11 +545,13 @@ public class yacysearch {
 
             // log
             Log.logInfo("LOCAL_SEARCH", "EXIT WORD SEARCH: " + theQuery.queryString + " - " +
-                    (theSearch.getRankingResult().getLocalIndexCount() + theSearch.getRankingResult().getRemoteResourceSize()) + " links found, " +
+                    "local-unfiltered(" + theSearch.getRankingResult().getLocalIndexCount() + "), " +
+                    "-local_miss(" + theSearch.getRankingResult().getMissCount() + "), " +
+                    "remote(" + theSearch.getRankingResult().getRemoteResourceSize() + ") links found, " +
                     (System.currentTimeMillis() - timestamp) + " ms");
 
             // prepare search statistics
-            theQuery.resultcount = theSearch.getRankingResult().getLocalIndexCount() + theSearch.getRankingResult().getRemoteResourceSize();
+            theQuery.resultcount = theSearch.getRankingResult().getLocalIndexCount() - theSearch.getRankingResult().getMissCount() + theSearch.getRankingResult().getRemoteResourceSize();
             theQuery.searchtime = System.currentTimeMillis() - timestamp;
             theQuery.urlretrievaltime = theSearch.result().getURLRetrievalTime();
             theQuery.snippetcomputationtime = theSearch.result().getSnippetComputationTime();
@@ -613,12 +615,14 @@ public class yacysearch {
                 Log.logException(e);
             }
             
+            int indexcount = theSearch.getRankingResult().getLocalIndexCount() - theSearch.getRankingResult().getMissCount() + theSearch.getRankingResult().getRemoteResourceSize();
             prop.put("num-results_offset", offset);
             prop.put("num-results_itemscount", Formatter.number(0, true));
             prop.put("num-results_itemsPerPage", itemsPerPage);
-            prop.put("num-results_totalcount", Formatter.number(theSearch.getRankingResult().getLocalIndexCount() + theSearch.getRankingResult().getRemoteResourceSize(), true));
+            prop.put("num-results_totalcount", Formatter.number(indexcount, true));
             prop.put("num-results_globalresults", (globalsearch) ? "1" : "0");
             prop.put("num-results_globalresults_localResourceSize", Formatter.number(theSearch.getRankingResult().getLocalIndexCount(), true));
+            prop.put("num-results_globalresults_localMissCount", Formatter.number(theSearch.getRankingResult().getMissCount(), true));
             prop.put("num-results_globalresults_remoteResourceSize", Formatter.number(theSearch.getRankingResult().getRemoteResourceSize(), true));
             prop.put("num-results_globalresults_remoteIndexCount", Formatter.number(theSearch.getRankingResult().getRemoteIndexCount(), true));
             prop.put("num-results_globalresults_remotePeerCount", Formatter.number(theSearch.getRankingResult().getRemotePeerCount(), true));
@@ -633,7 +637,7 @@ public class yacysearch {
                 resnav.append(QueryParams.navurl("html", thispage - 1, display, theQuery, null, originalUrlMask, navigation));
             	resnav.append("\"><img src=\"env/grafics/navdl.gif\" alt=\"arrowleft\" width=\"16\" height=\"16\" /></a>&nbsp;");
             }
-            final int numberofpages = Math.min(10, Math.max(1 + thispage, 1 + ((theSearch.getRankingResult().getLocalIndexCount() < 11) ? Math.max(30, theSearch.getRankingResult().getLocalResourceSize() + theSearch.getRankingResult().getRemoteResourceSize()) : theSearch.getRankingResult().getLocalIndexCount()) / theQuery.displayResults()));
+            final int numberofpages = Math.min(10,indexcount / theQuery.displayResults());
             for (int i = 0; i < numberofpages; i++) {
                 if (i == thispage) {
                     resnav.append("<img src=\"env/grafics/navs");
@@ -660,7 +664,7 @@ public class yacysearch {
             }
             String resnavs = resnav.toString();
             prop.put("num-results_resnav", resnavs);
-            prop.put("pageNavBottom", (theSearch.getRankingResult().getLocalIndexCount() - offset > 6) ? 1 : 0); // if there are more results than may fit on the page we add a navigation at the bottom
+            prop.put("pageNavBottom", (indexcount - offset > 6) ? 1 : 0); // if there are more results than may fit on the page we add a navigation at the bottom
             prop.put("pageNavBottom_resnav", resnavs);
         
             // generate the search result lines; the content will be produced by another servlet

htroot/yacysearchitem.java

@@ -87,12 +87,13 @@ public class yacysearchitem {
         final QueryParams theQuery = theSearch.getQuery();
         
         // dynamically update count values
-        final int totalcount = theSearch.getRankingResult().getLocalIndexCount() + theSearch.getRankingResult().getRemoteResourceSize();
+        final int totalcount = theSearch.getRankingResult().getLocalIndexCount() - theSearch.getRankingResult().getMissCount() + theSearch.getRankingResult().getRemoteResourceSize();
         final int offset = theQuery.neededResults() - theQuery.displayResults() + 1;
         prop.put("offset", offset);
         prop.put("itemscount", Formatter.number(Math.min((item < 0) ? theQuery.neededResults() : item + 1, totalcount)));
         prop.put("totalcount", Formatter.number(totalcount, true));
         prop.put("localResourceSize", Formatter.number(theSearch.getRankingResult().getLocalIndexCount(), true));
+        prop.put("localMissCount", Formatter.number(theSearch.getRankingResult().getMissCount(), true));
         prop.put("remoteResourceSize", Formatter.number(theSearch.getRankingResult().getRemoteResourceSize(), true));
         prop.put("remoteIndexCount", Formatter.number(theSearch.getRankingResult().getRemoteIndexCount(), true));
         prop.put("remotePeerCount", Formatter.number(theSearch.getRankingResult().getRemotePeerCount(), true));

htroot/yacysearchtrailer.java

@@ -169,7 +169,7 @@ public class yacysearchtrailer {
         String aboutBody = env.getConfig("about.body", "");
         String aboutHeadline = env.getConfig("about.headline", "");
         if ((aboutBody.length() == 0 && aboutHeadline.length() == 0) ||
-            theSearch.getRankingResult().getLocalIndexCount() +
+            theSearch.getRankingResult().getLocalIndexCount() - theSearch.getRankingResult().getMissCount() +
             theSearch.getRankingResult().getRemoteResourceSize() == 0) {
             prop.put("nav-about", 0);
         } else {

source/de/anomic/search/RankingProcess.java

@@ -569,6 +569,10 @@ public final class RankingProcess extends Thread {
         return this.misses.iterator();
     }
     
+    public int getMissCount() {
+        return this.misses.size();
+    }
+    
     public StaticScore<String> getNamespaceNavigator() {
         if (!this.query.navigators.equals("all") && this.query.navigators.indexOf("namespace") < 0) return new ScoreCluster<String>();
         if (this.namespaceNavigator.size() < 2) this.namespaceNavigator.clear(); // navigators with one entry are not useful

source/de/anomic/search/SearchEvent.java

@@ -197,7 +197,7 @@ public final class SearchEvent {
             }
             
             // start worker threads to fetch urls and snippets
-            this.resultFetcher = new ResultFetcher(loader, this.rankingProcess, query, peers, 300);
+            this.resultFetcher = new ResultFetcher(loader, this.rankingProcess, query, peers, 500);
         }
          
         // clean up events