- more control in access tracker: count number of returned search results (not only info how much is in the index)

- extended query params for this
- enhanced cora

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7430 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/AccessTracker_p.html

@@ -75,6 +75,7 @@
         <td>Offset</td>
         <td>Expected Results</td>
         <td>Returned Results</td>
+        <td>Known Results</td>
         <td>Used Time (ms)</td>
         <td>URL fetch (ms)</td>
         <td>Snippet comp (ms)</td>
@@ -88,6 +89,7 @@
         <td></td>
         <td></td>
         <td>&#216; #[querycount_avg]#</td>
+        <td>&#216; #[transmitcount_avg]#</td>
         <td>&#216; #[resultcount_avg]#</td>
         <td>&#216; #[resulttime_avg]#</td>
         <td>&#216; #[urltime_avg]#</td>
@@ -103,6 +105,7 @@
         <td>#[date]#</td>
         <td>#[offset]#</td>
         <td>#[querycount]#</td>
+        <td>#[transmitcount]#</td>
         <td>#[resultcount]#</td>
         <td>#[resulttime]#</td>
         <td>#[urltime]#</td>

htroot/AccessTracker_p.java

@@ -144,6 +144,7 @@ public class AccessTracker_p {
             QueryParams query;
             long qcountSum = 0;
             long rcountSum = 0;
+            long tcountSum = 0;
             long rcount = 0;
             long utimeSum = 0;
             long stimeSum = 0;
@@ -175,6 +176,7 @@ public class AccessTracker_p {
                     prop.put("page_list_" + m + "_queryhashes", QueryParams.anonymizedQueryHashes(query.queryHashes));
                 }
                 prop.putNum("page_list_" + m + "_querycount", query.itemsPerPage);
+                prop.putNum("page_list_" + m + "_transmitcount", query.transmitcount);
                 prop.putNum("page_list_" + m + "_resultcount", query.resultcount);
                 prop.putNum("page_list_" + m + "_urltime", query.urlretrievaltime);
                 prop.putNum("page_list_" + m + "_snippettime", query.snippetcomputationtime);
@@ -182,6 +184,7 @@ public class AccessTracker_p {
                 prop.putHTML("page_list_" + m + "_userAgent", query.userAgent);
                 qcountSum += query.itemsPerPage;
                 rcountSum += query.resultcount;
+                tcountSum += query.transmitcount;
                 utimeSum += query.urlretrievaltime;
                 stimeSum += query.snippetcomputationtime;
                 rtimeSum += query.searchtime;
@@ -203,24 +206,25 @@ public class AccessTracker_p {
                 // return empty values to not break the table view
                 prop.put("page_list", 1);
                 prop.put("page_list_0_dark", 1 );
-                prop.put("page_list_0_host", " ");
+                prop.put("page_list_0_host", "");
-                prop.put("page_list_0_date", " ");
+                prop.put("page_list_0_date", "");
-                prop.put("page_list_0_timestamp", " ");
+                prop.put("page_list_0_timestamp", "");
                 if (page == 2) {
                     // local search
                     prop.putNum("page_list_0_offset", "");
                     prop.put("page_list_0_querystring", "");
                 } else {
                     // remote search
-                    prop.put("page_list_0_peername", " ");
+                    prop.put("page_list_0_peername", "");
-                    prop.put("page_list_0_queryhashes", " ");
+                    prop.put("page_list_0_queryhashes", "");
                 }
                 prop.putNum("page_list_0_querycount", "");
+                prop.putNum("page_list_0_transmitcount", "");
                 prop.putNum("page_list_0_resultcount", "");
                 prop.putNum("page_list_0_urltime", "");
                 prop.putNum("page_list_0_snippettime", "");
                 prop.putNum("page_list_0_resulttime", "");
-                prop.put("page_list_0_userAgent", " ");
+                prop.put("page_list_0_userAgent", "");
             }
             if (rcount == 0) rcount = -1;
             prop.putNum("page_querycount_avg", (double) qcountSum / m);
@@ -228,6 +232,7 @@ public class AccessTracker_p {
             prop.putNum("page_urltime_avg", (double) utimeSum / m);
             prop.putNum("page_snippettime_avg", (double) stimeSum / m);
             prop.putNum("page_resulttime_avg", (double) rtimeSum / m);
+            prop.putNum("page_transmitcount_avg", (double) tcountSum / rcount);
             prop.putNum("page_resultcount_avg1", (double) rcountSum / rcount);
             prop.putNum("page_urltime_avg1", (double) utimeSum1 / rcount);
             prop.putNum("page_snippettime_avg1", (double) stimeSum1 / rcount);
@@ -274,12 +279,12 @@ public class AccessTracker_p {
             // return empty values to not break the table view if no results can be listed
             if (m==0) {                
                 prop.put("page_list", 1);
-                prop.put("page_list_0_dates_0_date", " ");
+                prop.put("page_list_0_dates_0_date", "");
                 prop.put("page_list_0_dates", 1);
                 prop.putNum("page_list_0_qph", "");
                 prop.put("page_list_0_dark", 1 );
-                prop.put("page_list_0_peername", " ");
+                prop.put("page_list_0_peername", "");
-                prop.put("page_list_0_host", " ");
+                prop.put("page_list_0_host", "");
                 prop.putNum("page_list_0_count", "");
             } else {
                 prop.put("page_list", m);

htroot/AccessTracker_p.xml

@@ -25,6 +25,7 @@
         <date timestamp="#[timestamp]#">#[date]#</date>
         <offset>#[offset]#</offset>
         <querycount>#[querycount]#</querycount>
+        <transmitcount>#[transmitcount]#</transmitcount>
         <resultcount>#[resultcount]#</resultcount>
         <resulttime>#[resulttime]#</resulttime>
         <urltime>#[urltime]#</urltime>

htroot/yacy/search.java

@@ -382,6 +382,7 @@ public final class search {
                     links.append("resource").append(i).append('=').append(resource).append(serverCore.CRLF_STRING);
                 }
             }
+            theQuery.transmitcount = accu.size() + 1;
             prop.put("links", links.toString());
             prop.put("linkcount", accu.size());
             EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.Type.RESULTLIST, "", accu.size(), System.currentTimeMillis() - timer), false);

htroot/yacysearch.rss

@@ -22,11 +22,9 @@
 		<opensearch:itemsPerPage>#[num-results_itemsPerPage]#</opensearch:itemsPerPage>
 		<atom:link rel="related" href="opensearchdescription.xml" type="application/opensearchdescription+xml"/>
 		<opensearch:Query role="request" searchTerms="#[rss_queryenc]#" />
-		
 #{results}#
 <!--#include virtual="yacysearchitem.xml?item=#[item]#&eventID=#[eventID]#" -->
 #{/results}#
-		
 <!--#include virtual="yacysearchtrailer.xml?eventID=#[eventID]#" -->
 	</channel>
 </rss>

htroot/yacysearch_location.java

@@ -93,7 +93,7 @@ public class yacysearch_location {
                 // get a queue of search results
                 String rssSearchServiceURL = "http://localhost:" + sb.getConfig("port", "8080") + "/yacysearch.rss";
                 BlockingQueue<RSSMessage> results = new LinkedBlockingQueue<RSSMessage>();
-                SearchSRURSS.searchSRURSS(results, rssSearchServiceURL, query, maximumTime, Integer.MAX_VALUE, false, false);
+                SearchSRURSS.searchSRURSS(results, rssSearchServiceURL, query, maximumTime, Integer.MAX_VALUE, false, false, null);
                 
                 // take the results and compute some locations
                 RSSMessage message;

htroot/yacysearchitem.java

@@ -172,7 +172,7 @@ public class yacysearchitem {
             } else {
                 prop.put("content_code", "");
             }
-            
+            theQuery.transmitcount = item + 1;
             return prop;
         }
         
@@ -203,6 +203,7 @@ public class yacysearchitem {
                 prop.put("content_item_nl", (item == 0) ? 0 : 1);
                 prop.put("content_item", 1);
             }
+            theQuery.transmitcount = item + 1;
             return prop;
         }
         
@@ -232,6 +233,7 @@ public class yacysearchitem {
             } else {
                 prop.put("content_items", "0");
             }
+            theQuery.transmitcount = item + 1;
             return prop;
         }
         

source/de/anomic/search/QueryParams.java

@@ -99,6 +99,7 @@ public final class QueryParams {
     public final Long time;
     // values that are set after a search:
     public int resultcount; // number of found results
+    public int transmitcount; // number of results that had been shown to the user
     public long searchtime, urlretrievaltime, snippetcomputationtime; // time to perform the search, to get all the urls, and to compute the snippets
     public boolean specialRights; // is true if the user has a special authorization and my use more database-extensive options
     public final String userAgent;
@@ -152,6 +153,7 @@ public final class QueryParams {
         this.navigators = "all";
         this.indexSegment = indexSegment;
         this.userAgent = userAgent;
+        this.transmitcount = 0;
     }
     
     public QueryParams(
@@ -206,6 +208,7 @@ public final class QueryParams {
         this.specialRights = specialRights;
         this.indexSegment = indexSegment;
         this.userAgent = userAgent;
+        this.transmitcount = 0;
     }
     
     public Segment getSegment() {

source/de/anomic/yacy/yacyClient.java

@@ -372,7 +372,7 @@ public final class yacyClient {
     public static RSSFeed search(final yacySeed targetSeed, String query, boolean verify, boolean global, long timeout, int startRecord, int maximumRecords) throws IOException {
         String address = (targetSeed == null || targetSeed == Switchboard.getSwitchboard().peers.mySeed()) ? "localhost:" + Switchboard.getSwitchboard().getConfig("port", "8080") : targetSeed.getClusterAddress();
         String urlBase = "http://" + address + "/yacysearch.rss";
-        return SearchSRURSS.loadSRURSS(urlBase, query, timeout, startRecord, maximumRecords, verify, global);
+        return SearchSRURSS.loadSRURSS(urlBase, query, timeout, startRecord, maximumRecords, verify, global, null);
     }
 
     @SuppressWarnings("unchecked")

source/net/yacy/cora/protocol/http/HTTPClient.java

@@ -114,7 +114,7 @@ public class HTTPClient {
     	HttpProtocolParams.setUserAgent(httpClient.getParams(), defaultAgent);
     }
     
-    private static HttpClient initConnectionManager() {
+    public static HttpClient initConnectionManager() {
 		// Create and initialize HTTP parameters
 		final HttpParams httpParams = new BasicHttpParams();
 		/**

source/net/yacy/cora/services/SearchHub.java

@@ -33,10 +33,13 @@ import net.yacy.cora.storage.ScoreMap;
 public class SearchHub {
 
     private static final String[] SRURSSServicesList = {
+        //"http://192.168.1.51:8000/yacysearch.rss"//,
+        "http://localhost:8008/yacysearch.rss"//,
+        /*
         "http://yacy.dyndns.org:8000/yacysearch.rss",
         "http://yacy.caloulinux.net:8085/yacysearch.rss",
         "http://algire.dyndns.org:8085/yacysearch.rss",
-        "http://breyvogel.dyndns.org:8002/yacysearch.rss"
+        "http://breyvogel.dyndns.org:8002/yacysearch.rss"*/
     };
 
     public final static SearchHub EMPTY = new SearchHub("", 0);
@@ -136,20 +139,23 @@ public class SearchHub {
      * @param verify
      * @param global
      */
-    public static void addSRURSSServices(SearchHub search, String[] rssServices, int count, boolean verify, boolean global) {
+    public static void addSRURSSServices(SearchHub search, String[] rssServices, int count, boolean verify, boolean global, String userAgent) {
         for (String service: rssServices) {
-            SearchSRURSS accumulator = new SearchSRURSS(search, service, count, verify, global);
+            SearchSRURSS accumulator = new SearchSRURSS(search, service, count, verify, global, userAgent);
             accumulator.start();
             search.addAccumulator(accumulator);
         }
     }
     
     public static void main(String[] args) {
+        HTTPClient.setDefaultUserAgent("searchhub");
+        HTTPClient.initConnectionManager();
+        
         StringBuilder sb = new StringBuilder();
         for (String s: args) sb.append(s).append(' ');
         String query = sb.toString().trim();
         SearchHub search = new SearchHub(query, 10000);
-        addSRURSSServices(search, SRURSSServicesList, 100, false, false);
+        addSRURSSServices(search, SRURSSServicesList, 100, false, false, "searchhub");
         try {Thread.sleep(100);} catch (InterruptedException e1) {}
         search.waitTermination();
         ScoreMap<String> result = search.getResults();
@@ -159,6 +165,6 @@ public class SearchHub {
             u = i.next();
             System.out.println("[" + result.get(u) + "] " + u);
         }
-        try {HTTPClient.closeConnectionManager();} catch (InterruptedException e) {}
+        try {HTTPClient.closeConnectionManager();} catch (InterruptedException e) { e.printStackTrace(); }
     }
 }

source/net/yacy/cora/services/SearchSRURSS.java

@@ -50,6 +50,7 @@ public class SearchSRURSS extends Thread implements SearchAccumulator {
     final boolean verify;
     final boolean global;
     final Map<RSSMessage, List<Integer>> result;
+    final String userAgent;
 
     private final BlockingQueue<RSSMessage> results;
     
@@ -60,7 +61,8 @@ public class SearchSRURSS extends Thread implements SearchAccumulator {
             final String urlBase,
             final int maximumRecordsInit,
             final boolean verify,
-            final boolean global) {
+            final boolean global,
+            final String userAgent) {
         this.results = new LinkedBlockingQueue<RSSMessage>();
         this.result = result;
         this.query = query;
@@ -69,6 +71,7 @@ public class SearchSRURSS extends Thread implements SearchAccumulator {
         this.maximumRecordsInit = maximumRecordsInit;
         this.verify = verify;
         this.global = global;
+        this.userAgent = userAgent;
     }
     
     public SearchSRURSS(
@@ -76,7 +79,8 @@ public class SearchSRURSS extends Thread implements SearchAccumulator {
             final String urlBase,
             final int maximumRecordsInit,
             final boolean verify,
-            final boolean global) {
+            final boolean global,
+            final String userAgent) {
         this.results = new LinkedBlockingQueue<RSSMessage>();
         this.result = search.getAccumulation();
         this.query = search.getQuery();
@@ -85,10 +89,11 @@ public class SearchSRURSS extends Thread implements SearchAccumulator {
         this.maximumRecordsInit = maximumRecordsInit;
         this.verify = verify;
         this.global = global;
+        this.userAgent = userAgent;
     }
     
     public void run() {
-        searchSRURSS(results, urlBase, query, timeoutInit, maximumRecordsInit, verify, global);
+        searchSRURSS(results, urlBase, query, timeoutInit, maximumRecordsInit, verify, global, userAgent);
         int p = 1;
         RSSMessage message;
         try {
@@ -111,7 +116,8 @@ public class SearchSRURSS extends Thread implements SearchAccumulator {
             final long timeoutInit,
             final int maximumRecordsInit,
             final boolean verify,
-            final boolean global) {
+            final boolean global,
+            final String userAgent) {
         Thread job = new Thread() {
             public void run() {
                 int startRecord = 0;
@@ -122,8 +128,9 @@ public class SearchSRURSS extends Thread implements SearchAccumulator {
                     long st = System.currentTimeMillis();
                     RSSFeed feed;
                     try {
-                        feed = loadSRURSS(urlBase, query, timeout, startRecord, recordsPerSession, verify, global);
+                        feed = loadSRURSS(urlBase, query, timeout, startRecord, recordsPerSession, verify, global, userAgent);
                     } catch (IOException e1) {
+                        e1.printStackTrace();
                         break mainloop;
                     }
                     if (feed == null || feed.isEmpty()) break mainloop;
@@ -134,13 +141,14 @@ public class SearchSRURSS extends Thread implements SearchAccumulator {
                         try {
                             queue.put(message);
                         } catch (InterruptedException e) {
+                            e.printStackTrace(); 
                             break innerloop;
                         }
                     }
                     startRecord += recordsPerSession;
                     timeout -= System.currentTimeMillis() - st;
                 }
-                try { queue.put(RSSMessage.POISON); } catch (InterruptedException e) {}
+                try { queue.put(RSSMessage.POISON); } catch (InterruptedException e) { e.printStackTrace(); }
             }
         };
         job.start();
@@ -165,7 +173,8 @@ public class SearchSRURSS extends Thread implements SearchAccumulator {
             int startRecord,
             int maximumRecords,
             boolean verify,
-            boolean global) throws IOException {
+            boolean global,
+            String userAgent) throws IOException {
         MultiProtocolURI uri = null;
         try {
             uri = new MultiProtocolURI(rssSearchServiceURL);
@@ -181,8 +190,9 @@ public class SearchSRURSS extends Thread implements SearchAccumulator {
             parts.put("maximumRecords", new StringBody(Long.toString(maximumRecords)));
             parts.put("verify", new StringBody(verify ? "true" : "false"));
             parts.put("resource", new StringBody(global ? "global" : "local"));
-            final byte[] result = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI(rssSearchServiceURL), (int) timeout, uri.getHost(), parts);
+            parts.put("nav", new StringBody("none"));
-            //String debug = new String(result); System.out.println("*** DEBUG: " + debug);
+            final byte[] result = HTTPConnector.getConnector(userAgent == null ? MultiProtocolURI.yacybotUserAgent : userAgent).post(new MultiProtocolURI(rssSearchServiceURL), (int) timeout, uri.getHost(), parts);
+            String debug = new String(result); System.out.println("*** DEBUG: " + debug);
             final RSSReader reader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result);
             if (reader == null) {
                 throw new IOException("cora.Search failed asking peer '" + uri.getHost() + "': probably bad response from remote peer (1), reader == null");

source/net/yacy/repository/LoaderDispatcher.java

@@ -265,7 +265,7 @@ public final class LoaderDispatcher {
         if (protocol.equals("ftp")) response = ftpLoader.load(request, true);
         if (protocol.equals("smb")) response = smbLoader.load(request, true);
         if (protocol.equals("file")) response = fileLoader.load(request, true);
-        if (response != null) {
+        if (response != null && response.getContent() != null) {
             // we got something. Now check if we want to store that to the cache
             // first check looks if we want to store the content to the cache
             if (!crawlProfile.storeHTCache()) {