much better timing when search globally; less blocking; more results earlier!

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@8084 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/yacy/search.java

@@ -93,6 +93,10 @@ public final class search {
         prop.put("links", "");
         prop.put("indexcount", "");
         prop.put("indexabstract", "");
+        prop.put("fwhop", ""); // hops (depth) of forwards that had been performed to construct this result
+        prop.put("fwsrc", ""); // peers that helped to construct this result
+        prop.put("fwrec", ""); // peers that would have helped to construct this result (recommendations)
+
 
         if (post == null || env == null) return prop;
         if (!Protocol.authentifyRequest(post, env)) return prop;
@@ -408,11 +412,6 @@ public final class search {
             EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEvent.Type.RESULTLIST, "", accu.size(), System.currentTimeMillis() - timer), false);
         }
 
-        // add information about forward peers
-        prop.put("fwhop", ""); // hops (depth) of forwards that had been performed to construct this result
-        prop.put("fwsrc", ""); // peers that helped to construct this result
-        prop.put("fwrec", ""); // peers that would have helped to construct this result (recommendations)
-
         // prepare search statistics
         theQuery.remotepeer = client == null ? null : sb.peers.lookupByIP(Domains.dnsResolve(client), true, false, false);
         theQuery.resultcount = (theSearch == null) ? 0 : joincount;

source/net/yacy/search/query/RWIProcess.java

@@ -173,6 +173,7 @@ public final class RWIProcess extends Thread {
             final boolean finalizeAddAtEnd) {
         // we collect the urlhashes and construct a list with urlEntry objects
         // attention: if minEntries is too high, this method will not terminate within the maxTime
+        //Log.logInfo("RWIProcess", "added a container, size = " + index.size());
 
         this.addRunning = true;
 

source/net/yacy/search/query/SearchEvent.java

@@ -371,15 +371,17 @@ public final class SearchEvent {
     }
 
     public ResultEntry oneResult(final int item, final long timeout) {
-        if ((this.query.domType == QueryParams.Searchdom.GLOBAL) || (this.query.domType == QueryParams.Searchdom.CLUSTER)) {
+        /*
+        if (this.query.domType == QueryParams.Searchdom.GLOBAL || this.query.domType == QueryParams.Searchdom.CLUSTER) {
             // this is a search using remote search threads. Also the local
             // search thread is started as background process
-            if ((this.localSearchThread != null) && (this.localSearchThread.isAlive())) {
+            if (this.localSearchThread != null && this.localSearchThread.isAlive()) {
                 // in case that the local search takes longer than some other
                 // remote search requests, wait that the local process terminates first
             	try {this.localSearchThread.join(300);} catch (final InterruptedException e) {}
             }
         }
+        */
         return this.resultFetcher.oneResult(item, timeout);
     }
 

source/net/yacy/search/query/SnippetProcess.java

@@ -145,10 +145,9 @@ public class SnippetProcess {
 
         // we must wait some time until the first result page is full to get enough elements for ranking
         final long waittimeout = System.currentTimeMillis() + 300;
-        while (
-          item == 0 &&
+        if (item == 0) while (
           (!this.rankingProcess.feedingIsFinished() || this.rankingProcess.sizeQueue() > 0) &&
-          this.result.sizeAvailable() < this.query.neededResults() /* + this.query.itemsPerPage */ &&
+          this.result.sizeAvailable() < this.query.neededResults() &&
           System.currentTimeMillis() < waittimeout &&
           anyWorkerAlive()
           ) {
@@ -163,17 +162,19 @@ public class SnippetProcess {
             return re;
         }
 
-        // deploy worker to get more results
-        final int neededInclPrefetch = this.query.neededResults() + ((MemoryControl.available() > 100 * 1024 * 1024) ? this.query.itemsPerPage : 0);
-        deployWorker(Math.min(20, this.query.itemsPerPage), neededInclPrefetch);
-
         // finally wait until enough results are there produced from the snippet fetch process
         WeakPriorityBlockingQueue.Element<ResultEntry> entry = null;
         while (System.currentTimeMillis() < finishTime) {
-            if (this.result.sizeAvailable() + this.rankingProcess.sizeQueue() <= item  && !anyWorkerAlive() && this.rankingProcess.feedingIsFinished()) break;
+            if (this.result.sizeAvailable() + this.rankingProcess.sizeQueue() <= item && this.rankingProcess.feedingIsFinished()) break; // the fail case
+
+            // deploy worker to get more results
+            if (!anyWorkerAlive()) {
+                final int neededInclPrefetch = this.query.neededResults() + ((MemoryControl.available() > 100 * 1024 * 1024) ? this.query.itemsPerPage : 0);
+                deployWorker(Math.min(20, this.query.itemsPerPage), neededInclPrefetch);
+            }
+
             try {entry = this.result.element(item, 50);} catch (final InterruptedException e) {break;}
             if (entry != null) break;
-            //if (!anyWorkerAlive() && this.rankingProcess.sizeQueue() == 0 && this.rankingProcess.feedingIsFinished()) break;
         }
 
         // finally, if there is something, return the result
@@ -390,6 +391,7 @@ public class SnippetProcess {
 
                     // get next entry
                     page = SnippetProcess.this.rankingProcess.takeURL(true, Math.min(100, this.timeout - System.currentTimeMillis()));
+                    //if (page != null) Log.logInfo("ResultFetcher", "got one page: " + page.metadata().url().toNormalform(true, false));
                     //if (page == null) page = rankedCache.takeURL(false, this.timeout - System.currentTimeMillis());
                     if (page == null) {
                         //Log.logWarning("ResultFetcher", "page == null");
@@ -429,7 +431,7 @@ public class SnippetProcess {
             } catch (final Exception e) {
                 Log.logException(e);
             }
-            Log.logInfo("SEARCH", "resultWorker thread " + this.id + " terminated");
+            //Log.logInfo("SEARCH", "resultWorker thread " + this.id + " terminated");
         }
 
         public void pleaseStop() {