bug fixes and speed enhancements for search

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@8085 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/yacysearch.java

@@ -629,8 +629,8 @@ public class yacysearch {
             // log
             Log.logInfo("LOCAL_SEARCH", "EXIT WORD SEARCH: " + theQuery.queryString + " - " +
                     "local-unfiltered(" + theSearch.getRankingResult().getLocalIndexCount() + "), " +
-                    "-local_miss(" + theSearch.getRankingResult().getMissCount() + "), " +
+                    "local_miss(" + theSearch.getRankingResult().getMissCount() + "), " +
-                    "-local_sortout(" + theSearch.getRankingResult().getSortOutCount() + "), " +
+                    "local_sortout(" + theSearch.getRankingResult().getSortOutCount() + "), " +
                     "remote(" + theSearch.getRankingResult().getRemoteResourceSize() + ") links found, " +
                     (System.currentTimeMillis() - timestamp) + " ms");
 

source/net/yacy/search/query/RWIProcess.java

@@ -153,11 +153,9 @@ public final class RWIProcess extends Thread {
             this.localSearchInclusion = search.inclusion();
             final ReferenceContainer<WordReference> index = search.joined();
             EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEvent.Type.JOIN, this.query.queryString, index.size(), System.currentTimeMillis() - timer), false);
-            if (index.isEmpty()) {
+            if (!index.isEmpty()) {
-                return;
+                add(index, true, "local index: " + this.query.getSegment().getLocation(), -1, true);
             }
-
-            add(index, true, "local index: " + this.query.getSegment().getLocation(), -1, true);
         } catch (final Exception e) {
             Log.logException(e);
         } finally {
@@ -299,7 +297,7 @@ public final class RWIProcess extends Thread {
     }
 
     public boolean feedingIsFinished() {
-    	return this.feeders.get() == 0;
+    	return this.feeders.get() <= 0;
     }
 
     private boolean testFlags(final WordReference ientry) {

source/net/yacy/search/query/SearchEvent.java

@@ -72,8 +72,8 @@ public final class SearchEvent {
     private QueryParams query;
     private final SeedDB peers;
     private final WorkTables workTables;
-    private RWIProcess rankingProcess; // ordered search results, grows dynamically as all the query threads enrich this container
+    private final RWIProcess rankingProcess; // ordered search results, grows dynamically as all the query threads enrich this container
-    private SnippetProcess resultFetcher;
+    private final SnippetProcess resultFetcher;
 
     private final SecondarySearchSuperviser secondarySearchSuperviser;
 
@@ -117,14 +117,15 @@ public final class SearchEvent {
         this.order = new ReferenceOrder(this.query.ranking, UTF8.getBytes(this.query.targetlang));
         final boolean remote = peers.sizeConnected() > 0 && (this.query.domType == QueryParams.Searchdom.CLUSTER || (this.query.domType == QueryParams.Searchdom.GLOBAL && peers.mySeed().getFlagAcceptRemoteIndex()));
         final long start = System.currentTimeMillis();
-        if (remote) {
-        	// initialize a ranking process that is the target for data
-        	// that is generated concurrently from local and global search threads
-            this.rankingProcess = new RWIProcess(this.query, this.order, max_results_preparation);
 
-            // start a local search concurrently
+        // initialize a ranking process that is the target for data
-            this.rankingProcess.start();
+        // that is generated concurrently from local and global search threads
+        this.rankingProcess = new RWIProcess(this.query, this.order, max_results_preparation);
+
+        // start a local search concurrently
+        this.rankingProcess.start();
 
+        if (remote) {
             // start global searches
             final long timer = System.currentTimeMillis();
             this.primarySearchThreads = (this.query.queryHashes.isEmpty()) ? null : RemoteSearch.primaryRemoteSearches(
@@ -159,15 +160,13 @@ public final class SearchEvent {
                 // no search since query is empty, user might have entered no data or filters have removed all search words
                 Log.logFine("SEARCH_EVENT", "NO SEARCH STARTED DUE TO EMPTY SEARCH REQUEST.");
             }
-
-            // start worker threads to fetch urls and snippets
-            this.resultFetcher = new SnippetProcess(loader, this.rankingProcess, this.query, this.peers, this.workTables, 3000, deleteIfSnippetFail);
         } else {
-            // do a local search
-            this.rankingProcess = new RWIProcess(this.query, this.order, max_results_preparation);
-
             if (generateAbstracts) {
-                this.rankingProcess.run(); // this is not started concurrently here on purpose!
+                // we need the results now
+                try {
+                    this.rankingProcess.join();
+                } catch (final Throwable e) {
+                }
                 // compute index abstracts
                 final long timer = System.currentTimeMillis();
                 int maxcount = -1;
@@ -193,21 +192,20 @@ public final class SearchEvent {
                 }
                 EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), Type.ABSTRACTS, "", this.rankingProcess.searchContainerMap().size(), System.currentTimeMillis() - timer), false);
             } else {
-                this.rankingProcess.start(); // start concurrently
+                // give process time to accumulate a certain amount of data
-                // but give process time to accumulate a certain amount of data
                 // before a reading process wants to get results from it
-                for (int i = 0; i < 10; i++) {
+                try {
-                    if (!this.rankingProcess.isAlive()) break;
+                    this.rankingProcess.join(100);
-                    try {Thread.sleep(10);} catch (final InterruptedException e) {}
+                } catch (final Throwable e) {
                 }
                 // this will reduce the maximum waiting time until results are available to 100 milliseconds
                 // while we always get a good set of ranked data
             }
-
-            // start worker threads to fetch urls and snippets
-            this.resultFetcher = new SnippetProcess(loader, this.rankingProcess, this.query, this.peers, this.workTables, 500, deleteIfSnippetFail);
         }
 
+        // start worker threads to fetch urls and snippets
+        this.resultFetcher = new SnippetProcess(loader, this.rankingProcess, this.query, this.peers, this.workTables, 5000, deleteIfSnippetFail);
+
         // clean up events
         SearchEventCache.cleanupEvents(false);
         EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), Type.CLEANUP, "", 0, 0), false);

source/net/yacy/search/query/SnippetProcess.java

@@ -165,7 +165,11 @@ public class SnippetProcess {
         // finally wait until enough results are there produced from the snippet fetch process
         WeakPriorityBlockingQueue.Element<ResultEntry> entry = null;
         while (System.currentTimeMillis() < finishTime) {
-            if (this.result.sizeAvailable() + this.rankingProcess.sizeQueue() <= item && this.rankingProcess.feedingIsFinished()) break; // the fail case
+
+            if (!anyWorkerAlive() && !this.rankingProcess.isAlive() && this.result.sizeAvailable() + this.rankingProcess.sizeQueue() <= item && this.rankingProcess.feedingIsFinished()) {
+                //Log.logInfo("SnippetProcess", "interrupted result fetching; item = " + item + "; this.result.sizeAvailable() = " + this.result.sizeAvailable() + ", this.rankingProcess.sizeQueue() = " + this.rankingProcess.sizeQueue());
+                break; // the fail case
+            }
 
             // deploy worker to get more results
             if (!anyWorkerAlive()) {
@@ -189,7 +193,7 @@ public class SnippetProcess {
 
     private int resultCounter = 0;
     public ResultEntry nextResult() {
-        final ResultEntry re = oneResult(this.resultCounter, 1000);
+        final ResultEntry re = oneResult(this.resultCounter, 3000);
         this.resultCounter++;
         return re;
     }
@@ -331,7 +335,7 @@ public class SnippetProcess {
             for (final Worker workerThread : this.workerThreads) {
                if ((workerThread != null) &&
                    (workerThread.isAlive()) &&
-                   (workerThread.busytime() < 1000)) return true;
+                   (workerThread.busytime() < 10000)) return true;
             }
         }
         return false;