enhanced snippet fetch strategy: concurrent snippet fetch even for offline-snippet searches. This improves speed since it is now possible to fetch snippets offline and parsing of source files from the htcache can be enhanced using concurrency. This improves local and remote search.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7156 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/search/RankingProcess.java

@@ -206,28 +206,30 @@ public final class RankingProcess extends Thread {
 			        continue;
 			    }
 			    
+                // count domZones
+                //this.domZones[DigestURI.domDomain(iEntry.metadataHash())]++;
+                
 			    // check site constraints
-			    if (query.sitehash != null && !new String(iEntry.metadataHash(), 6, 6).equals(query.sitehash)) {
-			        // filter out all domains that do not match with the site constraint
-			    	continue;
-			    }
-			    
-			    // count domZones
-			    //this.domZones[DigestURI.domDomain(iEntry.metadataHash())]++;
-			    
-			    // get statistics for host navigator
-			    if (nav_hosts && query.urlMask_isCatchall) {
-			        String uhb = new String(iEntry.urlHash);
-			        domhash = uhb.substring(6);
-			        this.hostNavigator.inc(domhash, uhb);
+			    domhash = new String(iEntry.metadataHash(), 6, 6);
+			    if (query.sitehash == null) {
+			        // no site constraint there; maybe collect host navigation information
+			        if (nav_hosts && query.urlMask_isCatchall) {
+	                    this.hostNavigator.inc(domhash, new String(iEntry.metadataHash()));
+	                }
+			    } else {
+			        if (!domhash.equals(query.sitehash)) {
+			            // filter out all domains that do not match with the site constraint
+			            continue;
+			        }
 			    }
 			    
-			    if (urlhashes.add(iEntry.metadataHash())) {
+			    // finally make a double-check and insert result to stack
+                if (urlhashes.add(iEntry.metadataHash())) {
                     stack.put(new ReverseElement<WordReferenceVars>(iEntry, this.order.cardinal(iEntry))); // inserts the element and removes the worst (which is smallest)
+
+                    // increase counter for statistics
+                    if (local) this.local_indexCount++; else this.remote_indexCount++;
                 }
-			    
-			    // increase counter for statistics
-			    if (local) this.local_indexCount++; else this.remote_indexCount++;
 			}
 
         } catch (InterruptedException e) {}

source/de/anomic/search/ReferenceOrder.java

@@ -74,7 +74,7 @@ public class ReferenceOrder {
         return out;
     }
     
-    public class NormalizeDistributor extends Thread {
+    private final class NormalizeDistributor extends Thread {
 
         ReferenceContainer<WordReference> container;
         LinkedBlockingQueue<WordReferenceVars> out;
@@ -118,7 +118,7 @@ public class ReferenceOrder {
     /**
      * normalize ranking: find minimum and maximum of separate ranking criteria
      */
-    public class NormalizeWorker extends Thread {
+    private class NormalizeWorker extends Thread {
         
         private final BlockingQueue<WordReferenceVars> out;
         private final Semaphore termination;
@@ -145,10 +145,10 @@ public class ReferenceOrder {
                 Integer count;
                 final Integer int1 = 1;
                 while ((iEntry = decodedEntries.take()) != WordReferenceVars.poison) {
-                    out.put(iEntry);
                     // find min/max
                     if (min == null) min = iEntry.clone(); else min.min(iEntry);
                     if (max == null) max = iEntry.clone(); else max.max(iEntry);
+                    out.put(iEntry); // must be after the min/max check to prevent that min/max is null in cardinal()
                     // update domcount
                     dom = new String(iEntry.metadataHash(), 6, 6);
                     count = doms0.get(dom);

source/de/anomic/search/ResultFetcher.java

@@ -106,7 +106,7 @@ public class ResultFetcher {
 
     public void deployWorker(int deployCount, int neededResults) {
     	if (anyWorkerAlive()) return;
-    	this.workerThreads = new Worker[(query.snippetCacheStrategy.isAllowedToFetchOnline()) ? deployCount : 1];
+    	this.workerThreads = new Worker[/*(query.snippetCacheStrategy.mustBeOffline()) ? 1 : */deployCount];
     	for (int i = 0; i < workerThreads.length; i++) {
     		this.workerThreads[i] = new Worker(i, 10000, query.snippetCacheStrategy, neededResults);
     		this.workerThreads[i].start();