enhancement during search process: fast fail of search in case that all index feeder have terminated.

This change should affect filtering and navigators and should cause that search navigation gets faster

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7614 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/search/RankingProcess.java

@@ -91,11 +91,13 @@ public final class RankingProcess extends Thread {
     private final ScoreMap<String> namespaceNavigator;
     private final ReferenceOrder order;
     private final long startTime;
+    private       boolean addRunning;
     
     public RankingProcess(final QueryParams query, final ReferenceOrder order, final int maxentries) {
         // we collect the urlhashes and construct a list with urlEntry objects
         // attention: if minEntries is too high, this method will not terminate within the maxTime
         // sortorder: 0 = hash, 1 = url, 2 = ranking
+        this.addRunning = true;
         this.localSearchInclusion = null;
         this.stack = new WeakPriorityBlockingQueue<WordReferenceVars>(maxentries);
         this.doubleDomCache = new ConcurrentHashMap<String, WeakPriorityBlockingQueue<WordReferenceVars>>();
@@ -150,7 +152,7 @@ public final class RankingProcess extends Thread {
                 return;
             }
             
-            add(index, true, "local index: " + this.query.getSegment().getLocation(), -1);
+            add(index, true, "local index: " + this.query.getSegment().getLocation(), -1, true);
         } catch (final Exception e) {
             Log.logException(e);
         } finally {
@@ -158,10 +160,17 @@ public final class RankingProcess extends Thread {
         }
     }
     
-    public void add(final ReferenceContainer<WordReference> index, final boolean local, String resourceName, final int fullResource) {
+    public void add(
+            final ReferenceContainer<WordReference> index,
+            final boolean local,
+            String resourceName,
+            final int fullResource,
+            final boolean finalizeAddAtEnd) {
         // we collect the urlhashes and construct a list with urlEntry objects
         // attention: if minEntries is too high, this method will not terminate within the maxTime
 
+        this.addRunning = true;
+        
         assert (index != null);
         if (index.isEmpty()) return;
         
@@ -245,7 +254,9 @@ public final class RankingProcess extends Thread {
                 }
             }
 
-        } catch (InterruptedException e) {}
+        } catch (InterruptedException e) {} finally {
+            if (finalizeAddAtEnd) this.addRunning = false;
+        }
         
         //if ((query.neededResults() > 0) && (container.size() > query.neededResults())) remove(true, true);
         EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(query.id(true), SearchEvent.Type.PRESORT, resourceName, index.size(), System.currentTimeMillis() - timer), false);
@@ -297,11 +308,12 @@ public final class RankingProcess extends Thread {
         WeakPriorityBlockingQueue.Element<WordReferenceVars> rwi = null;
         
         // take one entry from the stack if there are entries on that stack or the feeding is not yet finished
-        if (!feedingIsFinished() || stack.sizeQueue() > 0) try {
+        try {
             //System.out.println("stack.poll: feeders = " + this.feeders + ", stack.sizeQueue = " + stack.sizeQueue());
             int loops = 0; // a loop counter to terminate the reading if all the results are from the same domain
             long timeout = System.currentTimeMillis() + waitingtime;
-            while (this.query.itemsPerPage < 1 || loops++ < this.query.itemsPerPage) {
+            while (((!feedingIsFinished() && this.addRunning) || stack.sizeQueue() > 0) &&
+                   (this.query.itemsPerPage < 1 || loops++ < this.query.itemsPerPage)) {
                 if (waitingtime <= 0) {
                     rwi = stack.poll();
                 } else timeoutloop:while (System.currentTimeMillis() < timeout) {

source/de/anomic/search/ResultFetcher.java

@@ -319,13 +319,13 @@ public class ResultFetcher {
 
                     // check if we have enough
                     if (result.sizeAvailable() >= this.neededResults) {
-                        System.out.println(result.sizeAvailable() + " = result.sizeAvailable() >= this.neededResults = " + this.neededResults);
+                        Log.logWarning("ResultFetcher", result.sizeAvailable() + " = result.sizeAvailable() >= this.neededResults = " + this.neededResults);
                         break;
                     }
 
                     // check if we can succeed if we try to take another url
                     if (rankingProcess.feedingIsFinished() && rankingProcess.sizeQueue() == 0) {
-                        System.out.println("rankingProcess.feedingIsFinished() && rankingProcess.sizeQueue() == 0");
+                        Log.logWarning("ResultFetcher", "rankingProcess.feedingIsFinished() && rankingProcess.sizeQueue() == 0");
                         break;
                     }
     

source/de/anomic/search/Segment.java

@@ -247,7 +247,7 @@ public class Segment {
                 try {
                     container = ReferenceContainer.emptyContainer(Segment.wordReferenceFactory, wordhash, 1);
                     container.add(ientry);
-                    rankingProcess.add(container, true, sourceName, -1);
+                    rankingProcess.add(container, true, sourceName, -1, !i.hasNext());
                 } catch (RowSpaceExceededException e) {
                     continue;
                 }

source/de/anomic/yacy/yacyClient.java

@@ -506,7 +506,7 @@ public final class yacyClient {
         // store remote result to local result container
         // insert one container into the search result buffer
         // one is enough, only the references are used, not the word
-        containerCache.add(container[0], false, target.getName() + "/" + target.hash, result.joincount);
+        containerCache.add(container[0], false, target.getName() + "/" + target.hash, result.joincount, true);
 
         // insert the containers to the index
         for (ReferenceContainer<WordReference> c: container) try {

source/net/yacy/cora/protocol/Domains.java

@@ -813,10 +813,15 @@ public class Domains {
             System.err.println(e);
         }
         */
+        InetAddress a;
+        a = dnsResolve("yacy.net"); System.out.println(a);
+        a = dnsResolve("kaskelix.de"); System.out.println(a);
+        a = dnsResolve("yacy.net"); System.out.println(a);
+        
         try { Thread.sleep(1000);} catch (InterruptedException e) {} // get time for class init
         System.out.println("myPublicLocalIP: " + myPublicLocalIP());
-        for (final InetAddress a : myIntranetIPs()) {
+        for (final InetAddress b : myIntranetIPs()) {
-            System.out.println("Intranet IP: " + a);
+            System.out.println("Intranet IP: " + b);
         }
     }
 }

source/net/yacy/document/parser/html/ContentScraper.java

@@ -39,6 +39,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
+import java.util.regex.Pattern;
 
 import javax.swing.event.EventListenerList;
 
@@ -426,6 +427,9 @@ public class ContentScraper extends AbstractScraper implements Scraper {
         return s;
     }
 
+    private final static Pattern commaSepPattern = Pattern.compile(" |,");
+    private final static Pattern semicSepPattern = Pattern.compile(" |;");
+    
     public Set<String> getContentLanguages() {
         // i.e. <meta name="DC.language" content="en" scheme="DCTERMS.RFC3066">
         // or <meta http-equiv="content-language" content="en">
@@ -433,7 +437,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {
         if (s == null) s = metas.get("dc.language");
         if (s == null) return null;
         Set<String> hs = new HashSet<String>();
-        String[] cl = s.split(" |,");
+        String[] cl = commaSepPattern.split(s);
         int p;
         for (int i = 0; i < cl.length; i++) {
             cl[i] = cl[i].toLowerCase();
@@ -452,8 +456,8 @@ public class ContentScraper extends AbstractScraper implements Scraper {
         if (s.length() == 0) {
             return MultiProtocolURI.splitpattern.split(getTitle().toLowerCase());
         }
-        if (s.contains(",")) return s.split(" |,");
+        if (s.contains(",")) return commaSepPattern.split(s);
-        if (s.contains(";")) return s.split(" |;");
+        if (s.contains(";")) return semicSepPattern.split(s);
         return s.split("\\s");
     }
     

source/net/yacy/kelondro/index/Row.java

@@ -34,6 +34,7 @@ import java.util.Map;
 import java.util.StringTokenizer;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Pattern;
 
 import net.yacy.cora.document.UTF8;
 import net.yacy.kelondro.logging.Log;
@@ -49,6 +50,8 @@ import net.yacy.kelondro.util.kelondroException;
 
 public final class Row {
 
+    private final static Pattern commaPattern = Pattern.compile(",");
+    
     protected final Column[]        row;
     public final int[]              colstart;
     public final ByteOrder          objectOrder;
@@ -281,7 +284,7 @@ public final class Row {
         public Entry(String external, final boolean decimalCardinal) {
             // parse external form
             if (external.length() > 0 && external.charAt(0) == '{') external = external.substring(1, external.length() - 1);
-            final String[] elts = external.split(",");
+            final String[] elts = commaPattern.split(external);
             if (nickref == null) genNickRef();
             String nick;
             int p;