enhanced strategy in host browser

limit number of fresh hosts in round robin hashes

source/net/yacy/crawler/HostBalancer.java

@@ -114,7 +114,7 @@ public class HostBalancer implements Balancer {
      * return immediately (as large unfinished crawls may take longer to load)
      */
     private void init(final boolean async) {
-    	if(async) {
+        if(async) {
             Thread t = new Thread("HostBalancer.init") {
                 @Override
                 public void run() {
@@ -123,9 +123,9 @@ public class HostBalancer implements Balancer {
             };
 
             t.start();            
-    	} else {
-    		runInit();
-    	}
+        } else {
+            runInit();
+        }
     }
 
     /**
@@ -230,11 +230,11 @@ public class HostBalancer implements Balancer {
         return c;
     }
 
-	/**
-	 * @return true when the URL is queued is this or any other HostBalancer
-	 *         instance (as {@link #depthCache} is shared between all HostBalancer
-	 *         instances)
-	 */
+    /**
+     * @return true when the URL is queued is this or any other HostBalancer
+     *         instance (as {@link #depthCache} is shared between all HostBalancer
+     *         instances)
+     */
     @Override
     public boolean has(final byte[] urlhashb) {
         if (depthCache.has(urlhashb)) return true;
@@ -331,14 +331,21 @@ public class HostBalancer implements Balancer {
                             if (size <= 10) {smallStacksExist = true; break smallsearch;}
                         }
                     }
-                    if (singletonStacksExist || smallStacksExist) {
-                        Iterator<String> i = this.roundRobinHostHashes.iterator();
-                        smallstacks: while (i.hasNext()) {
-                            if (this.roundRobinHostHashes.size() <= 10) break smallstacks; // don't shrink the hosts until nothing is left
-                            String s = i.next();
-                            HostQueue hq = this.queues.get(s);
-                            if (hq == null) {i.remove(); continue smallstacks;}
-                            int delta = Latency.waitingRemainingGuessed(hq.getHost(), hq.getPort(), s, robots, ClientIdentification.yacyInternetCrawlerAgent);
+                    Set<String> freshhosts = new HashSet<>();
+                    Iterator<String> i = this.roundRobinHostHashes.iterator();
+                    smallstacks: while (i.hasNext()) {
+                        if (this.roundRobinHostHashes.size() <= 10) break smallstacks; // don't shrink the hosts until nothing is left
+                        String hosthash = i.next();
+                        HostQueue hq = this.queues.get(hosthash);
+                        if (hq == null) {i.remove(); continue smallstacks;}
+                        int delta = Latency.waitingRemainingGuessed(hq.getHost(), hq.getPort(), hosthash, robots, ClientIdentification.yacyInternetCrawlerAgent);
+                        if (delta == Integer.MIN_VALUE) {
+                            // never-crawled hosts; we do not want to have too many of them in here. Loading new hosts means: waiting for robots.txt to load
+                            freshhosts.add(hosthash);
+                            i.remove();
+                            continue smallstacks;
+                        }
+                        if (singletonStacksExist || smallStacksExist) {
                             if (delta < 0) continue; // keep all non-waiting stacks; they are useful to speed up things
                             // to protect all small stacks which have a fast throughput, remove all with long waiting time
                             if (delta >= 1000) {i.remove(); continue smallstacks;}
@@ -350,6 +357,10 @@ public class HostBalancer implements Balancer {
                             }
                         }
                     }
+                    // put at least one of the fresh hosts back
+                    if (freshhosts.size() > 0) this.roundRobinHostHashes.add(freshhosts.iterator().next());
+
+                    // result
                     if (this.roundRobinHostHashes.size() == 1) {
                         if (log.isFine()) log.fine("(re-)initialized the round-robin queue with one host");
                     } else {
@@ -545,7 +556,7 @@ public class HostBalancer implements Balancer {
     @Override
     public List<Request> getDomainStackReferences(String host, int maxcount, long maxtime) {
         if (host == null) {
-        	return Collections.emptyList();
+            return Collections.emptyList();
         }
         try {
             HostQueue hq = this.queues.get(DigestURL.hosthash(host, host.startsWith("ftp.") ? 21 : 80));