enhanced strategy in host browser

limit number of fresh hosts in round robin hashes

source/net/yacy/crawler/HostBalancer.java

@@ -331,14 +331,21 @@ public class HostBalancer implements Balancer {
                             if (size <= 10) {smallStacksExist = true; break smallsearch;}
                         }
                     }
-                    if (singletonStacksExist || smallStacksExist) {
+                    Set<String> freshhosts = new HashSet<>();
                     Iterator<String> i = this.roundRobinHostHashes.iterator();
                     smallstacks: while (i.hasNext()) {
                         if (this.roundRobinHostHashes.size() <= 10) break smallstacks; // don't shrink the hosts until nothing is left
-                            String s = i.next();
-                            HostQueue hq = this.queues.get(s);
+                        String hosthash = i.next();
+                        HostQueue hq = this.queues.get(hosthash);
                         if (hq == null) {i.remove(); continue smallstacks;}
-                            int delta = Latency.waitingRemainingGuessed(hq.getHost(), hq.getPort(), s, robots, ClientIdentification.yacyInternetCrawlerAgent);
+                        int delta = Latency.waitingRemainingGuessed(hq.getHost(), hq.getPort(), hosthash, robots, ClientIdentification.yacyInternetCrawlerAgent);
+                        if (delta == Integer.MIN_VALUE) {
+                            // never-crawled hosts; we do not want to have too many of them in here. Loading new hosts means: waiting for robots.txt to load
+                            freshhosts.add(hosthash);
+                            i.remove();
+                            continue smallstacks;
+                        }
+                        if (singletonStacksExist || smallStacksExist) {
                             if (delta < 0) continue; // keep all non-waiting stacks; they are useful to speed up things
                             // to protect all small stacks which have a fast throughput, remove all with long waiting time
                             if (delta >= 1000) {i.remove(); continue smallstacks;}
@@ -350,6 +357,10 @@ public class HostBalancer implements Balancer {
                             }
                         }
                     }
+                    // put at least one of the fresh hosts back
+                    if (freshhosts.size() > 0) this.roundRobinHostHashes.add(freshhosts.iterator().next());
+
+                    // result
                     if (this.roundRobinHostHashes.size() == 1) {
                         if (log.isFine()) log.fine("(re-)initialized the round-robin queue with one host");
                     } else {