fixes to balancer when crawling filesystem (problem was: host == null)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6906 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/crawler/Balancer.java

@@ -50,6 +50,7 @@ public class Balancer {
     private static final String indexSuffix = "9.db";
     private static final int EcoFSBufferSize = 1000;
     private static final int objectIndexBufferSize = 1000;
+    private static final String localhost = "localhost";
 
     // class variables
     private final ConcurrentHashMap<String, LinkedList<byte[]>> domainStacks; // a map from host name to lists with url hashs
@@ -259,8 +260,9 @@ public class Balancer {
         }
     }
     
-    private void pushHashToDomainStacks(final String host, final byte[] urlhash, final int maxstacksize) {
+    private void pushHashToDomainStacks(String host, final byte[] urlhash, final int maxstacksize) {
         // extend domain stack
+        if (host == null) host = localhost;
         LinkedList<byte[]> domainList = domainStacks.get(host);
         if (domainList == null) {
             // create new list
@@ -273,8 +275,9 @@ public class Balancer {
         }
     }
     
-    private void removeHashFromDomainStacks(final String host, final byte[] urlhash) {
+    private void removeHashFromDomainStacks(String host, final byte[] urlhash) {
         // extend domain stack
+        if (host == null) host = localhost;
         final LinkedList<byte[]> domainList = domainStacks.get(host);
         if (domainList == null) return;
         final Iterator<byte[]> i = domainList.iterator();
@@ -400,7 +403,9 @@ public class Balancer {
 		            }
 		        	try {
                         this.urlFileIndex.put(rowEntry);
-                        this.domainStacks.remove(crawlEntry.url().getHost());
+                        String host = crawlEntry.url().getHost();
+                        if (host == null) host = localhost;
+                        this.domainStacks.remove(host);
                         failhash = nexthash;
                     } catch (RowSpaceExceededException e) {
                         Log.logException(e);

source/de/anomic/crawler/Latency.java

@@ -37,42 +37,52 @@ public class Latency {
     private static final ConcurrentHashMap<String, Host> map = new ConcurrentHashMap<String, Host>();
     
     public static void update(MultiProtocolURI url, long time) {
-        Host h = map.get(url.getHost());
+        String host = url.getHost();
+        if (host == null) return;
+        Host h = map.get(host);
         if (h == null) {
-            h = new Host(url.getHost(), time);
-            map.put(url.getHost(), h);
+            h = new Host(host, time);
+            map.put(host, h);
         } else {
             h.update(time);
         }
     }
     
     public static void update(MultiProtocolURI url) {
-        Host h = map.get(url.getHost());
+        String host = url.getHost();
+        if (host == null) return;
+        Host h = map.get(host);
         if (h == null) {
-            h = new Host(url.getHost(), 3000);
-            map.put(url.getHost(), h);
+            h = new Host(host, 3000);
+            map.put(host, h);
         } else {
             h.update();
         }
     }
     
     public static void slowdown(MultiProtocolURI url) {
-        Host h = map.get(url.getHost());
+        String host = url.getHost();
+        if (host == null) return;
+        Host h = map.get(host);
         if (h == null) {
-            h = new Host(url.getHost(), 3000);
-            map.put(url.getHost(), h);
+            h = new Host(host, 3000);
+            map.put(host, h);
         } else {
             h.slowdown();
         }
     }
     
     public static Host host(MultiProtocolURI url) {
-        return map.get(url.getHost());
+        String host = url.getHost();
+        if (host == null) return null;
+        return map.get(host);
     }
     
     public static int average(MultiProtocolURI url) {
-        Host h = map.get(url.getHost());
-        if (h == null) return 1000;
+        String host = url.getHost();
+        if (host == null) return 0;
+        Host h = map.get(host);
+        if (h == null) return 0;
         return h.average();
     }
     
@@ -105,8 +115,9 @@ public class Latency {
      *         which expresses how long the time is over the minimum waiting time.
      */
     public static long waitingRemainingGuessed(String hostname, final long minimumLocalDelta, final long minimumGlobalDelta) {
+        if (hostname == null) return 0;
         Host host = map.get(hostname);
-        if (host == null) return Long.MIN_VALUE;
+        if (host == null) return 0;
         
         // the time since last access to the domain is the basis of the remaining calculation
         final long timeSinceLastAccess = System.currentTimeMillis() - host.lastacc();
@@ -187,7 +198,7 @@ public class Latency {
         
         // first check if the domain was _ever_ accessed before
         Host host = host(url);
-        if (host == null) return "host " + url.getHost() + " never accessed before -> 0"; // no delay if host is new
+        if (host == null) return "host " + host + " never accessed before -> 0"; // no delay if host is new
         
         StringBuilder s = new StringBuilder(50);