*) Changes on httpc dns caching

- Bugfix: old dns cache did not handle case insensitive hostnames correctly. 
   - adding a possibility to set domain name patterns defining hostnames that should not be cached by the httpc dns cache
     e.g. borg-300.dyndns.org
     This can be done by setting the new httpc.nameCacheNoCachingPatterns property
   - using httpc.dnsResolve wherever possible within the sourcecode
     [httpd.java,plasmaCrawlStacker.java]

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1044 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/yacy/hello.java

@@ -50,6 +50,7 @@ import java.net.InetAddress;
 import java.util.Date;
 
 import de.anomic.http.httpHeader;
+import de.anomic.http.httpc;
 import de.anomic.server.serverCore;
 import de.anomic.server.serverDate;
 import de.anomic.server.serverObjects;
@@ -108,7 +109,8 @@ public final class hello {
             boolean isLocalIP = false;
             if (serverCore.portForwardingEnabled) {
                 try {
-                    final InetAddress clientAddress = InetAddress.getByName(clientip);                    
+                    final InetAddress clientAddress = httpc.dnsResolve(clientip);   
+                    if (clientAddress != null) {                        
                         if (clientAddress.isAnyLocalAddress() || clientAddress.isLoopbackAddress()) {
                             isLocalIP = true;
                         } else {
@@ -120,6 +122,7 @@ public final class hello {
                                 }
                             }  
                         }
+                    }
                 } catch (Exception e) {}
             }
 

source/de/anomic/http/httpc.java

@@ -46,6 +46,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.PushbackInputStream;
+import java.net.Inet4Address;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.MalformedURLException;
@@ -59,7 +60,9 @@ import java.util.Date;
 import java.util.Enumeration;
 import java.util.GregorianCalendar;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.Locale;
 import java.util.TimeZone;
 import java.util.zip.GZIPInputStream;
@@ -118,6 +121,8 @@ public final class httpc {
 
     // the dns cache
     private static final HashMap nameCacheHit = new HashMap();
+    public static final LinkedList nameCacheNoCachingPatterns = new LinkedList();
+    public static final HashSet nameCacheNoCachingList = new HashSet();
     //private static HashSet nameCacheMiss = new HashSet();
 
     /**
@@ -386,46 +391,67 @@ public final class httpc {
     * @param host Hostname of the host in demand.
     * @return String with the ip. null, if the host could not be resolved.
     */
-    public static String dnsResolve(String host) {
+    public static InetAddress dnsResolve(String host) {
-        String ip = (String) nameCacheHit.get(host);
+        if ((host == null)||(host.length() == 0)) return null;
+        host = host.toLowerCase().trim();
+        
+        // trying to resolve host by doing a name cache lookup
+        InetAddress ip = (InetAddress) nameCacheHit.get(host);
         if (ip != null) return ip;
+        
         // if (nameCacheMiss.contains(host)) return null;
         try {  
-            ip = InetAddress.getByName(host).getHostAddress();
+            boolean doCaching = true;
-            if ((ip != null) && (!(ip.equals("127.0.0.1"))) && (!(ip.equals("localhost")))) {
+            ip = InetAddress.getByName(host);
-                if (host.indexOf("dyndns") < 0) nameCacheHit.put(host, ip);
+            if (
-                return ip;
+                    (ip == null) ||
+                    (ip.isLoopbackAddress()) ||
+                    (nameCacheNoCachingList.contains(ip.getHostName()))
+            ) {
+                doCaching = false;
+            } else {
+                Iterator noCachingPatternIter = nameCacheNoCachingPatterns.iterator();
+                while (noCachingPatternIter.hasNext()) {
+                    String nextPattern = (String) noCachingPatternIter.next();
+                    if (ip.getHostName().matches(nextPattern)) {
+                        // disallow dns caching for this host
+                        nameCacheNoCachingList.add(ip.getHostName());
+                        doCaching = false;
+                        break;
                     }
-            return null;
-        } catch (UnknownHostException e) {
-            //nameCacheMiss.add(host);
                 }
-        return null;
             }
             
-    /**
+            if (doCaching) nameCacheHit.put(ip.getHostName(), ip);
-    * Checks wether an hostname already is in the DNS-cache.
+            return ip;
-    * FIXME: This method should use dnsResolve, as the code is 90% identical?
-    *
-    * @param host Searched for hostname.
-    * @return true, if the hostname already is in the cache.
-    */
-    public static boolean dnsFetch(String host) {
-        if ((nameCacheHit.get(host) != null) /*|| (nameCacheMiss.contains(host)) */) return false;
-        if (host.indexOf("dyndns") < 0) return false;
-        try {
-            String ip = InetAddress.getByName(host).getHostAddress();
-            if ((ip != null) && (!(ip.equals("127.0.0.1"))) && (!(ip.equals("localhost")))) {
-                nameCacheHit.put(host, ip);
-                return true;
-            }
-            return false;
         } catch (UnknownHostException e) {
             //nameCacheMiss.add(host);
-            return false;
         }
+        return null;
     }
 
+//    /**
+//    * Checks wether an hostname already is in the DNS-cache.
+//    * FIXME: This method should use dnsResolve, as the code is 90% identical?
+//    *
+//    * @param host Searched for hostname.
+//    * @return true, if the hostname already is in the cache.
+//    */
+//    public static boolean dnsFetch(String host) {
+//        if ((nameCacheHit.get(host) != null) /*|| (nameCacheMiss.contains(host)) */) return false;
+//        try {
+//            String ip = InetAddress.getByName(host).getHostAddress();
+//            if ((ip != null) && (!(ip.equals("127.0.0.1"))) && (!(ip.equals("localhost")))) {
+//                nameCacheHit.put(host, ip);
+//                return true;
+//            }
+//            return false;
+//        } catch (UnknownHostException e) {
+//            //nameCacheMiss.add(host);
+//            return false;
+//        }
+//    }
+
     /**
     * Returns the given date in an HTTP-usable format.
     *
@@ -527,13 +553,13 @@ public final class httpc {
             }
             
             this.host = server + ((port == 80) ? "" : (":" + port));
-            String hostip;
+            InetAddress hostip;
-            if ((server.equals("localhost")) || (server.equals("127.0.0.1")) || (server.startsWith("192.168.")) || (server.startsWith("10."))) {
+//            if ((server.equals("localhost")) || (server.equals("127.0.0.1")) || (server.startsWith("192.168.")) || (server.startsWith("10."))) {
-                hostip = server;
+//                hostip = server;
-            } else {
+//            } else {
                 hostip = dnsResolve(server);
                 if (hostip == null) throw new UnknownHostException(server);
-            }
+//            }
 
             // creating a socket
             this.socket = (ssl) 

source/de/anomic/http/httpd.java

@@ -1062,9 +1062,11 @@ public final class httpd implements serverHandler {
             String clientIP = conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP,"127.0.0.1");
             
             // check if ip is local ip address
-            try {
+            InetAddress hostAddress = httpc.dnsResolve(clientIP);
-                InetAddress hostAddress = InetAddress.getByName(clientIP);
+            if (hostAddress == null) {
-                if (hostAddress.isSiteLocalAddress() || hostAddress.isLoopbackAddress()) {
+                tp.put("host", serverCore.publicLocalIP().getHostAddress());
+                tp.put("port", switchboard.getConfig("port", "8080"));                    
+            } else if (hostAddress.isSiteLocalAddress() || hostAddress.isLoopbackAddress()) {
                 tp.put("host", serverCore.publicLocalIP().getHostAddress());
                 tp.put("port", switchboard.getConfig("port", "8080"));
             } else {
@@ -1073,11 +1075,6 @@ public final class httpd implements serverHandler {
                         ? Integer.toString(serverCore.portForwarding.getPort()) 
                                 : switchboard.getConfig("port", "8080"));
             }                   
-            } catch (UnknownHostException e) {
-                tp.put("host", serverCore.publicLocalIP().getHostAddress());
-                tp.put("port", switchboard.getConfig("port", "8080"));
-            }            
-            
 
             tp.put("peerName", yacyCore.seedDB.mySeed.getName());
             tp.put("errorMessageType", errorcase);            
@@ -1318,9 +1315,12 @@ public final class httpd implements serverHandler {
         
         boolean isThisHostIP = false;
         try {
-            InetAddress hostAddress = InetAddress.getByName(hostName);
+            //InetAddress hostAddress = InetAddress.getByName(hostName);
-            InetAddress forwardingAddress = InetAddress.getByName(serverCore.portForwarding.getHost());
+            InetAddress hostAddress = httpc.dnsResolve(hostName);
+            //InetAddress forwardingAddress = InetAddress.getByName(serverCore.portForwarding.getHost());
+            InetAddress forwardingAddress = httpc.dnsResolve(serverCore.portForwarding.getHost());
             
+            if ((hostAddress==null)||(forwardingAddress==null)) return false;
             if (hostAddress.equals(forwardingAddress)) return true;              
         } catch (Exception e) {}   
         return isThisHostIP;        
@@ -1331,7 +1331,10 @@ public final class httpd implements serverHandler {
         
         boolean isThisHostIP = false;
         try {
-            final InetAddress clientAddress = InetAddress.getByName(hostName);
+//             final InetAddress clientAddress = InetAddress.getByName(hostName);
+            final InetAddress clientAddress = httpc.dnsResolve(hostName);
+            if (clientAddress == null) return false;
+            
             if (clientAddress.isAnyLocalAddress() || clientAddress.isLoopbackAddress()) return true;
             
             final InetAddress[] localAddress = InetAddress.getAllByName(InetAddress.getLocalHost().getHostName());

source/de/anomic/plasma/plasmaCrawlStacker.java

@@ -206,9 +206,13 @@ public final class plasmaCrawlStacker {
         }
         
         // check if ip is local ip address
-        try {
+        InetAddress hostAddress = httpc.dnsResolve(nexturl.getHost());
-            InetAddress hostAddress = InetAddress.getByName(nexturl.getHost());
+        if (hostAddress == null) {
-            if (hostAddress.isSiteLocalAddress()) {
+            reason = "denied_(unknown_host)";
+            this.log.logFine("Unknown host in URL '" + nexturlString + "'." +
+                    "Stack processing time: " + (System.currentTimeMillis()-startTime));
+            return reason;                
+        } else if (hostAddress.isSiteLocalAddress()) {
             reason = "denied_(private_ip_address)";
             this.log.logFine("Host in URL '" + nexturlString + "' has private ip address." +
                     "Stack processing time: " + (System.currentTimeMillis()-startTime));
@@ -219,12 +223,6 @@ public final class plasmaCrawlStacker {
                     "Stack processing time: " + (System.currentTimeMillis()-startTime));
             return reason;                  
         }
-        } catch (UnknownHostException e) {
-            reason = "denied_(unknown_host)";
-            this.log.logFine("Unknown host in URL '" + nexturlString + "'." +
-                             "Stack processing time: " + (System.currentTimeMillis()-startTime));
-            return reason;
-        }
         
         // check blacklist
         String hostlow = nexturl.getHost().toLowerCase();

source/de/anomic/plasma/plasmaHTCache.java

@@ -57,6 +57,7 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.util.HashSet;
 import java.io.IOException;
+import java.net.InetAddress;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.Date;
@@ -336,14 +337,14 @@ public final class plasmaHTCache {
         // start to prefetch ip's from dns                       
         String dom;
         long start = System.currentTimeMillis();
-        String ip, result = "";
+        String result = "";
         c = 0;
         while ((doms.size() > 0) && (c < 50) && ((System.currentTimeMillis() - start) < 60000)) {
             dom = (String) doms.getMaxObject();
-            ip = httpc.dnsResolve(dom);
+            InetAddress ip = httpc.dnsResolve(dom);
             if (ip == null) continue;
-            result += ", " + dom + "=" + ip;
+            result += ", " + dom + "=" + ip.getHostAddress();
-            this.log.logConfig("PRE-FILLED " + dom + "=" + ip);
+            this.log.logConfig("PRE-FILLED " + dom + "=" + ip.getHostAddress());
             c++;
             doms.deleteScore(dom);
             // wait a short while to prevent that this looks like a DoS

source/de/anomic/plasma/plasmaSwitchboard.java

@@ -449,9 +449,20 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
         testresult = facilityDB.selectLong("statistik", (new serverDate()).toShortString(false).substring(0, 11));
         */
         
+        /*
+         * Initializing httpc
+         */
         // initializing yacyDebugMode
         httpc.yacyDebugMode = getConfig("yacyDebugMode", "false").equals("true");
         
+        // init nameCacheNoCachingList
+        String noCachingList = getConfig("httpc.nameCacheNoCachingPatterns","");
+        String[] noCachingEntries = noCachingList.split(",");
+        for (int i=0; i<noCachingEntries.length; i++) {
+            String entry = noCachingEntries[i].trim();
+            httpc.nameCacheNoCachingPatterns.add(entry);
+        }
+        
         // generate snippets cache
         log.logConfig("Initializing Snippet Cache");
         snippetCache = new plasmaSnippetCache(this,cacheManager, parser,log);

source/de/anomic/server/logging/serverLog.java

@@ -142,7 +142,7 @@ public final class serverLog {
         Logger.getLogger(appName).fine(message);
     }
     public static void logFine(String appName, String message, Throwable thrown) {
-        Logger.getLogger(appName).log(Level.FINER,message,thrown);
+        Logger.getLogger(appName).log(Level.FINE,message,thrown);
     }
     
     public static void logFiner(String appName, String message) {

yacy.init

@@ -651,3 +651,6 @@ searchProcessRemoteTime_f = 5
 searchProcessRemoteCount_f = 100
 searchProcessRemoteTime_s = 10
 searchProcessRemoteCount_s = 10
+
+# a list of domain name patterns that should not be cached by the httpc dns cache
+httpc.nameCacheNoCachingPatterns = .*.dyndns.org, .*.dynalias.org