added dns resolve to HTTPClient POST using a dns cache to prevent that that not-thread-safe built-in dns cache inside apache http client is used

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7513 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/search/Switchboard.java

@@ -1786,7 +1786,7 @@ public final class Switchboard extends serverSwitch {
                 
                 // process the next hyperlink
                 nextUrl = nextEntry.getKey();
-                String u = nextUrl.toNormalform(true, true, true);
+                String u = nextUrl.toNormalform(true, true, false, true);
                 if (!(u.startsWith("http://") || u.startsWith("ftp://") || u.startsWith("smb://") || u.startsWith("file://"))) continue;
                 // enqueue the hyperlink into the pre-notice-url db
                 try {

source/net/yacy/cora/document/MultiProtocolURI.java

@@ -845,19 +845,19 @@ public class MultiProtocolURI implements Serializable, Comparable<MultiProtocolU
     }
     
     public String toNormalform(final boolean excludeReference, final boolean stripAmp) {
-        return toNormalform(excludeReference, stripAmp, false);
+        return toNormalform(excludeReference, stripAmp, false, false);
     }
     
     private static final Pattern ampPattern = Pattern.compile("&amp;");
-    public String toNormalform(final boolean excludeReference, final boolean stripAmp, final boolean removeSessionID) {
-        String result = toNormalform0(excludeReference, removeSessionID); 
+    public String toNormalform(final boolean excludeReference, final boolean stripAmp, final boolean resolveHost, final boolean removeSessionID) {
+        String result = toNormalform0(excludeReference, resolveHost, removeSessionID); 
         if (stripAmp) {
             result = ampPattern.matcher(result).replaceAll("&");
         }
         return result;
     }
     
-    private String toNormalform0(final boolean excludeReference, final boolean removeSessionID) {
+    private String toNormalform0(final boolean excludeReference, final boolean resolveHost, final boolean removeSessionID) {
         // generates a normal form of the URL
         boolean defaultPort = false;
         if (this.protocol.equals("mailto")) {
@@ -882,7 +882,12 @@ public class MultiProtocolURI implements Serializable, Comparable<MultiProtocolU
                 u.append(this.userInfo);
                 u.append("@");
             }
-            u.append(this.getHost().toLowerCase());
+            if (resolveHost) {
+                u.append(Domains.dnsResolve(this.getHost().toLowerCase()).getHostAddress());
+            } else {
+                u.append(this.getHost().toLowerCase());
+            }
+            
         }
         if (!defaultPort) {
             u.append(":");

source/net/yacy/cora/protocol/http/HTTPClient.java

@@ -481,6 +481,7 @@ public class HTTPClient {
     	        //assert entity.getContentLength() >= 0;
     	        assert !hrequest.expectContinue();
     	    }
+    	    
 			httpResponse = httpClient.execute(httpUriRequest, httpContext);
         } catch (Exception e) {
             //e.printStackTrace();

source/net/yacy/cora/protocol/http/HTTPConnector.java

@@ -80,7 +80,7 @@ public class HTTPConnector {
 		client.setUserAgent(this.userAgent);
 		client.setHost(vhost);
 		
-		return client.POSTbytes(url.toNormalform(false, false), post, usegzip);
+		return client.POSTbytes(url.toNormalform(false, false, true, false), post, usegzip);
 	}
 
 }

source/net/yacy/kelondro/data/meta/DigestURI.java

@@ -205,7 +205,7 @@ public class DigestURI extends MultiProtocolURI implements Serializable {
         final StringBuilder hashs = new StringBuilder(12);
         assert hashs.length() == 0;
         // form the 'local' part of the hash
-        String normalform = toNormalform(true, true, true);
+        String normalform = toNormalform(true, true, false, true);
         String b64l = Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(normalform));
         if (b64l.length() < 5) return null;
         hashs.append(b64l.substring(0, 5)); // 5 chars

test/net/yacy/cora/document/MultiProtocolURITest.java

@@ -28,7 +28,7 @@ public class MultiProtocolURITest {
 		for (int i=0; i<testURIs.length; i++) {
 			MultiProtocolURI uri = new MultiProtocolURI(testURIs[i][0]);
     
-   			assertEquals(uri.toNormalform(true, true, true), testURIs[i][1]);
+   			assertEquals(uri.toNormalform(true, true, false, true), testURIs[i][1]);
 		}
 	}
 }