From cd19d0517ea58a07a0792d1dab86fa17857ca784 Mon Sep 17 00:00:00 2001
From: orbiter <orbiter@6c8d7289-2bf4-0310-a012-ef5d649a1542>
Date: Tue, 22 Feb 2011 22:58:19 +0000
Subject: [PATCH] added dns resolve to HTTPClient POST using a dns cache to
 prevent that that not-thread-safe built-in dns cache inside apache http
 client is used

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7513 6c8d7289-2bf4-0310-a012-ef5d649a1542
---
 source/de/anomic/search/Switchboard.java          |  2 +-
 .../net/yacy/cora/document/MultiProtocolURI.java  | 15 ++++++++++-----
 .../net/yacy/cora/protocol/http/HTTPClient.java   |  1 +
 .../yacy/cora/protocol/http/HTTPConnector.java    |  2 +-
 source/net/yacy/kelondro/data/meta/DigestURI.java |  2 +-
 .../yacy/cora/document/MultiProtocolURITest.java  |  2 +-
 6 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/source/de/anomic/search/Switchboard.java b/source/de/anomic/search/Switchboard.java
index 073bb9e08..a5a7022ef 100644
--- a/source/de/anomic/search/Switchboard.java
+++ b/source/de/anomic/search/Switchboard.java
@@ -1786,7 +1786,7 @@ public final class Switchboard extends serverSwitch {
                 
                 // process the next hyperlink
                 nextUrl = nextEntry.getKey();
-                String u = nextUrl.toNormalform(true, true, true);
+                String u = nextUrl.toNormalform(true, true, false, true);
                 if (!(u.startsWith("http://") || u.startsWith("ftp://") || u.startsWith("smb://") || u.startsWith("file://"))) continue;
                 // enqueue the hyperlink into the pre-notice-url db
                 try {
diff --git a/source/net/yacy/cora/document/MultiProtocolURI.java b/source/net/yacy/cora/document/MultiProtocolURI.java
index 2b8a7d9f3..bf95caaf1 100644
--- a/source/net/yacy/cora/document/MultiProtocolURI.java
+++ b/source/net/yacy/cora/document/MultiProtocolURI.java
@@ -845,19 +845,19 @@ public class MultiProtocolURI implements Serializable, Comparable<MultiProtocolU
     }
     
     public String toNormalform(final boolean excludeReference, final boolean stripAmp) {
-        return toNormalform(excludeReference, stripAmp, false);
+        return toNormalform(excludeReference, stripAmp, false, false);
     }
     
     private static final Pattern ampPattern = Pattern.compile("&amp;");
-    public String toNormalform(final boolean excludeReference, final boolean stripAmp, final boolean removeSessionID) {
-        String result = toNormalform0(excludeReference, removeSessionID); 
+    public String toNormalform(final boolean excludeReference, final boolean stripAmp, final boolean resolveHost, final boolean removeSessionID) {
+        String result = toNormalform0(excludeReference, resolveHost, removeSessionID); 
         if (stripAmp) {
             result = ampPattern.matcher(result).replaceAll("&");
         }
         return result;
     }
     
-    private String toNormalform0(final boolean excludeReference, final boolean removeSessionID) {
+    private String toNormalform0(final boolean excludeReference, final boolean resolveHost, final boolean removeSessionID) {
         // generates a normal form of the URL
         boolean defaultPort = false;
         if (this.protocol.equals("mailto")) {
@@ -882,7 +882,12 @@ public class MultiProtocolURI implements Serializable, Comparable<MultiProtocolU
                 u.append(this.userInfo);
                 u.append("@");
             }
-            u.append(this.getHost().toLowerCase());
+            if (resolveHost) {
+                u.append(Domains.dnsResolve(this.getHost().toLowerCase()).getHostAddress());
+            } else {
+                u.append(this.getHost().toLowerCase());
+            }
+            
         }
         if (!defaultPort) {
             u.append(":");
diff --git a/source/net/yacy/cora/protocol/http/HTTPClient.java b/source/net/yacy/cora/protocol/http/HTTPClient.java
index e812a3146..302f5a860 100644
--- a/source/net/yacy/cora/protocol/http/HTTPClient.java
+++ b/source/net/yacy/cora/protocol/http/HTTPClient.java
@@ -481,6 +481,7 @@ public class HTTPClient {
     	        //assert entity.getContentLength() >= 0;
     	        assert !hrequest.expectContinue();
     	    }
+    	    
 			httpResponse = httpClient.execute(httpUriRequest, httpContext);
         } catch (Exception e) {
             //e.printStackTrace();
diff --git a/source/net/yacy/cora/protocol/http/HTTPConnector.java b/source/net/yacy/cora/protocol/http/HTTPConnector.java
index 316d31763..d5fc4f7f3 100644
--- a/source/net/yacy/cora/protocol/http/HTTPConnector.java
+++ b/source/net/yacy/cora/protocol/http/HTTPConnector.java
@@ -80,7 +80,7 @@ public class HTTPConnector {
 		client.setUserAgent(this.userAgent);
 		client.setHost(vhost);
 		
-		return client.POSTbytes(url.toNormalform(false, false), post, usegzip);
+		return client.POSTbytes(url.toNormalform(false, false, true, false), post, usegzip);
 	}
 
 }
diff --git a/source/net/yacy/kelondro/data/meta/DigestURI.java b/source/net/yacy/kelondro/data/meta/DigestURI.java
index 777da510c..c6f6021fa 100644
--- a/source/net/yacy/kelondro/data/meta/DigestURI.java
+++ b/source/net/yacy/kelondro/data/meta/DigestURI.java
@@ -205,7 +205,7 @@ public class DigestURI extends MultiProtocolURI implements Serializable {
         final StringBuilder hashs = new StringBuilder(12);
         assert hashs.length() == 0;
         // form the 'local' part of the hash
-        String normalform = toNormalform(true, true, true);
+        String normalform = toNormalform(true, true, false, true);
         String b64l = Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(normalform));
         if (b64l.length() < 5) return null;
         hashs.append(b64l.substring(0, 5)); // 5 chars
diff --git a/test/net/yacy/cora/document/MultiProtocolURITest.java b/test/net/yacy/cora/document/MultiProtocolURITest.java
index 4f55c509d..3f18c5309 100644
--- a/test/net/yacy/cora/document/MultiProtocolURITest.java
+++ b/test/net/yacy/cora/document/MultiProtocolURITest.java
@@ -28,7 +28,7 @@ public class MultiProtocolURITest {
 		for (int i=0; i<testURIs.length; i++) {
 			MultiProtocolURI uri = new MultiProtocolURI(testURIs[i][0]);
     
-   			assertEquals(uri.toNormalform(true, true, true), testURIs[i][1]);
+   			assertEquals(uri.toNormalform(true, true, false, true), testURIs[i][1]);
 		}
 	}
 }