... migrating to HttpComponents-Client-4.x ...

make the occurrence of multiple header-keys possible

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7031 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/crawler/RobotsTxt.java

@@ -355,7 +355,7 @@ public class RobotsTxt {
 //            res = client.GET(robotsURL.toString());
             robotsTxt = client.GETbytes(robotsURL.toString());
             final int code = client.getHttpResponse().getStatusLine().getStatusCode();
-            final ResponseHeader header = new ResponseHeader(null, client.getHeaderHashMap());
+            final ResponseHeader header = new ResponseHeader(client.getHttpResponse().getAllHeaders());
             
             // check the response status
 //            if (res.getStatusLine().startsWith("2")) {

source/de/anomic/crawler/retrieval/HTTPLoader.java

@@ -127,7 +127,7 @@ public final class HTTPLoader {
             // send request
 //            res = client.GET(request.url().toString(), maxFileSize);
         	final byte[] responseBody = client.GETbytes(request.url().toString(), maxFileSize);
-        	final ResponseHeader header = new ResponseHeader(null, client.getHeaderHashMap());
+        	final ResponseHeader header = new ResponseHeader(client.getHttpResponse().getAllHeaders());
         	final int code = client.getHttpResponse().getStatusLine().getStatusCode();
             // FIXME: 30*-handling (bottom) is never reached
             // we always get the final content because httpClient.followRedirects = true
@@ -258,7 +258,7 @@ public final class HTTPLoader {
             // send request
 //            res = client.GET(request.url().toString(), Long.MAX_VALUE);
         	final byte[] responseBody = client.GETbytes(request.url().toString(), Long.MAX_VALUE);
-        	final ResponseHeader header = new ResponseHeader(null, client.getHeaderHashMap());
+        	final ResponseHeader header = new ResponseHeader(client.getHttpResponse().getAllHeaders());
         	final int code = client.getHttpResponse().getStatusLine().getStatusCode();
             // FIXME: 30*-handling (bottom) is never reached
             // we always get the final content because httpClient.followRedirects = true

source/de/anomic/http/server/HTTPDProxyHandler.java

@@ -486,7 +486,7 @@ public final class HTTPDProxyHandler {
                 conProp.put(HeaderFramework.CONNECTION_PROP_CLIENT_REQUEST_HEADER, requestHeader);
 
 //                final ResponseHeader responseHeader = res.getResponseHeader();
-                final ResponseHeader responseHeader = new ResponseHeader(null, client.getHeaderHashMap());
+                final ResponseHeader responseHeader = new ResponseHeader(client.getHttpResponse().getAllHeaders());
                 // determine if it's an internal error of the httpc
                 if (responseHeader.isEmpty()) {
 //                    throw new Exception(res.getStatusLine());
@@ -845,7 +845,7 @@ public final class HTTPDProxyHandler {
 //            if (responseHeader.isEmpty()) {
 //                throw new Exception(res.getStatusLine());
 //            }      
-            final ResponseHeader responseHeader = new ResponseHeader(null, client.getHeaderHashMap());
+            final ResponseHeader responseHeader = new ResponseHeader(client.getHttpResponse().getAllHeaders());
             if (responseHeader.isEmpty()) {
                 throw new Exception(client.getHttpResponse().getStatusLine().toString());
             } 
@@ -976,7 +976,7 @@ public final class HTTPDProxyHandler {
 	            if (log.isFinest()) log.logFinest(reqID +"    response status: "+ client.getHttpResponse().getStatusLine());
 	            
 //	            final ResponseHeader responseHeader = res.getResponseHeader();
-	            final ResponseHeader responseHeader = new ResponseHeader(null, client.getHeaderHashMap());
+	            final ResponseHeader responseHeader = new ResponseHeader(client.getHttpResponse().getAllHeaders());
 	            // determine if it's an internal error of the httpc
 	            if (responseHeader.isEmpty()) {
 //	                throw new Exception(res.getStatusLine());
@@ -1307,7 +1307,7 @@ public final class HTTPDProxyHandler {
             try {
 //                response = remoteProxy.CONNECT(host, port);
             	remoteProxy.HEADResponse("http://" + host + ":" + port);
-            	ResponseHeader header = new ResponseHeader(null, remoteProxy.getHeaderHashMap());
+            	ResponseHeader header = new ResponseHeader(remoteProxy.getHttpResponse().getAllHeaders());
             	
                 // outputs a logline to the serverlog with the current status
 //                log.logInfo("CONNECT-RESPONSE: status=" + response.getStatusLine() + ", header=" + response.getResponseHeader().toString());

source/de/anomic/http/server/ResponseHeader.java

@@ -30,6 +30,8 @@ import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
 
+import org.apache.http.Header;
+
 import net.yacy.kelondro.logging.Log;
 
 
@@ -43,6 +45,13 @@ public class ResponseHeader extends HeaderFramework {
         super();
     }
 
+    public ResponseHeader(Header[] headers) {
+        super();
+        for (final Header h : headers) {
+        	this.add(h.getName(), h.getValue());
+        }
+    }
+    
     public ResponseHeader(final HashMap<String, String> reverseMappingCache) {
         super(reverseMappingCache);
     }

source/de/anomic/search/Switchboard.java

@@ -2133,8 +2133,8 @@ public final class Switchboard extends serverSwitch {
             log.logInfo("dhtTransferJob: too many connections in httpc pool : " + ConnectionInfo.getCount());
             // close unused connections
 //            Client.cleanup();
-        } else if (kbytesUp > 512) {
+        } else if (kbytesUp > 128) {
-        	log.logInfo("dhtTransferJob: too much upload, currently uploading: " + kbytesUp + " Kb");
+        	log.logInfo("dhtTransferJob: too much upload(1), currently uploading: " + kbytesUp + " Kb");
         } else {
             byte[] startHash = null, limitHash = null;
             int tries = 10;
@@ -2167,8 +2167,8 @@ public final class Switchboard extends serverSwitch {
             log.logInfo("dhtTransferJob: too many connections in httpc pool : " + ConnectionInfo.getCount());
             // close unused connections
 //            Client.cleanup();
-        } else if (kbytesUp > 512) {
+        } else if (kbytesUp > 256) {
-        	log.logInfo("dhtTransferJob: too much upload, currently uploading: " + kbytesUp + " Kb");
+        	log.logInfo("dhtTransferJob: too much upload(2), currently uploading: " + kbytesUp + " Kb");
         } else {
         	boolean dequeued = this.dhtDispatcher.dequeueContainer();
         	hasDoneSomething = hasDoneSomething | dequeued;
@@ -2363,7 +2363,7 @@ public final class Switchboard extends serverSwitch {
                     final long start = System.currentTimeMillis();
 //                    header = Client.whead(url.toString(), reqHeader); 
                     client.HEADResponse(url.toString());
-                    header = new ResponseHeader(null, client.getHeaderHashMap());
+                    header = new ResponseHeader(client.getHttpResponse().getAllHeaders());
                     final long loadtime = System.currentTimeMillis() - start;
 //                    if (header == null) {
                     if (header == null) {

source/de/anomic/yacy/yacyRelease.java

@@ -313,7 +313,7 @@ public final class yacyRelease extends yacyVersion {
 //            res = client.GET(this.getUrl().toString());
             client.setTimout(120000);
             client.GET(this.getUrl().toString());
-            final ResponseHeader header = new ResponseHeader(null, client.getHeaderHashMap());
+            final ResponseHeader header = new ResponseHeader(client.getHttpResponse().getAllHeaders());
 
 //            final boolean unzipped = res.getResponseHeader().gzip() && (res.getResponseHeader().mime().toLowerCase().equals("application/x-tar")); // if true, then the httpc has unzipped the file
             final boolean unzipped = header.gzip() && (header.mime().toLowerCase().equals("application/x-tar")); // if true, then the httpc has unzipped the file

source/net/yacy/cora/protocol/Client.java

@@ -29,7 +29,6 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.UnsupportedEncodingException;
-import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.Set;
 import java.util.Map.Entry;
@@ -46,7 +45,6 @@ import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpHead;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.client.params.CookiePolicy;
 import org.apache.http.client.params.HttpClientParams;
 import org.apache.http.conn.ClientConnectionManager;
 import org.apache.http.conn.params.ConnManagerParams;
@@ -149,9 +147,6 @@ public class Client {
 		schemeRegistry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80));
 		schemeRegistry.register(new Scheme("https", SSLSocketFactory.getSocketFactory(), 443));
 
-		// cookie policy
-		HttpClientParams.setCookiePolicy(httpParams, CookiePolicy.RFC_2965);
-
 		ClientConnectionManager clientConnectionManager = new ThreadSafeClientConnManager(httpParams, schemeRegistry);
 
 		httpClient = new DefaultHttpClient(clientConnectionManager, httpParams);
@@ -331,15 +326,6 @@ public class Client {
 		return httpResponse;
 	}
 	
-	public HashMap<String, String> getHeaderHashMap() {
-		if (httpResponse == null) return null;
-		final HashMap<String, String> hmap = new HashMap<String, String>();
-		for (Header h : httpResponse.getAllHeaders()) {
-			hmap.put(h.getName(), h.getValue());
-		}
-		return hmap;
-	}
-	
 	public void writeTo(final OutputStream outputStream) throws IOException {
 		if (httpResponse != null && currentRequest != null) {
 			final HttpEntity httpEntity = httpResponse.getEntity();