diff --git a/source/de/anomic/crawler/RobotsTxt.java b/source/de/anomic/crawler/RobotsTxt.java index ac0d212e4..ca28356bd 100644 --- a/source/de/anomic/crawler/RobotsTxt.java +++ b/source/de/anomic/crawler/RobotsTxt.java @@ -355,7 +355,7 @@ public class RobotsTxt { // res = client.GET(robotsURL.toString()); robotsTxt = client.GETbytes(robotsURL.toString()); final int code = client.getHttpResponse().getStatusLine().getStatusCode(); - final ResponseHeader header = new ResponseHeader(null, client.getHeaderHashMap()); + final ResponseHeader header = new ResponseHeader(client.getHttpResponse().getAllHeaders()); // check the response status // if (res.getStatusLine().startsWith("2")) { diff --git a/source/de/anomic/crawler/retrieval/HTTPLoader.java b/source/de/anomic/crawler/retrieval/HTTPLoader.java index 08672c031..3d1100e49 100644 --- a/source/de/anomic/crawler/retrieval/HTTPLoader.java +++ b/source/de/anomic/crawler/retrieval/HTTPLoader.java @@ -127,7 +127,7 @@ public final class HTTPLoader { // send request // res = client.GET(request.url().toString(), maxFileSize); final byte[] responseBody = client.GETbytes(request.url().toString(), maxFileSize); - final ResponseHeader header = new ResponseHeader(null, client.getHeaderHashMap()); + final ResponseHeader header = new ResponseHeader(client.getHttpResponse().getAllHeaders()); final int code = client.getHttpResponse().getStatusLine().getStatusCode(); // FIXME: 30*-handling (bottom) is never reached // we always get the final content because httpClient.followRedirects = true @@ -258,7 +258,7 @@ public final class HTTPLoader { // send request // res = client.GET(request.url().toString(), Long.MAX_VALUE); final byte[] responseBody = client.GETbytes(request.url().toString(), Long.MAX_VALUE); - final ResponseHeader header = new ResponseHeader(null, client.getHeaderHashMap()); + final ResponseHeader header = new ResponseHeader(client.getHttpResponse().getAllHeaders()); final int code = client.getHttpResponse().getStatusLine().getStatusCode(); // FIXME: 30*-handling (bottom) is never reached // we always get the final content because httpClient.followRedirects = true diff --git a/source/de/anomic/http/server/HTTPDProxyHandler.java b/source/de/anomic/http/server/HTTPDProxyHandler.java index c38ef1379..75adf98f5 100644 --- a/source/de/anomic/http/server/HTTPDProxyHandler.java +++ b/source/de/anomic/http/server/HTTPDProxyHandler.java @@ -486,7 +486,7 @@ public final class HTTPDProxyHandler { conProp.put(HeaderFramework.CONNECTION_PROP_CLIENT_REQUEST_HEADER, requestHeader); // final ResponseHeader responseHeader = res.getResponseHeader(); - final ResponseHeader responseHeader = new ResponseHeader(null, client.getHeaderHashMap()); + final ResponseHeader responseHeader = new ResponseHeader(client.getHttpResponse().getAllHeaders()); // determine if it's an internal error of the httpc if (responseHeader.isEmpty()) { // throw new Exception(res.getStatusLine()); @@ -845,7 +845,7 @@ public final class HTTPDProxyHandler { // if (responseHeader.isEmpty()) { // throw new Exception(res.getStatusLine()); // } - final ResponseHeader responseHeader = new ResponseHeader(null, client.getHeaderHashMap()); + final ResponseHeader responseHeader = new ResponseHeader(client.getHttpResponse().getAllHeaders()); if (responseHeader.isEmpty()) { throw new Exception(client.getHttpResponse().getStatusLine().toString()); } @@ -976,7 +976,7 @@ public final class HTTPDProxyHandler { if (log.isFinest()) log.logFinest(reqID +" response status: "+ client.getHttpResponse().getStatusLine()); // final ResponseHeader responseHeader = res.getResponseHeader(); - final ResponseHeader responseHeader = new ResponseHeader(null, client.getHeaderHashMap()); + final ResponseHeader responseHeader = new ResponseHeader(client.getHttpResponse().getAllHeaders()); // determine if it's an internal error of the httpc if (responseHeader.isEmpty()) { // throw new Exception(res.getStatusLine()); @@ -1307,7 +1307,7 @@ public final class HTTPDProxyHandler { try { // response = remoteProxy.CONNECT(host, port); remoteProxy.HEADResponse("http://" + host + ":" + port); - ResponseHeader header = new ResponseHeader(null, remoteProxy.getHeaderHashMap()); + ResponseHeader header = new ResponseHeader(remoteProxy.getHttpResponse().getAllHeaders()); // outputs a logline to the serverlog with the current status // log.logInfo("CONNECT-RESPONSE: status=" + response.getStatusLine() + ", header=" + response.getResponseHeader().toString()); diff --git a/source/de/anomic/http/server/ResponseHeader.java b/source/de/anomic/http/server/ResponseHeader.java index bede38ab1..27c280509 100755 --- a/source/de/anomic/http/server/ResponseHeader.java +++ b/source/de/anomic/http/server/ResponseHeader.java @@ -30,6 +30,8 @@ import java.util.Date; import java.util.HashMap; import java.util.Map; +import org.apache.http.Header; + import net.yacy.kelondro.logging.Log; @@ -42,6 +44,13 @@ public class ResponseHeader extends HeaderFramework { public ResponseHeader() { super(); } + + public ResponseHeader(Header[] headers) { + super(); + for (final Header h : headers) { + this.add(h.getName(), h.getValue()); + } + } public ResponseHeader(final HashMap reverseMappingCache) { super(reverseMappingCache); diff --git a/source/de/anomic/search/Switchboard.java b/source/de/anomic/search/Switchboard.java index f64b46331..352f6ba02 100644 --- a/source/de/anomic/search/Switchboard.java +++ b/source/de/anomic/search/Switchboard.java @@ -2133,8 +2133,8 @@ public final class Switchboard extends serverSwitch { log.logInfo("dhtTransferJob: too many connections in httpc pool : " + ConnectionInfo.getCount()); // close unused connections // Client.cleanup(); - } else if (kbytesUp > 512) { - log.logInfo("dhtTransferJob: too much upload, currently uploading: " + kbytesUp + " Kb"); + } else if (kbytesUp > 128) { + log.logInfo("dhtTransferJob: too much upload(1), currently uploading: " + kbytesUp + " Kb"); } else { byte[] startHash = null, limitHash = null; int tries = 10; @@ -2167,8 +2167,8 @@ public final class Switchboard extends serverSwitch { log.logInfo("dhtTransferJob: too many connections in httpc pool : " + ConnectionInfo.getCount()); // close unused connections // Client.cleanup(); - } else if (kbytesUp > 512) { - log.logInfo("dhtTransferJob: too much upload, currently uploading: " + kbytesUp + " Kb"); + } else if (kbytesUp > 256) { + log.logInfo("dhtTransferJob: too much upload(2), currently uploading: " + kbytesUp + " Kb"); } else { boolean dequeued = this.dhtDispatcher.dequeueContainer(); hasDoneSomething = hasDoneSomething | dequeued; @@ -2363,7 +2363,7 @@ public final class Switchboard extends serverSwitch { final long start = System.currentTimeMillis(); // header = Client.whead(url.toString(), reqHeader); client.HEADResponse(url.toString()); - header = new ResponseHeader(null, client.getHeaderHashMap()); + header = new ResponseHeader(client.getHttpResponse().getAllHeaders()); final long loadtime = System.currentTimeMillis() - start; // if (header == null) { if (header == null) { diff --git a/source/de/anomic/yacy/yacyRelease.java b/source/de/anomic/yacy/yacyRelease.java index 4e43a7335..ea6ec2119 100644 --- a/source/de/anomic/yacy/yacyRelease.java +++ b/source/de/anomic/yacy/yacyRelease.java @@ -313,7 +313,7 @@ public final class yacyRelease extends yacyVersion { // res = client.GET(this.getUrl().toString()); client.setTimout(120000); client.GET(this.getUrl().toString()); - final ResponseHeader header = new ResponseHeader(null, client.getHeaderHashMap()); + final ResponseHeader header = new ResponseHeader(client.getHttpResponse().getAllHeaders()); // final boolean unzipped = res.getResponseHeader().gzip() && (res.getResponseHeader().mime().toLowerCase().equals("application/x-tar")); // if true, then the httpc has unzipped the file final boolean unzipped = header.gzip() && (header.mime().toLowerCase().equals("application/x-tar")); // if true, then the httpc has unzipped the file diff --git a/source/net/yacy/cora/protocol/Client.java b/source/net/yacy/cora/protocol/Client.java index 820093bf9..6f1d95d8a 100644 --- a/source/net/yacy/cora/protocol/Client.java +++ b/source/net/yacy/cora/protocol/Client.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.UnsupportedEncodingException; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Set; import java.util.Map.Entry; @@ -46,7 +45,6 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpHead; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.client.params.CookiePolicy; import org.apache.http.client.params.HttpClientParams; import org.apache.http.conn.ClientConnectionManager; import org.apache.http.conn.params.ConnManagerParams; @@ -148,9 +146,6 @@ public class Client { final SchemeRegistry schemeRegistry = new SchemeRegistry(); schemeRegistry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80)); schemeRegistry.register(new Scheme("https", SSLSocketFactory.getSocketFactory(), 443)); - - // cookie policy - HttpClientParams.setCookiePolicy(httpParams, CookiePolicy.RFC_2965); ClientConnectionManager clientConnectionManager = new ThreadSafeClientConnManager(httpParams, schemeRegistry); @@ -331,15 +326,6 @@ public class Client { return httpResponse; } - public HashMap getHeaderHashMap() { - if (httpResponse == null) return null; - final HashMap hmap = new HashMap(); - for (Header h : httpResponse.getAllHeaders()) { - hmap.put(h.getName(), h.getValue()); - } - return hmap; - } - public void writeTo(final OutputStream outputStream) throws IOException { if (httpResponse != null && currentRequest != null) { final HttpEntity httpEntity = httpResponse.getEntity();