*) bugfix for usage of httpc without gzip content encoding

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@369 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
theli 20 years ago
parent 0e2c33ee55
commit 57c30f1d78

@ -665,7 +665,7 @@ public final class httpc {
public response GET(String path, httpHeader requestHeader) throws IOException { public response GET(String path, httpHeader requestHeader) throws IOException {
//serverLog.logDebug("HTTPC", handle + " requested GET '" + path + "', time = " + (System.currentTimeMillis() - handle)); //serverLog.logDebug("HTTPC", handle + " requested GET '" + path + "', time = " + (System.currentTimeMillis() - handle));
try { try {
boolean zipped = (this.allowContentEncoding) ? true : httpd.shallTransportZipped(path); boolean zipped = (!this.allowContentEncoding) ? false : httpd.shallTransportZipped(path);
send(httpHeader.METHOD_GET, path, requestHeader, zipped); send(httpHeader.METHOD_GET, path, requestHeader, zipped);
response r = new response(zipped); response r = new response(zipped);
//serverLog.logDebug("HTTPC", handle + " returned GET '" + path + "', time = " + (System.currentTimeMillis() - handle)); //serverLog.logDebug("HTTPC", handle + " returned GET '" + path + "', time = " + (System.currentTimeMillis() - handle));

@ -724,7 +724,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
} else { } else {
if (e.getMessage().indexOf("Corrupt GZIP trailer") >= 0) { if (e.getMessage().indexOf("Corrupt GZIP trailer") >= 0) {
// just do nothing, we leave it this way // just do nothing, we leave it this way
this.theLogger.logDebug("ignoring bad gzip trail for URL " + url + " (" + e.getMessage() + ")",e); this.theLogger.logDebug("ignoring bad gzip trail for URL " + url + " (" + e.getMessage() + ")");
this.forceConnectionClose(); this.forceConnectionClose();
} else if ((remote != null)&&(remote.isClosed())) { // TODO: query for broken pipe } else if ((remote != null)&&(remote.isClosed())) { // TODO: query for broken pipe
errorMessage = "destination host unexpectedly closed connection"; errorMessage = "destination host unexpectedly closed connection";

@ -282,7 +282,7 @@ public final class plasmaCrawlWorker extends Thread {
httpHeader requestHeader = new httpHeader(); httpHeader requestHeader = new httpHeader();
requestHeader.put("User-Agent", httpdProxyHandler.userAgent); requestHeader.put("User-Agent", httpdProxyHandler.userAgent);
requestHeader.put("Referer", referer); requestHeader.put("Referer", referer);
requestHeader.put("Accept-Encoding", "gzip,deflate"); if (useContentEncodingGzip) requestHeader.put("Accept-Encoding", "gzip,deflate");
//System.out.println("CRAWLER_REQUEST_HEADER=" + requestHeader.toString()); // DEBUG //System.out.println("CRAWLER_REQUEST_HEADER=" + requestHeader.toString()); // DEBUG

@ -176,6 +176,7 @@ public final class serverCodings {
return out; return out;
} catch (ArrayIndexOutOfBoundsException e) { } catch (ArrayIndexOutOfBoundsException e) {
// maybe the input was not base64 // maybe the input was not base64
e.printStackTrace();
return null; return null;
} }
} }

Loading…
Cancel
Save