From 959f448e5f0ab5cd77a506b48dd63c5b2eb0c8ff Mon Sep 17 00:00:00 2001 From: danielr Date: Sat, 12 Apr 2008 11:39:48 +0000 Subject: [PATCH] - disabled redirects in proxy (so client sees real path) - added connection stats (only connections currently in use) - remove "old" connections (closed or idle for some time) - synchronized shared parts of proxyHandler git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4682 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/Connections_p.html | 3 +- htroot/Connections_p.java | 3 +- source/de/anomic/http/HttpConnectionInfo.java | 111 ++++++++++++-- .../anomic/http/JakartaCommonsHttpClient.java | 123 +++++++++++----- .../http/JakartaCommonsHttpResponse.java | 16 +- source/de/anomic/http/httpdProxyHandler.java | 139 +++++++++++------- .../de/anomic/plasma/plasmaSwitchboard.java | 5 +- source/de/anomic/server/serverFileUtils.java | 2 +- 8 files changed, 287 insertions(+), 115 deletions(-) diff --git a/htroot/Connections_p.html b/htroot/Connections_p.html index 7565e85df..c02155e4d 100644 --- a/htroot/Connections_p.html +++ b/htroot/Connections_p.html @@ -36,8 +36,7 @@

Outgoing Connections

-

Details currently not available!

-

Showing #[clientActive]# active outgoing connections:

+

Showing #[clientActive]# pooled outgoing connections used as:

diff --git a/htroot/Connections_p.java b/htroot/Connections_p.java index 879425bf4..3e5820336 100644 --- a/htroot/Connections_p.java +++ b/htroot/Connections_p.java @@ -50,6 +50,7 @@ import java.io.UnsupportedEncodingException; import java.net.InetAddress; import java.net.URLEncoder; import java.util.Properties; +import java.util.Set; import de.anomic.http.HttpConnectionInfo; import de.anomic.http.JakartaCommonsHttpClient; @@ -229,7 +230,7 @@ public final class Connections_p { prop.putNum("numActivePending", numActivePending); // client sessions - HttpConnectionInfo[] a = JakartaCommonsHttpClient.allConnections(); + Set a = HttpConnectionInfo.getAllConnections(); // TODO sorting // Arrays.sort(a, httpc.connectionTimeComparatorInstance); int c = 0; diff --git a/source/de/anomic/http/HttpConnectionInfo.java b/source/de/anomic/http/HttpConnectionInfo.java index 5f5459655..c5c1ed4e1 100644 --- a/source/de/anomic/http/HttpConnectionInfo.java +++ b/source/de/anomic/http/HttpConnectionInfo.java @@ -26,7 +26,9 @@ package de.anomic.http; -import org.apache.commons.httpclient.HttpConnection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; /** * Information about a connection @@ -35,23 +37,18 @@ import org.apache.commons.httpclient.HttpConnection; * @since 07.04.2008 */ public class HttpConnectionInfo { + /** + * a list of all current connections + */ + private final static Set allConnections = Collections + .synchronizedSet(new HashSet()); + private final String protocol; private final String targetHost; private final String command; private final int id; private final long initTime; - /** - * constructor using org.apache.commons.httpclient.HttpConnection - * - * @param connection - */ - public HttpConnectionInfo(final HttpConnection connection) { - this(connection.getProtocol().toString(), ((connection.getPort() == 80) ? connection.getHost() - : connection.getHost() + ":" + connection.getPort()), "unknown command", connection.hashCode(), - System.currentTimeMillis()); - } - /** * constructor setting all data * @@ -111,4 +108,94 @@ public class HttpConnectionInfo { public int getID() { return id; } + + /** + * @return the allConnections + */ + public static Set getAllConnections() { + return allConnections; + } + + /** + * add a connection to the list of all current connections + * + * @param conInfo + */ + public static void addConnection(final HttpConnectionInfo conInfo) { + allConnections.add(conInfo); + } + + /** + * remove a connection from the list of all current connections + * + * @param conInfo + */ + public static void removeConnection(final HttpConnectionInfo conInfo) { + allConnections.remove(conInfo); + } + + /** + * connections with same id {@link equals()} another + * + * @param id + */ + public static void removeConnection(final int id) { + removeConnection(new HttpConnectionInfo(null, null, null, id, 0)); + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + StringBuilder string = new StringBuilder(50); + string.append("ID "); + string.append(getID()); + string.append(", "); + string.append(getProtocol()); + string.append("://"); + string.append(getTargetHost()); + string.append(" "); + string.append(getCommand()); + string.append(", since "); + string.append(getLifetime()); + string.append(" ms"); + return string.toString(); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#hashCode() + */ + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + id; + return result; + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + final HttpConnectionInfo other = (HttpConnectionInfo) obj; + if (id != other.id) { + return false; + } + return true; + } } diff --git a/source/de/anomic/http/JakartaCommonsHttpClient.java b/source/de/anomic/http/JakartaCommonsHttpClient.java index 0320e1a40..ea1785c8d 100644 --- a/source/de/anomic/http/JakartaCommonsHttpClient.java +++ b/source/de/anomic/http/JakartaCommonsHttpClient.java @@ -9,7 +9,7 @@ // $LastChangedBy: orbiter $ // // LICENSE -// +// // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or @@ -41,6 +41,7 @@ import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpException; import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; +import org.apache.commons.httpclient.URIException; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.methods.HeadMethod; import org.apache.commons.httpclient.methods.InputStreamRequestEntity; @@ -77,15 +78,19 @@ public class JakartaCommonsHttpClient { * set options for client */ // set user-agent - yacyVersion thisversion = yacyVersion.thisVersion(); - apacheHttpClient.getParams().setParameter(HttpMethodParams.USER_AGENT, + final yacyVersion thisversion = yacyVersion.thisVersion(); + apacheHttpClient.getParams().setParameter( + HttpMethodParams.USER_AGENT, "yacy/" + ((thisversion == null) ? "0.0" : thisversion.releaseNr) + " (www.yacy.net; " + de.anomic.http.HttpClient.getSystemOST() + ") " + - getCurrentUserAgent().replace(';', ':')); // last ; must be before location (this is parsed) + getCurrentUserAgent().replace(';', ':')); // last ; must be + // before location + // (this is parsed) // only one retry - apacheHttpClient.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(1, false)); - + apacheHttpClient.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, + new DefaultHttpMethodRetryHandler(1, false)); + /** * set options for connection manager */ @@ -94,31 +99,39 @@ public class JakartaCommonsHttpClient { conManager.getParams().setConnectionTimeout(60000); // set a default timeout conManager.getParams().setDefaultMaxConnectionsPerHost(20); // prevent DoS by mistake // TODO should this be configurable? - + // accept self-signed or untrusted certificates - Protocol.registerProtocol("https", new Protocol("https", (ProtocolSocketFactory)new EasySSLProtocolSocketFactory(), 443)); - - + Protocol.registerProtocol("https", new Protocol("https", + (ProtocolSocketFactory) new EasySSLProtocolSocketFactory(), 443)); + /** - * set network timeout properties. - * see: http://java.sun.com/j2se/1.5.0/docs/guide/net/properties.html - * These properties specify the default connect and read timeout (resp.) - * for the protocol handler used by java.net.URLConnection. - * the java.net.URLConnection is also used by JakartaCommons HttpClient, see + * set network timeout properties. see: http://java.sun.com/j2se/1.5.0/docs/guide/net/properties.html These + * properties specify the default connect and read timeout (resp.) for the protocol handler used by + * java.net.URLConnection. the java.net.URLConnection is also used by JakartaCommons HttpClient, see * http://hc.apache.org/httpclient-3.x/apidocs/org/apache/commons/httpclient/util/HttpURLConnection.html */ // specify the timeout, in milliseconds, to establish the connection to the host. // For HTTP connections, it is the timeout when establishing the connection to the HTTP server. - System.setProperty("sun.net.client.defaultConnectTimeout","10000"); - + System.setProperty("sun.net.client.defaultConnectTimeout", "10000"); + // specify the response timeout, in milliseconds, when reading from an input stream // after a connection is established with a resource - System.setProperty("sun.net.client.defaultReadTimeout","60000"); + System.setProperty("sun.net.client.defaultReadTimeout", "60000"); } + + /** + * every x milliseconds do a cleanup (close old connections) + */ + private final static int cleanupIntervall = 60000; + /** + * time the last cleanup was started + */ + private static long lastCleanup = 0; private final Map openStreams = new HashMap(); private Header[] headers = new Header[0]; private httpRemoteProxyConfig proxyConfig = null; + private boolean followRedirects = true; /** * constructs a new Client with given parameters @@ -163,6 +176,15 @@ public class JakartaCommonsHttpClient { apacheHttpClient.getParams().setIntParameter(HttpMethodParams.SO_TIMEOUT, timeout); } + /** + * should redirects automatically be followed? + * + * @param follow + */ + public void setFollowRedirects(final boolean follow) { + followRedirects = follow; + } + /* * (non-Javadoc) * @@ -242,7 +264,8 @@ public class JakartaCommonsHttpClient { if (file.isFile() && file.canRead()) { // read file final ByteArrayOutputStream fileData = new ByteArrayOutputStream(); - serverFileUtils.copyToStream(new BufferedInputStream(new FileInputStream(file)), new BufferedOutputStream(fileData)); + serverFileUtils.copyToStream(new BufferedInputStream(new FileInputStream(file)), + new BufferedOutputStream(fileData)); value = fileData.toByteArray(); } } @@ -368,39 +391,66 @@ public class JakartaCommonsHttpClient { */ private JakartaCommonsHttpResponse execute(final HttpMethod method) throws IOException, HttpException { assert method != null : "precondition violated: method != null"; + method.setFollowRedirects(followRedirects); // set header for (final Header header : headers) { method.setRequestHeader(header); } + // set proxy final httpRemoteProxyConfig proxyConfig = getProxyConfig(method.getURI().getHost()); addProxyAuth(method, proxyConfig); final HostConfiguration hostConfig = getProxyHostConfig(proxyConfig); + + // statistics + HttpConnectionInfo.addConnection(generateConInfo(method)); + // execute (send request) if (hostConfig == null) { apacheHttpClient.executeMethod(method); } else { apacheHttpClient.executeMethod(hostConfig, method); } + // return response return new JakartaCommonsHttpResponse(method); } + /** + * @param method + * @return + */ + private HttpConnectionInfo generateConInfo(final HttpMethod method) { + int port = 80; + String host = null; + String protocol = null; + try { + port = method.getURI().getPort(); + host = method.getURI().getHost(); + protocol = method.getURI().getScheme(); + } catch (final URIException e) { + // should not happen, because method is already executed + } + return new HttpConnectionInfo(protocol, (port == 80) ? host : host + ":" + port, method.getName() + " " + + method.getPath() + "?" + method.getQueryString(), method.hashCode(), System.currentTimeMillis()); + } + /** * if necessary adds a header for proxy-authentication * * @param method * @param proxyConfig */ - private void addProxyAuth(HttpMethod method, httpRemoteProxyConfig proxyConfig) { - if(proxyConfig != null && proxyConfig.useProxy()) { + private void addProxyAuth(final HttpMethod method, final httpRemoteProxyConfig proxyConfig) { + if (proxyConfig != null && proxyConfig.useProxy()) { final String remoteProxyUser = proxyConfig.getProxyUser(); if (remoteProxyUser != null && remoteProxyUser.length() > 0) { if (remoteProxyUser.contains(":")) { serverLog.logWarning("HTTPC", "Proxy authentication contains invalid characters, trying anyway"); } final String remoteProxyPwd = proxyConfig.getProxyPwd(); - final String credentials = kelondroBase64Order.standardCoder.encodeString(remoteProxyUser.replace(":", "") + + final String credentials = kelondroBase64Order.standardCoder.encodeString(remoteProxyUser.replace(":", + "") + ":" + remoteProxyPwd); method.setRequestHeader(httpHeader.PROXY_AUTHORIZATION, "Basic " + credentials); } @@ -432,7 +482,7 @@ public class JakartaCommonsHttpClient { // generate http-configuration if (proxyConfig != null && proxyConfig.useProxy()) { // new config based on client (default) - HostConfiguration hostConfig = new HostConfiguration(apacheHttpClient.getHostConfiguration()); + final HostConfiguration hostConfig = new HostConfiguration(apacheHttpClient.getHostConfiguration()); // add proxy hostConfig.setProxy(proxyConfig.getProxyHost(), proxyConfig.getProxyPort()); return hostConfig; @@ -447,9 +497,10 @@ public class JakartaCommonsHttpClient { * @param date The Date-Object to be converted. * @return String with the date. */ - public static String date2String(Date date) { - if (date == null) + public static String date2String(final Date date) { + if (date == null) { return ""; + } return DateUtil.formatDate(date); } @@ -523,15 +574,6 @@ public class JakartaCommonsHttpClient { return (String) apacheHttpClient.getParams().getParameter(HttpMethodParams.USER_AGENT); } - /** - * a list of all connections (not yet implemented) - * - * @return - */ - public static HttpConnectionInfo[] allConnections() { - return new HttpConnectionInfo[0]; - } - /** * number of active connections * @@ -540,4 +582,17 @@ public class JakartaCommonsHttpClient { public static int connectionCount() { return conManager.getConnectionsInPool(); } + + /** + * remove unused connections + */ + public static void cleanup() { + // do it only once a while + final long now = System.currentTimeMillis(); + if(now - lastCleanup > cleanupIntervall) { + lastCleanup = now; + conManager.closeIdleConnections(120000); + conManager.deleteClosedConnections(); + } + } } \ No newline at end of file diff --git a/source/de/anomic/http/JakartaCommonsHttpResponse.java b/source/de/anomic/http/JakartaCommonsHttpResponse.java index 944463837..65dd4cb33 100644 --- a/source/de/anomic/http/JakartaCommonsHttpResponse.java +++ b/source/de/anomic/http/JakartaCommonsHttpResponse.java @@ -9,7 +9,7 @@ // $LastChangedBy: orbiter $ // // LICENSE -// +// // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or @@ -88,19 +88,19 @@ public class JakartaCommonsHttpResponse { * @throws IOException */ public byte[] getData() throws IOException { - if (this.responseBody == null) { + if (responseBody == null) { InputStream instream = null; try { instream = getDataAsStream(); if (instream != null) { - ByteArrayOutputStream outstream = new ByteArrayOutputStream(); - byte[] buffer = new byte[4096]; + final ByteArrayOutputStream outstream = new ByteArrayOutputStream(); + final byte[] buffer = new byte[4096]; int len; while ((len = instream.read(buffer)) > 0) { outstream.write(buffer, 0, len); } outstream.close(); - this.responseBody = outstream.toByteArray(); + responseBody = outstream.toByteArray(); } } finally { if (instream != null) { @@ -108,7 +108,7 @@ public class JakartaCommonsHttpResponse { } } } - return this.responseBody; + return responseBody; } /** @@ -118,7 +118,7 @@ public class JakartaCommonsHttpResponse { */ public InputStream getDataAsStream() throws IOException { InputStream inStream = method.getResponseBodyAsStream(); - if(getResponseHeader().gzip()) { + if (getResponseHeader().gzip()) { inStream = new GZIPInputStream(inStream); } // count bytes for overall http-statistics @@ -132,6 +132,8 @@ public class JakartaCommonsHttpResponse { */ public void closeStream() { method.releaseConnection(); + // statistics + HttpConnectionInfo.removeConnection(method.hashCode()); } /* diff --git a/source/de/anomic/http/httpdProxyHandler.java b/source/de/anomic/http/httpdProxyHandler.java index 4ed2d5e44..032b83946 100644 --- a/source/de/anomic/http/httpdProxyHandler.java +++ b/source/de/anomic/http/httpdProxyHandler.java @@ -89,6 +89,7 @@ import java.util.Arrays; import java.util.Date; import java.util.HashSet; import java.util.Iterator; +import java.util.Map; import java.util.Properties; import java.util.logging.FileHandler; import java.util.logging.Level; @@ -96,6 +97,7 @@ import java.util.logging.LogManager; import java.util.logging.Logger; import java.util.zip.GZIPOutputStream; +import de.anomic.data.htmlTools; import de.anomic.htmlFilter.htmlFilterContentTransformer; import de.anomic.htmlFilter.htmlFilterTransformer; import de.anomic.htmlFilter.htmlFilterWriter; @@ -238,9 +240,9 @@ public final class httpdProxyHandler { private static final serverLog proxyLog = new serverLog("PROXY.access"); /** - * Reusable {@link StringBuffer} for logging + * Reusable {@link StringBuilder} for logging */ - private static final StringBuffer logMessage = new StringBuffer(); + private static final StringBuilder logMessage = new StringBuilder(); /** * Reusable {@link StringBuffer} to generate the useragent string @@ -278,7 +280,9 @@ public final class httpdProxyHandler { */ if (requestHeader.containsKey(httpHeader.COOKIE)) { Object[] entry = new Object[]{new Date(), clienthost, requestHeader.getMultiple(httpHeader.COOKIE)}; - switchboard.outgoingCookies.put(targethost, entry); + synchronized(switchboard.outgoingCookies) { + switchboard.outgoingCookies.put(targethost, entry); + } } } @@ -300,7 +304,9 @@ public final class httpdProxyHandler { */ if (respondHeader.containsKey(httpHeader.SET_COOKIE)) { Object[] entry = new Object[]{new Date(), targetclient, respondHeader.getMultiple(httpHeader.SET_COOKIE)}; - switchboard.incomingCookies.put(serverhost, entry); + synchronized(switchboard.incomingCookies) { + switchboard.incomingCookies.put(serverhost, entry); + } } } @@ -484,7 +490,6 @@ public final class httpdProxyHandler { private static void fulfillRequestFromWeb(Properties conProp, yacyURL url,String ext, httpHeader requestHeader, httpHeader cachedResponseHeader, File cacheFile, OutputStream respond) { GZIPOutputStream gzippedOut = null; - httpChunkedOutputStream chunkedOut = null; Writer hfos = null; JakartaCommonsHttpResponse res = null; @@ -520,6 +525,7 @@ public final class httpdProxyHandler { // setup HTTP-client final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader, null); + client.setFollowRedirects(false); final String connectHost = hostPart(host, port, yAddress); final String getUrl = "http://"+ connectHost + remotePath; @@ -530,31 +536,19 @@ public final class httpdProxyHandler { res = client.GET(getUrl); conProp.put(httpHeader.CONNECTION_PROP_CLIENT_REQUEST_HEADER,requestHeader); - // determine if it's an internal error of the httpc final httpHeader responseHeader = res.getResponseHeader(); + // determine if it's an internal error of the httpc if (responseHeader.size() == 0) { throw new Exception(res.getStatusLine()); } - // if the content length is not set we have to use chunked transfer encoding - long contentLength = responseHeader.contentLength(); - if (contentLength < 0) { - // according to http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html - // a 204,304 message must not contain a message body. - // Therefore we need to set the content-length to 0. - if (res.getStatusCode() == 204 || - res.getStatusCode() == 304) { - responseHeader.put(httpHeader.CONTENT_LENGTH,"0"); - } else { - if (httpVer.equals(httpHeader.HTTP_VERSION_0_9) || httpVer.equals(httpHeader.HTTP_VERSION_1_0)) { - conProp.setProperty(httpHeader.CONNECTION_PROP_PERSISTENT,"close"); - } else { - chunkedOut = new httpChunkedOutputStream(respond); - } - responseHeader.remove(httpHeader.CONTENT_LENGTH); - } - } + final httpChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, res.getStatusCode(), respond); +// if (((String)requestHeader.get(httpHeader.ACCEPT_ENCODING,"")).indexOf("gzip") != -1) { +// zipped = new GZIPOutputStream((chunked != null) ? chunked : respond); +// res.responseHeader.put(httpHeader.CONTENT_ENCODING, "gzip"); +// res.responseHeader.remove(httpHeader.CONTENT_LENGTH); +// } // the cache does either not exist or is (supposed to be) stale long sizeBeforeDelete = -1; @@ -582,6 +576,7 @@ public final class httpdProxyHandler { ); // handle file types and make (possibly transforming) output stream + final OutputStream outStream = (gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond); if ( (!transformer.isIdentityTransformer()) && (plasmaParser.supportedHTMLContent(url,responseHeader.mime())) @@ -590,11 +585,11 @@ public final class httpdProxyHandler { theLogger.logFine("create transformer for URL " + url); //hfos = new htmlFilterOutputStream((gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond), null, transformer, (ext.length() == 0)); final String charSet = httpHeader.getCharSet(responseHeader); - hfos = new htmlFilterWriter((gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond),charSet, null, transformer, (ext.length() == 0)); + hfos = new htmlFilterWriter(outStream,charSet, null, transformer, (ext.length() == 0)); } else { // simply pass through without parsing theLogger.logFine("create passthrough for URL " + url + ", extension '" + ext + "', mime-type '" + responseHeader.mime() + "'"); - hfos = new OutputStreamWriter((gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond), httpHeader.getCharSet(res.getResponseHeader())); + hfos = new OutputStreamWriter(outStream, httpHeader.getCharSet(res.getResponseHeader())); } // handle incoming cookies @@ -630,6 +625,7 @@ public final class httpdProxyHandler { */ ((storeHTCache) || (isSupportedContent)) ) { + final long contentLength = responseHeader.contentLength(); // we write a new cache entry if ((contentLength > 0) && (contentLength < 1048576)) // if the length is known and < 1 MB { @@ -730,11 +726,54 @@ public final class httpdProxyHandler { } } catch (Exception e) { // deleting cached content - if (cacheFile.exists()) cacheFile.delete(); + if (cacheFile.exists()) cacheFile.delete(); handleProxyException(e,conProp,respond,url); } } + /** + * determines in which form the response should be send and sets header accordingly + * if the content length is not set we need to use chunked content encoding + * Implemented: + * if !content-length + * switch httpVer + * case 0.9: + * case 1.0: + * close connection after transfer + * break; + * default: + * new ChunkedStream around respond + * end if + * + * @param conProp + * @param responseHeader + * @param statusCode + * @param respond + * @return + */ + private static httpChunkedOutputStream setTransferEncoding(Properties conProp, final httpHeader responseHeader, + int statusCode, OutputStream respond) { + final String httpVer = conProp.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER); + httpChunkedOutputStream chunkedOut = null; + // gzipped response is ungzipped an therefor the length is unknown + if (responseHeader.gzip() || responseHeader.contentLength() < 0) { + // according to http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html + // a 204,304 message must not contain a message body. + // Therefore we need to set the content-length to 0. + if (statusCode == 204 || statusCode == 304) { + responseHeader.put(httpHeader.CONTENT_LENGTH, "0"); + } else { + if (httpVer.equals(httpHeader.HTTP_VERSION_0_9) || httpVer.equals(httpHeader.HTTP_VERSION_1_0)) { + forceConnectionClose(conProp); + } else { + chunkedOut = new httpChunkedOutputStream(respond); + } + responseHeader.remove(httpHeader.CONTENT_LENGTH); + } + } + return chunkedOut; + } + private static void fulfillRequestFromCache( Properties conProp, yacyURL url, @@ -798,12 +837,13 @@ public final class httpdProxyHandler { String charSet = httpHeader.getCharSet(cachedResponseHeader); // make a transformer + final OutputStream outStream = (gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond); if (( !transformer.isIdentityTransformer()) && (ext == null || !plasmaParser.supportedHTMLFileExtContains(url)) && (plasmaParser.HTMLParsableMimeTypesContains(cachedResponseHeader.mime()))) { - hfos = new htmlFilterWriter((chunkedOut != null) ? chunkedOut : respond, charSet, null, transformer, (ext.length() == 0)); + hfos = new htmlFilterWriter(outStream, charSet, null, transformer, (ext.length() == 0)); } else { - hfos = (gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond); + hfos = outStream; } // send also the complete body now from the cache @@ -897,7 +937,7 @@ public final class httpdProxyHandler { Date requestDate = new Date(); // remember the time... conProp.put(httpHeader.CONNECTION_PROP_REQUEST_START,new Long(requestDate.getTime())); if (yacyTrigger) de.anomic.yacy.yacyCore.triggerOnlineAction(); - switchboard.proxyLastAccess = System.currentTimeMillis(); + switchboard.proxyLastAccess = System.currentTimeMillis(); // using an ByteCount OutputStream to count the send bytes respond = new httpdByteCountOutputStream(respond,conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE).length() + 2,"PROXY"); @@ -907,8 +947,6 @@ public final class httpdProxyHandler { String args = conProp.getProperty(httpHeader.CONNECTION_PROP_ARGS); String httpVer = conProp.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER); - switchboard.proxyLastAccess = System.currentTimeMillis(); - int port, pos; if ((pos = host.indexOf(":")) < 0) { port = 80; @@ -958,6 +996,7 @@ public final class httpdProxyHandler { // setup HTTP-client JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader, null); + client.setFollowRedirects(false); // generate request-url final String connectHost = hostPart(host, port, yAddress); @@ -1060,6 +1099,7 @@ public final class httpdProxyHandler { // setup HTTP-client JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader, null); + client.setFollowRedirects(false); final String connectHost = hostPart(host, port, yAddress); client.setProxy(getProxyConfig(connectHost)); @@ -1084,31 +1124,13 @@ public final class httpdProxyHandler { // sending the request res = client.POST(getUrl, body); - // determine if it's an internal error of the httpc final httpHeader responseHeader = res.getResponseHeader(); + // determine if it's an internal error of the httpc if (responseHeader.size() == 0) { throw new Exception(res.getStatusLine()); } - // if the content length is not set we need to use chunked content encoding - long contentLength = responseHeader.contentLength(); - httpChunkedOutputStream chunked = null; - if (contentLength <= 0) { - // according to http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html - // a 204,304 message must not contain a message body. - // Therefore we need to set the content-length to 0. - if (res.getStatusCode() == 204 || - res.getStatusCode() == 304) { - responseHeader.put(httpHeader.CONTENT_LENGTH,"0"); - } else { - if (httpVer.equals("HTTP/0.9") || httpVer.equals("HTTP/1.0")) { - forceConnectionClose(conProp); - } else { - chunked = new httpChunkedOutputStream(respond); - } - responseHeader.remove(httpHeader.CONTENT_LENGTH); - } - } + final httpChunkedOutputStream chunked = setTransferEncoding(conProp, responseHeader, res.getStatusCode(), respond); prepareResponseHeader(responseHeader, res.getHttpVer()); @@ -1175,6 +1197,7 @@ public final class httpdProxyHandler { // TODO gzip again? set "correct" encoding? if(responseHeader.gzip()) { responseHeader.remove(httpHeader.CONTENT_ENCODING); + responseHeader.remove(httpHeader.CONTENT_LENGTH); // remove gziped length } } @@ -1240,6 +1263,7 @@ public final class httpdProxyHandler { (proxyConfig.useProxy4SSL()) ) { JakartaCommonsHttpClient remoteProxy = new JakartaCommonsHttpClient(timeout, requestHeader, proxyConfig); + remoteProxy.setFollowRedirects(false); // should not be needed, but safe is safe JakartaCommonsHttpResponse response = null; try { @@ -1485,6 +1509,7 @@ public final class httpdProxyHandler { (exceptionMsg.indexOf("server has closed connection") >= 0) )) { errorMessage = exceptionMsg; + e.printStackTrace(); } else { errorMessage = "Unexpected Error. " + e.getClass().getName() + ": " + e.getMessage(); unknownError = true; @@ -1501,7 +1526,7 @@ public final class httpdProxyHandler { } } else { if (unknownError) { - theLogger.logWarning("Error while processing request '" + + theLogger.logSevere("Unknown Error while processing request '" + conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE,"unknown") + "':" + "\n" + Thread.currentThread().getName() + "\n" + errorMessage,e); @@ -1615,7 +1640,7 @@ public final class httpdProxyHandler { return detailedErrorMsgMap; } - private static String generateUserAgent(httpHeader requestHeaders) { + private static synchronized String generateUserAgent(httpHeader requestHeaders) { userAgentStr.setLength(0); String browserUserAgent = (String) requestHeaders.get(httpHeader.USER_AGENT, proxyUserAgent); @@ -1658,7 +1683,7 @@ public final class httpdProxyHandler { * e.g.
* 1117528623.857 178 192.168.1.201 TCP_MISS/200 1069 GET http://www.yacy.de/ - DIRECT/81.169.145.74 text/html */ - private final static void logProxyAccess(Properties conProp) { + private final static synchronized void logProxyAccess(Properties conProp) { if (!doAccessLogging) return; @@ -1737,13 +1762,13 @@ public final class httpdProxyHandler { logMessage.append(mime); // sending the logging message to the logger - proxyLog.logFine(new String(logMessage)); + proxyLog.logFine(logMessage.toString()); } /** * @param remoteProxyConfig the remoteProxyConfig to set */ - public static void setRemoteProxyConfig(httpRemoteProxyConfig remoteProxyConfig) { + public static synchronized void setRemoteProxyConfig(httpRemoteProxyConfig remoteProxyConfig) { httpdProxyHandler.remoteProxyConfig = remoteProxyConfig; } diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 0a0b4e321..d8be3e71f 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -220,7 +220,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch outgoingCookies, incomingCookies; public kelondroMapTable facilityDB; public plasmaParser parser; - public long proxyLastAccess, localSearchLastAccess, remoteSearchLastAccess; + public volatile long proxyLastAccess, localSearchLastAccess, remoteSearchLastAccess; public yacyCore yc; public userDB userDB; public bookmarksDB bookmarksDB; @@ -1918,6 +1918,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch
Protocol