- disabled redirects in proxy (so client sees real path)

- added connection stats (only connections currently in use)
- remove "old" connections (closed or idle for some time)
- synchronized shared parts of proxyHandler


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4682 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/Connections_p.html

@@ -36,8 +36,7 @@
     </table>
 
 	<h3>Outgoing Connections</h3>
-	<p><strong>Details currently not available!</strong></p>
-    <p>Showing #[clientActive]# active outgoing connections:</p>
+	<p>Showing #[clientActive]# pooled outgoing connections used as:</p>
     <table border="0" cellpadding="2" cellspacing="1">
       <tr class="TableHeader" valign="bottom">
         <td>Protocol</td>

htroot/Connections_p.java

@@ -50,6 +50,7 @@ import java.io.UnsupportedEncodingException;
 import java.net.InetAddress;
 import java.net.URLEncoder;
 import java.util.Properties;
+import java.util.Set;
 
 import de.anomic.http.HttpConnectionInfo;
 import de.anomic.http.JakartaCommonsHttpClient;
@@ -229,7 +230,7 @@ public final class Connections_p {
         prop.putNum("numActivePending", numActivePending);
         
         // client sessions
-        HttpConnectionInfo[] a = JakartaCommonsHttpClient.allConnections();
+        Set<HttpConnectionInfo> a = HttpConnectionInfo.getAllConnections();
         // TODO sorting
 //        Arrays.sort(a, httpc.connectionTimeComparatorInstance);
         int c = 0;

source/de/anomic/http/HttpConnectionInfo.java

@@ -26,7 +26,9 @@
 
 package de.anomic.http;
 
-import org.apache.commons.httpclient.HttpConnection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
 
 /**
  * Information about a connection
@@ -35,23 +37,18 @@ import org.apache.commons.httpclient.HttpConnection;
  * @since 07.04.2008
  */
 public class HttpConnectionInfo {
+    /**
+     * a list of all current connections
+     */
+    private final static Set<HttpConnectionInfo> allConnections = Collections
+            .synchronizedSet(new HashSet<HttpConnectionInfo>());
+
     private final String protocol;
     private final String targetHost;
     private final String command;
     private final int id;
     private final long initTime;
 
-    /**
-     * constructor using org.apache.commons.httpclient.HttpConnection
-     * 
-     * @param connection
-     */
-    public HttpConnectionInfo(final HttpConnection connection) {
-        this(connection.getProtocol().toString(), ((connection.getPort() == 80) ? connection.getHost() 
-                : connection.getHost() + ":" + connection.getPort()), "unknown command", connection.hashCode(),
-                System.currentTimeMillis());
-    }
-
     /**
      * constructor setting all data
      * 
@@ -111,4 +108,94 @@ public class HttpConnectionInfo {
     public int getID() {
         return id;
     }
+
+    /**
+     * @return the allConnections
+     */
+    public static Set<HttpConnectionInfo> getAllConnections() {
+        return allConnections;
+    }
+
+    /**
+     * add a connection to the list of all current connections
+     * 
+     * @param conInfo
+     */
+    public static void addConnection(final HttpConnectionInfo conInfo) {
+        allConnections.add(conInfo);
+    }
+
+    /**
+     * remove a connection from the list of all current connections
+     * 
+     * @param conInfo
+     */
+    public static void removeConnection(final HttpConnectionInfo conInfo) {
+        allConnections.remove(conInfo);
+    }
+
+    /**
+     * connections with same id {@link equals()} another
+     * 
+     * @param id
+     */
+    public static void removeConnection(final int id) {
+        removeConnection(new HttpConnectionInfo(null, null, null, id, 0));
+    }
+    
+    /* (non-Javadoc)
+     * @see java.lang.Object#toString()
+     */
+    @Override
+    public String toString() {
+        StringBuilder string = new StringBuilder(50);
+        string.append("ID ");
+        string.append(getID());
+        string.append(", ");
+        string.append(getProtocol());
+        string.append("://");
+        string.append(getTargetHost());
+        string.append(" ");
+        string.append(getCommand());
+        string.append(", since ");
+        string.append(getLifetime());
+        string.append(" ms");
+        return string.toString();
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see java.lang.Object#hashCode()
+     */
+    @Override
+    public int hashCode() {
+        final int prime = 31;
+        int result = 1;
+        result = prime * result + id;
+        return result;
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see java.lang.Object#equals(java.lang.Object)
+     */
+    @Override
+    public boolean equals(final Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        final HttpConnectionInfo other = (HttpConnectionInfo) obj;
+        if (id != other.id) {
+            return false;
+        }
+        return true;
+    }
 }

source/de/anomic/http/JakartaCommonsHttpClient.java

@@ -41,6 +41,7 @@ import org.apache.commons.httpclient.HttpClient;
 import org.apache.commons.httpclient.HttpException;
 import org.apache.commons.httpclient.HttpMethod;
 import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
+import org.apache.commons.httpclient.URIException;
 import org.apache.commons.httpclient.methods.GetMethod;
 import org.apache.commons.httpclient.methods.HeadMethod;
 import org.apache.commons.httpclient.methods.InputStreamRequestEntity;
@@ -77,14 +78,18 @@ public class JakartaCommonsHttpClient {
          * set options for client
          */
         // set user-agent
-        yacyVersion thisversion = yacyVersion.thisVersion();
-        apacheHttpClient.getParams().setParameter(HttpMethodParams.USER_AGENT,
+        final yacyVersion thisversion = yacyVersion.thisVersion();
+        apacheHttpClient.getParams().setParameter(
+                                                  HttpMethodParams.USER_AGENT,
                                                   "yacy/" + ((thisversion == null) ? "0.0" : thisversion.releaseNr) +
                                                           " (www.yacy.net; " +
                                                           de.anomic.http.HttpClient.getSystemOST() + ") " +
-                                                          getCurrentUserAgent().replace(';', ':')); // last ; must be before location (this is parsed)
+                                                          getCurrentUserAgent().replace(';', ':')); // last ; must be
+                                                                                                    // before location
+                                                                                                    // (this is parsed)
         // only one retry
-        apacheHttpClient.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(1, false));
+        apacheHttpClient.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
+                                                  new DefaultHttpMethodRetryHandler(1, false));
 
         /**
          * set options for connection manager
@@ -96,29 +101,37 @@ public class JakartaCommonsHttpClient {
         // TODO should this be configurable?
 
         // accept self-signed or untrusted certificates
-        Protocol.registerProtocol("https", new Protocol("https", (ProtocolSocketFactory)new EasySSLProtocolSocketFactory(), 443));
-        
+        Protocol.registerProtocol("https", new Protocol("https",
+                (ProtocolSocketFactory) new EasySSLProtocolSocketFactory(), 443));
 
         /**
-         * set network timeout properties.
-         * see: http://java.sun.com/j2se/1.5.0/docs/guide/net/properties.html
-         * These properties specify the default connect and read timeout (resp.)
-         * for the protocol handler used by java.net.URLConnection.
-         * the java.net.URLConnection is also used by JakartaCommons HttpClient, see
+         * set network timeout properties. see: http://java.sun.com/j2se/1.5.0/docs/guide/net/properties.html These
+         * properties specify the default connect and read timeout (resp.) for the protocol handler used by
+         * java.net.URLConnection. the java.net.URLConnection is also used by JakartaCommons HttpClient, see
          * http://hc.apache.org/httpclient-3.x/apidocs/org/apache/commons/httpclient/util/HttpURLConnection.html
          */
         // specify the timeout, in milliseconds, to establish the connection to the host.
         // For HTTP connections, it is the timeout when establishing the connection to the HTTP server.
-        System.setProperty("sun.net.client.defaultConnectTimeout","10000");
+        System.setProperty("sun.net.client.defaultConnectTimeout", "10000");
 
         // specify the response timeout, in milliseconds, when reading from an input stream
         // after a connection is established with a resource
-        System.setProperty("sun.net.client.defaultReadTimeout","60000"); 
+        System.setProperty("sun.net.client.defaultReadTimeout", "60000");
     }
     
+    /**
+     * every x milliseconds do a cleanup (close old connections)
+     */
+    private final static int cleanupIntervall = 60000;
+    /**
+     * time the last cleanup was started
+     */
+    private static long lastCleanup = 0;
+
     private final Map<HttpMethod, InputStream> openStreams = new HashMap<HttpMethod, InputStream>();
     private Header[] headers = new Header[0];
     private httpRemoteProxyConfig proxyConfig = null;
+    private boolean followRedirects = true;
 
     /**
      * constructs a new Client with given parameters
@@ -163,6 +176,15 @@ public class JakartaCommonsHttpClient {
         apacheHttpClient.getParams().setIntParameter(HttpMethodParams.SO_TIMEOUT, timeout);
     }
 
+    /**
+     * should redirects automatically be followed?
+     * 
+     * @param follow
+     */
+    public void setFollowRedirects(final boolean follow) {
+        followRedirects = follow;
+    }
+
     /*
      * (non-Javadoc)
      * 
@@ -242,7 +264,8 @@ public class JakartaCommonsHttpClient {
                     if (file.isFile() && file.canRead()) {
                         // read file
                         final ByteArrayOutputStream fileData = new ByteArrayOutputStream();
-                        serverFileUtils.copyToStream(new BufferedInputStream(new FileInputStream(file)), new BufferedOutputStream(fileData));
+                        serverFileUtils.copyToStream(new BufferedInputStream(new FileInputStream(file)),
+                                                     new BufferedOutputStream(fileData));
                         value = fileData.toByteArray();
                     }
                 }
@@ -368,39 +391,66 @@ public class JakartaCommonsHttpClient {
      */
     private JakartaCommonsHttpResponse execute(final HttpMethod method) throws IOException, HttpException {
         assert method != null : "precondition violated: method != null";
+        method.setFollowRedirects(followRedirects);
         // set header
         for (final Header header : headers) {
             method.setRequestHeader(header);
         }
+        
         // set proxy
         final httpRemoteProxyConfig proxyConfig = getProxyConfig(method.getURI().getHost());
         addProxyAuth(method, proxyConfig);
         final HostConfiguration hostConfig = getProxyHostConfig(proxyConfig);
+        
+        // statistics
+        HttpConnectionInfo.addConnection(generateConInfo(method));
+        
         // execute (send request)
         if (hostConfig == null) {
             apacheHttpClient.executeMethod(method);
         } else {
             apacheHttpClient.executeMethod(hostConfig, method);
         }
+        
         // return response
         return new JakartaCommonsHttpResponse(method);
     }
 
+    /**
+     * @param method
+     * @return
+     */
+    private HttpConnectionInfo generateConInfo(final HttpMethod method) {
+        int port = 80;
+        String host = null;
+        String protocol = null;
+        try {
+            port = method.getURI().getPort();
+            host = method.getURI().getHost();
+            protocol = method.getURI().getScheme();
+        } catch (final URIException e) {
+            // should not happen, because method is already executed
+        }
+        return new HttpConnectionInfo(protocol, (port == 80) ? host : host + ":" + port, method.getName() + " " +
+                method.getPath() + "?" + method.getQueryString(), method.hashCode(), System.currentTimeMillis());
+    }
+
     /**
      * if necessary adds a header for proxy-authentication
      * 
      * @param method
      * @param proxyConfig
      */
-    private void addProxyAuth(HttpMethod method, httpRemoteProxyConfig proxyConfig) {
-        if(proxyConfig != null && proxyConfig.useProxy()) {
+    private void addProxyAuth(final HttpMethod method, final httpRemoteProxyConfig proxyConfig) {
+        if (proxyConfig != null && proxyConfig.useProxy()) {
             final String remoteProxyUser = proxyConfig.getProxyUser();
             if (remoteProxyUser != null && remoteProxyUser.length() > 0) {
                 if (remoteProxyUser.contains(":")) {
                     serverLog.logWarning("HTTPC", "Proxy authentication contains invalid characters, trying anyway");
                 }
                 final String remoteProxyPwd = proxyConfig.getProxyPwd();
-                final String credentials = kelondroBase64Order.standardCoder.encodeString(remoteProxyUser.replace(":", "") +
+                final String credentials = kelondroBase64Order.standardCoder.encodeString(remoteProxyUser.replace(":",
+                                                                                                                  "") +
                         ":" + remoteProxyPwd);
                 method.setRequestHeader(httpHeader.PROXY_AUTHORIZATION, "Basic " + credentials);
             }
@@ -432,7 +482,7 @@ public class JakartaCommonsHttpClient {
         // generate http-configuration
         if (proxyConfig != null && proxyConfig.useProxy()) {
             // new config based on client (default)
-            HostConfiguration hostConfig = new HostConfiguration(apacheHttpClient.getHostConfiguration());
+            final HostConfiguration hostConfig = new HostConfiguration(apacheHttpClient.getHostConfiguration());
             // add proxy
             hostConfig.setProxy(proxyConfig.getProxyHost(), proxyConfig.getProxyPort());
             return hostConfig;
@@ -447,9 +497,10 @@ public class JakartaCommonsHttpClient {
      * @param date The Date-Object to be converted.
      * @return String with the date.
      */
-    public static String date2String(Date date) {
-        if (date == null)
+    public static String date2String(final Date date) {
+        if (date == null) {
             return "";
+        }
 
         return DateUtil.formatDate(date);
     }
@@ -524,20 +575,24 @@ public class JakartaCommonsHttpClient {
     }
 
     /**
-     * a list of all connections (not yet implemented)
+     * number of active connections
      * 
      * @return
      */
-    public static HttpConnectionInfo[] allConnections() {
-        return new HttpConnectionInfo[0];
+    public static int connectionCount() {
+        return conManager.getConnectionsInPool();
     }
 
     /**
-     * number of active connections
-     * 
-     * @return
+     * remove unused connections
      */
-    public static int connectionCount() {
-        return conManager.getConnectionsInPool();
+    public static void cleanup() {
+        // do it only once a while
+        final long now = System.currentTimeMillis(); 
+        if(now - lastCleanup > cleanupIntervall) {
+            lastCleanup = now;
+            conManager.closeIdleConnections(120000);
+            conManager.deleteClosedConnections();
+        }
     }
 }

source/de/anomic/http/JakartaCommonsHttpResponse.java

@@ -88,19 +88,19 @@ public class JakartaCommonsHttpResponse {
      * @throws IOException
      */
     public byte[] getData() throws IOException {
-        if (this.responseBody == null) {
+        if (responseBody == null) {
             InputStream instream = null;
             try {
                 instream = getDataAsStream();
                 if (instream != null) {
-                    ByteArrayOutputStream outstream = new ByteArrayOutputStream();
-                    byte[] buffer = new byte[4096];
+                    final ByteArrayOutputStream outstream = new ByteArrayOutputStream();
+                    final byte[] buffer = new byte[4096];
                     int len;
                     while ((len = instream.read(buffer)) > 0) {
                         outstream.write(buffer, 0, len);
                     }
                     outstream.close();
-                    this.responseBody = outstream.toByteArray();
+                    responseBody = outstream.toByteArray();
                 }
             } finally {
                 if (instream != null) {
@@ -108,7 +108,7 @@ public class JakartaCommonsHttpResponse {
                 }
             }
         }
-        return this.responseBody;
+        return responseBody;
     }
 
     /**
@@ -118,7 +118,7 @@ public class JakartaCommonsHttpResponse {
      */
     public InputStream getDataAsStream() throws IOException {
         InputStream inStream = method.getResponseBodyAsStream();
-        if(getResponseHeader().gzip()) {
+        if (getResponseHeader().gzip()) {
             inStream = new GZIPInputStream(inStream);
         }
         // count bytes for overall http-statistics
@@ -132,6 +132,8 @@ public class JakartaCommonsHttpResponse {
      */
     public void closeStream() {
         method.releaseConnection();
+        // statistics
+        HttpConnectionInfo.removeConnection(method.hashCode());
     }
 
     /*

source/de/anomic/http/httpdProxyHandler.java

@@ -89,6 +89,7 @@ import java.util.Arrays;
 import java.util.Date;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.Map;
 import java.util.Properties;
 import java.util.logging.FileHandler;
 import java.util.logging.Level;
@@ -96,6 +97,7 @@ import java.util.logging.LogManager;
 import java.util.logging.Logger;
 import java.util.zip.GZIPOutputStream;
 
+import de.anomic.data.htmlTools;
 import de.anomic.htmlFilter.htmlFilterContentTransformer;
 import de.anomic.htmlFilter.htmlFilterTransformer;
 import de.anomic.htmlFilter.htmlFilterWriter;
@@ -238,9 +240,9 @@ public final class httpdProxyHandler {
     private static final serverLog proxyLog = new serverLog("PROXY.access");
     
     /**
-     * Reusable {@link StringBuffer} for logging
+     * Reusable {@link StringBuilder} for logging
      */
-    private static final StringBuffer logMessage = new StringBuffer();
+    private static final StringBuilder logMessage = new StringBuilder();
     
     /**
      * Reusable {@link StringBuffer} to generate the useragent string
@@ -278,9 +280,11 @@ public final class httpdProxyHandler {
          */
         if (requestHeader.containsKey(httpHeader.COOKIE)) {
             Object[] entry = new Object[]{new Date(), clienthost, requestHeader.getMultiple(httpHeader.COOKIE)};
+            synchronized(switchboard.outgoingCookies) {
                 switchboard.outgoingCookies.put(targethost, entry);
             }
         }
+    }
     
     public static void handleIncomingCookies(httpHeader respondHeader, String serverhost, String targetclient) {
         /*
@@ -300,9 +304,11 @@ public final class httpdProxyHandler {
          */
         if (respondHeader.containsKey(httpHeader.SET_COOKIE)) {
             Object[] entry = new Object[]{new Date(), targetclient, respondHeader.getMultiple(httpHeader.SET_COOKIE)};
+            synchronized(switchboard.incomingCookies) {
                 switchboard.incomingCookies.put(serverhost, entry);
             }
         }
+    }
     
     /**
      * @param conProp a collection of properties about the connection, like URL
@@ -484,7 +490,6 @@ public final class httpdProxyHandler {
     private static void fulfillRequestFromWeb(Properties conProp, yacyURL url,String ext, httpHeader requestHeader, httpHeader cachedResponseHeader, File cacheFile, OutputStream respond) {
         
         GZIPOutputStream gzippedOut = null; 
-        httpChunkedOutputStream chunkedOut = null;
         Writer hfos = null;
         
         JakartaCommonsHttpResponse res = null;                
@@ -520,6 +525,7 @@ public final class httpdProxyHandler {
             
             // setup HTTP-client
             final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader, null);
+            client.setFollowRedirects(false);
             
             final String connectHost = hostPart(host, port, yAddress);
             final String getUrl = "http://"+ connectHost + remotePath;
@@ -530,31 +536,19 @@ public final class httpdProxyHandler {
             res = client.GET(getUrl);
             conProp.put(httpHeader.CONNECTION_PROP_CLIENT_REQUEST_HEADER,requestHeader);
             
-            // determine if it's an internal error of the httpc
             final httpHeader responseHeader = res.getResponseHeader();
+            // determine if it's an internal error of the httpc
             if (responseHeader.size() == 0) {
                 throw new Exception(res.getStatusLine());
             }
             
-            // if the content length is not set we have to use chunked transfer encoding
-            long contentLength = responseHeader.contentLength();
-            if (contentLength < 0) {
-                // according to http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
-                // a 204,304 message must not contain a message body.
-                // Therefore we need to set the content-length to 0.
-                if (res.getStatusCode() == 204 || 
-                    res.getStatusCode() == 304) {
-                    responseHeader.put(httpHeader.CONTENT_LENGTH,"0");
-                } else {
-                    if (httpVer.equals(httpHeader.HTTP_VERSION_0_9) || httpVer.equals(httpHeader.HTTP_VERSION_1_0)) {
-                        conProp.setProperty(httpHeader.CONNECTION_PROP_PERSISTENT,"close");
-                    } else {
-                        chunkedOut = new httpChunkedOutputStream(respond);
-                    }
-                    responseHeader.remove(httpHeader.CONTENT_LENGTH);
-                }
-            }        
+            final httpChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, res.getStatusCode(), respond);        
             
+//          if (((String)requestHeader.get(httpHeader.ACCEPT_ENCODING,"")).indexOf("gzip") != -1) {
+//          zipped = new GZIPOutputStream((chunked != null) ? chunked : respond);
+//          res.responseHeader.put(httpHeader.CONTENT_ENCODING, "gzip");
+//          res.responseHeader.remove(httpHeader.CONTENT_LENGTH);
+//          }
 
             // the cache does either not exist or is (supposed to be) stale
             long sizeBeforeDelete = -1;
@@ -582,6 +576,7 @@ public final class httpdProxyHandler {
             );
 
             // handle file types and make (possibly transforming) output stream
+            final OutputStream outStream = (gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond);
             if (
                     (!transformer.isIdentityTransformer()) &&
                     (plasmaParser.supportedHTMLContent(url,responseHeader.mime()))
@@ -590,11 +585,11 @@ public final class httpdProxyHandler {
                 theLogger.logFine("create transformer for URL " + url);
                 //hfos = new htmlFilterOutputStream((gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond), null, transformer, (ext.length() == 0));
                 final String charSet = httpHeader.getCharSet(responseHeader);
-                hfos = new htmlFilterWriter((gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond),charSet, null, transformer, (ext.length() == 0));
+                hfos = new htmlFilterWriter(outStream,charSet, null, transformer, (ext.length() == 0));
             } else {
                 // simply pass through without parsing
                 theLogger.logFine("create passthrough for URL " + url + ", extension '" + ext + "', mime-type '" + responseHeader.mime() + "'");
-                hfos = new OutputStreamWriter((gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond), httpHeader.getCharSet(res.getResponseHeader()));
+                hfos = new OutputStreamWriter(outStream, httpHeader.getCharSet(res.getResponseHeader()));
             }
             
             // handle incoming cookies
@@ -630,6 +625,7 @@ public final class httpdProxyHandler {
                      */
                     ((storeHTCache) || (isSupportedContent))
             ) {
+                final long contentLength = responseHeader.contentLength();
                 // we write a new cache entry
                 if ((contentLength > 0) && (contentLength < 1048576)) // if the length is known and < 1 MB
                 {
@@ -735,6 +731,49 @@ public final class httpdProxyHandler {
         }
     }
 
+    /**
+     * determines in which form the response should be send and sets header accordingly
+     * if the content length is not set we need to use chunked content encoding 
+     * Implemented:
+     * if !content-length
+     *  switch httpVer
+     *    case 0.9:
+     *    case 1.0:
+     *      close connection after transfer
+     *      break;
+     *    default:
+     *      new ChunkedStream around respond
+     * end if
+     * 
+     * @param conProp
+     * @param responseHeader
+     * @param statusCode
+     * @param respond
+     * @return
+     */
+    private static httpChunkedOutputStream setTransferEncoding(Properties conProp, final httpHeader responseHeader,
+            int statusCode, OutputStream respond) {
+        final String httpVer = conProp.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER);
+        httpChunkedOutputStream chunkedOut = null;
+        // gzipped response is ungzipped an therefor the length is unknown
+        if (responseHeader.gzip() || responseHeader.contentLength() < 0) {
+            // according to http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
+            // a 204,304 message must not contain a message body.
+            // Therefore we need to set the content-length to 0.
+            if (statusCode == 204 || statusCode == 304) {
+                responseHeader.put(httpHeader.CONTENT_LENGTH, "0");
+            } else {
+                if (httpVer.equals(httpHeader.HTTP_VERSION_0_9) || httpVer.equals(httpHeader.HTTP_VERSION_1_0)) {
+                    forceConnectionClose(conProp);
+                } else {
+                    chunkedOut = new httpChunkedOutputStream(respond);
+                }
+                responseHeader.remove(httpHeader.CONTENT_LENGTH);
+            }
+        }
+        return chunkedOut;
+    }
+
     private static void fulfillRequestFromCache(
             Properties conProp, 
             yacyURL url,
@@ -798,12 +837,13 @@ public final class httpdProxyHandler {
                 String charSet = httpHeader.getCharSet(cachedResponseHeader);
                 
                 // make a transformer
+                final OutputStream outStream = (gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond);
                 if (( !transformer.isIdentityTransformer()) &&
                         (ext == null || !plasmaParser.supportedHTMLFileExtContains(url)) &&
                         (plasmaParser.HTMLParsableMimeTypesContains(cachedResponseHeader.mime()))) {
-                    hfos = new htmlFilterWriter((chunkedOut != null) ? chunkedOut : respond, charSet, null, transformer, (ext.length() == 0));
+                    hfos = new htmlFilterWriter(outStream, charSet, null, transformer, (ext.length() == 0));
                 } else {
-                    hfos = (gzippedOut != null) ? gzippedOut : ((chunkedOut != null)? chunkedOut : respond);
+                    hfos = outStream;
                 }
                 
                 // send also the complete body now from the cache
@@ -907,8 +947,6 @@ public final class httpdProxyHandler {
             String args = conProp.getProperty(httpHeader.CONNECTION_PROP_ARGS); 
             String httpVer = conProp.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER);
             
-            switchboard.proxyLastAccess = System.currentTimeMillis();
-            
             int port, pos;
             if ((pos = host.indexOf(":")) < 0) {
                 port = 80;
@@ -958,6 +996,7 @@ public final class httpdProxyHandler {
             
             // setup HTTP-client
             JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader, null);
+            client.setFollowRedirects(false);
             
             // generate request-url
             final String connectHost = hostPart(host, port, yAddress);
@@ -1060,6 +1099,7 @@ public final class httpdProxyHandler {
             
             // setup HTTP-client
             JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader, null);
+            client.setFollowRedirects(false);
             
             final String connectHost = hostPart(host, port, yAddress);
             client.setProxy(getProxyConfig(connectHost));
@@ -1084,31 +1124,13 @@ public final class httpdProxyHandler {
             // sending the request
             res = client.POST(getUrl, body);
             
-            // determine if it's an internal error of the httpc
             final httpHeader responseHeader = res.getResponseHeader();
+            // determine if it's an internal error of the httpc
             if (responseHeader.size() == 0) {
                 throw new Exception(res.getStatusLine());
             }                                  
             
-            // if the content length is not set we need to use chunked content encoding
-            long contentLength = responseHeader.contentLength();
-            httpChunkedOutputStream chunked = null;
-            if (contentLength <= 0) {
-                // according to http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
-                // a 204,304 message must not contain a message body.
-                // Therefore we need to set the content-length to 0.
-                if (res.getStatusCode() == 204 || 
-                    res.getStatusCode() == 304) {
-                    responseHeader.put(httpHeader.CONTENT_LENGTH,"0");
-                } else {                
-                    if (httpVer.equals("HTTP/0.9") || httpVer.equals("HTTP/1.0")) {
-                        forceConnectionClose(conProp);
-                    } else {
-                        chunked = new httpChunkedOutputStream(respond);
-                    }
-                    responseHeader.remove(httpHeader.CONTENT_LENGTH);                
-                }
-            }
+            final httpChunkedOutputStream chunked = setTransferEncoding(conProp, responseHeader, res.getStatusCode(), respond);
             
             prepareResponseHeader(responseHeader, res.getHttpVer()); 
             
@@ -1175,6 +1197,7 @@ public final class httpdProxyHandler {
         // TODO gzip again? set "correct" encoding?
         if(responseHeader.gzip()) {
             responseHeader.remove(httpHeader.CONTENT_ENCODING);
+            responseHeader.remove(httpHeader.CONTENT_LENGTH); // remove gziped length
         }
     }
 
@@ -1240,6 +1263,7 @@ public final class httpdProxyHandler {
                 (proxyConfig.useProxy4SSL())
         ) {
             JakartaCommonsHttpClient remoteProxy = new JakartaCommonsHttpClient(timeout, requestHeader, proxyConfig);
+            remoteProxy.setFollowRedirects(false); // should not be needed, but safe is safe 
 
             JakartaCommonsHttpResponse response = null;
             try {
@@ -1485,6 +1509,7 @@ public final class httpdProxyHandler {
                      (exceptionMsg.indexOf("server has closed connection") >= 0)
                   )) { 
                     errorMessage = exceptionMsg;
+                    e.printStackTrace();
                 } else {
                     errorMessage = "Unexpected Error. " + e.getClass().getName() + ": " + e.getMessage();
                     unknownError = true;
@@ -1501,7 +1526,7 @@ public final class httpdProxyHandler {
                 }
             } else {
                 if (unknownError) {
-                    theLogger.logWarning("Error while processing request '" + 
+                    theLogger.logSevere("Unknown Error while processing request '" + 
                             conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE,"unknown") + "':" +
                             "\n" + Thread.currentThread().getName() + 
                             "\n" + errorMessage,e);
@@ -1615,7 +1640,7 @@ public final class httpdProxyHandler {
         return detailedErrorMsgMap;
     }
     
-    private static String generateUserAgent(httpHeader requestHeaders) {
+    private static synchronized String generateUserAgent(httpHeader requestHeaders) {
         userAgentStr.setLength(0);
         
         String browserUserAgent = (String) requestHeaders.get(httpHeader.USER_AGENT, proxyUserAgent);
@@ -1658,7 +1683,7 @@ public final class httpdProxyHandler {
      * e.g.<br>
      * <code>1117528623.857    178 192.168.1.201 TCP_MISS/200 1069 GET http://www.yacy.de/ - DIRECT/81.169.145.74 text/html</code>
      */
-    private final static void logProxyAccess(Properties conProp) {
+    private final static synchronized void logProxyAccess(Properties conProp) {
         
         if (!doAccessLogging) return;
         
@@ -1737,13 +1762,13 @@ public final class httpdProxyHandler {
         logMessage.append(mime);        
         
         // sending the logging message to the logger
-        proxyLog.logFine(new String(logMessage));
+        proxyLog.logFine(logMessage.toString());
     }
 
     /**
      * @param remoteProxyConfig the remoteProxyConfig to set
      */
-    public static void setRemoteProxyConfig(httpRemoteProxyConfig remoteProxyConfig) {
+    public static synchronized void setRemoteProxyConfig(httpRemoteProxyConfig remoteProxyConfig) {
         httpdProxyHandler.remoteProxyConfig = remoteProxyConfig;
     }
 

source/de/anomic/plasma/plasmaSwitchboard.java

@@ -220,7 +220,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
     public  HashMap<String, Object[]>   outgoingCookies, incomingCookies;
     public  kelondroMapTable            facilityDB;
     public  plasmaParser                parser;
-    public  long                        proxyLastAccess, localSearchLastAccess, remoteSearchLastAccess;
+    public  volatile long                        proxyLastAccess, localSearchLastAccess, remoteSearchLastAccess;
     public  yacyCore                    yc;
     public  userDB                      userDB;
     public  bookmarksDB                 bookmarksDB;
@@ -1919,6 +1919,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
         try {
             boolean hasDoneSomething = false;
             
+            // close unused connections
+            JakartaCommonsHttpClient.cleanup();
+
             // do transmission of CR-files
             checkInterruption();
             int count = rankingOwnDistribution.size() / 100;

source/de/anomic/server/serverFileUtils.java

@@ -557,7 +557,7 @@ public final class serverFileUtils {
     /**
      * copies the input stream to all writers (byte per byte)
      * @param data
-     * @param writers
+     * @param writer
      * @param charSet
      * @return
      * @throws IOException