- HTTPC ConnectionInfo entfernen bei Exceptions, unnötigen Code entfernt

- FTPC (GET-)connections bei Fehlern auf jeden Fall schliessen


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4701 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/http/HttpConnectionInfo.java

@@ -44,7 +44,8 @@ public class HttpConnectionInfo {
      */
     private final static Set<HttpConnectionInfo> allConnections = Collections
             .synchronizedSet(new HashSet<HttpConnectionInfo>());
-    private final static int staleAfterMillis = 1800 * 1000;
+    // this is only for statistics, so it can be bigger to see lost connectionInfos
+    private final static int staleAfterMillis = 30 * 60000; // 30 minutes
 
     private final String protocol;
     private final String targetHost;
@@ -156,11 +157,13 @@ public class HttpConnectionInfo {
     public static void cleanUp() {
         try {
             synchronized (allConnections) {
+                final int sizeBefore = allConnections.size();
                 for(HttpConnectionInfo con: allConnections) {
                     if(con.getLifetime() > staleAfterMillis) {
                         allConnections.remove(con);
                     }
                 }
+                serverLog.logFine("HTTPC", "cleanUp ConnectionInfo removed "+ (sizeBefore - allConnections.size()));
             }
         } catch (java.util.ConcurrentModificationException e) {
             serverLog.logWarning("HTTPC", "cleanUp ConnectionInfo interrupted by ConcurrentModificationException");

source/de/anomic/http/JakartaCommonsHttpClient.java

@@ -29,9 +29,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Date;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
 import org.apache.commons.httpclient.ConnectMethod;
 import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
@@ -122,13 +120,18 @@ public class JakartaCommonsHttpClient {
     /**
      * every x milliseconds do a cleanup (close old connections)
      */
-    private final static int cleanupIntervall = 60000;
+    private static final int cleanupIntervall = 60000;
+    /**
+     * close connections when they are not used for this time
+     * or otherwise:
+     * hold connections this time open to reuse them
+     */
+    private static final long closeConnectionsAfterMillis = 120000;
     /**
      * time the last cleanup was started
      */
     private static long lastCleanup = 0;
 
-    private final Map<HttpMethod, InputStream> openStreams = new HashMap<HttpMethod, InputStream>();
     private Header[] headers = new Header[0];
     private httpRemoteProxyConfig proxyConfig = null;
     private boolean followRedirects = true;
@@ -310,31 +313,6 @@ public class JakartaCommonsHttpClient {
         return execute(connect);
     }
 
-    /*
-     * (non-Javadoc)
-     * 
-     * @see de.anomic.http.HttpClient#closeStream()
-     */
-    public void closeStream() {
-        // close all opened streams (by this instance)
-        synchronized (openStreams) {
-            for (final HttpMethod method : openStreams.keySet()) {
-                try {
-                    // close stream
-                    openStreams.get(method).close();
-                } catch (final IOException e) {
-                    serverLog.logSevere("HTTPC", "connection cannot be closed! will unset variable.");
-                    e.printStackTrace();
-                } finally {
-                    // remove from pool
-                    openStreams.remove(method);
-                }
-                // free the connection
-                method.releaseConnection();
-            }
-        }
-    }
-
     /**
      * adds the yacy-header to the method
      * 
@@ -410,10 +388,16 @@ public class JakartaCommonsHttpClient {
         HttpConnectionInfo.addConnection(generateConInfo(method));
         
         // execute (send request)
-        if (hostConfig == null) {
-            apacheHttpClient.executeMethod(method);
-        } else {
-            apacheHttpClient.executeMethod(hostConfig, method);
+        try {
+            if (hostConfig == null) {
+                apacheHttpClient.executeMethod(method);
+            } else {
+                apacheHttpClient.executeMethod(hostConfig, method);
+            }
+        } catch (IOException e) {
+            // cleanUp statistics
+            HttpConnectionInfo.removeConnection(generateConInfo(method));
+            throw e;
         }
         
         // return response
@@ -596,7 +580,7 @@ public class JakartaCommonsHttpClient {
         final long now = System.currentTimeMillis(); 
         if(now - lastCleanup > cleanupIntervall) {
             lastCleanup = now;
-            conManager.closeIdleConnections(120000);
+            conManager.closeIdleConnections(closeConnectionsAfterMillis);
             conManager.deleteClosedConnections();
             HttpConnectionInfo.cleanUp();
         }

source/de/anomic/net/ftpc.java

@@ -2261,36 +2261,44 @@ public class ftpc {
 
         // starting data transaction
         if (status == 1) {
-            final Socket data = getDataSocket();
-            final InputStream ClientStream = data.getInputStream();
-
-            // create local file
-            RandomAccessFile outFile;
-            if (fileDest == null) {
-                outFile = new RandomAccessFile(fileName, "rw");
-            } else {
-                outFile = new RandomAccessFile(fileDest, "rw");
-            }
-
-            // write remote file to local file
-            final byte[] block = new byte[blockSize];
-            int numRead;
+            Socket data = null;
+            InputStream ClientStream = null;
+            RandomAccessFile outFile = null;
             int length = 0;
-
-            while ((numRead = ClientStream.read(block)) != -1) {
-                outFile.write(block, 0, numRead);
-                length = length + numRead;
+            try {
+                data = getDataSocket();
+                ClientStream = data.getInputStream();
+    
+                // create local file
+                if (fileDest == null) {
+                    outFile = new RandomAccessFile(fileName, "rw");
+                } else {
+                    outFile = new RandomAccessFile(fileDest, "rw");
+                }
+    
+                // write remote file to local file
+                final byte[] block = new byte[blockSize];
+                int numRead;
+    
+                while ((numRead = ClientStream.read(block)) != -1) {
+                    outFile.write(block, 0, numRead);
+                    length = length + numRead;
+                }
+    
+                // after stream is empty we should get control completion echo
+                reply = receive();
+                // boolean success = !isNotPositiveCompletion(reply);
+            } finally {
+                // shutdown connection
+                if(outFile != null) {
+                    outFile.close();
+                }
+                if(ClientStream != null) {
+                    ClientStream.close();
+                }
+                closeDataSocket();
             }
 
-            // after stream is empty we should get control completion echo
-            reply = receive();
-            // boolean success = !isNotPositiveCompletion(reply);
-
-            // shutdown connection
-            outFile.close();
-            ClientStream.close();
-            closeDataSocket();
-
             // if (!success) throw new IOException(reply);
 
             // write statistics