- refacotring of stream copy handling to support time-consuming operations

- made usage of BufferedStreams explizit to distinct different copy method in serverFileUtils (byte-by-byte and using an own buffer)
- introduced another timeout setting (java internal property)
- more restrictions to clients accessing a single host (a security setting to prevent DoS by mistake)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4674 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/data/robotsParser.java

@@ -43,6 +43,7 @@
 
 package de.anomic.data;
 
+import java.io.BufferedOutputStream;
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
 import java.io.File;
@@ -432,7 +433,7 @@ public final class robotsParser{
                     
                     // downloading the content
                     serverByteBuffer sbb = new serverByteBuffer();
-                    Saver.writeContent(res, sbb, null);
+                    Saver.writeContent(res, new BufferedOutputStream(sbb), null);
                     robotsTxt = sbb.getBytes();
                     
                     downloadEnd = System.currentTimeMillis();                    

source/de/anomic/http/HttpResponse.java

@@ -25,6 +25,8 @@
 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 package de.anomic.http;
 
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -101,13 +103,16 @@ public interface HttpResponse {
          * @throws IOException
          * @throws UnsupportedEncodingException
          */
-        public static void writeContent(HttpResponse res, OutputStream hfos, OutputStream byteStream) throws IOException,
+        public static void writeContent(HttpResponse res, BufferedOutputStream hfos, BufferedOutputStream byteStream) throws IOException,
                 UnsupportedEncodingException {
             try {
                 InputStream data = res.getDataAsStream();
-                OutputStream[] streams = (byteStream == null ? new OutputStream[] { (OutputStream) hfos }
+                if (byteStream == null) {
-                        : new OutputStream[] { (OutputStream) hfos, byteStream });
+                    serverFileUtils.copyToStream(new BufferedInputStream(data), hfos);
-                serverFileUtils.copyToStreams(data, streams);
+                } else {
+                    serverFileUtils.copyToStreams(new BufferedInputStream(data), hfos, byteStream);
+                }
+                
                 
             } finally {
                 res.closeStream();

source/de/anomic/http/JakartaCommonsHttpClient.java

@@ -25,12 +25,13 @@
 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 package de.anomic.http;
 
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.OutputStream;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
@@ -88,10 +89,29 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient {
          */
         // conManager.getParams().setDefaultMaxConnectionsPerHost(4); // default 2
         conManager.getParams().setMaxTotalConnections(50); // default 20
+        conManager.getParams().setConnectionTimeout(60000); // set a default timeout
+        conManager.getParams().setDefaultMaxConnectionsPerHost(10); // prevent DoS by mistake
         // TODO should this be configurable?
         
         // accept self-signed or untrusted certificates
         Protocol.registerProtocol("https", new Protocol("https", (ProtocolSocketFactory)new EasySSLProtocolSocketFactory(), 443));
+        
+        
+        /**
+         * set network timeout properties.
+         * see: http://java.sun.com/j2se/1.5.0/docs/guide/net/properties.html
+         * These properties specify the default connect and read timeout (resp.)
+         * for the protocol handler used by java.net.URLConnection.
+         * the java.net.URLConnection is also used by JakartaCommons HttpClient, see
+         * http://hc.apache.org/httpclient-3.x/apidocs/org/apache/commons/httpclient/util/HttpURLConnection.html
+         */
+        // specify the timeout, in milliseconds, to establish the connection to the host.
+        // For HTTP connections, it is the timeout when establishing the connection to the HTTP server.
+        System.setProperty("sun.net.client.defaultConnectTimeout","10000");
+        
+        // specify the response timeout, in milliseconds, when reading from an input stream
+        // after a connection is established with a resource
+        System.setProperty("sun.net.client.defaultReadTimeout","60000"); 
     }
 
     private final Map<HttpMethod, InputStream> openStreams = new HashMap<HttpMethod, InputStream>();
@@ -219,7 +239,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient {
                     if (file.isFile() && file.canRead()) {
                         // read file
                         final ByteArrayOutputStream fileData = new ByteArrayOutputStream();
-                        serverFileUtils.copyToStreams(new FileInputStream(file), new OutputStream[] { fileData });
+                        serverFileUtils.copyToStream(new BufferedInputStream(new FileInputStream(file)), new BufferedOutputStream(fileData));
                         value = fileData.toByteArray();
                     }
                 }

source/de/anomic/http/httpdProxyHandler.java

@@ -62,6 +62,7 @@
 
 package de.anomic.http;
 
+import java.io.BufferedOutputStream;
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
@@ -1102,7 +1103,7 @@ public final class httpdProxyHandler {
                                     responseHeader);
             
             // respondHeader(respond, res.status, res.responseHeader);
-            Saver.writeContent(res, (chunked != null) ? chunked : respond, null);
+            Saver.writeContent(res, (chunked != null) ? new BufferedOutputStream(chunked) : new BufferedOutputStream(respond), null);
             if (chunked != null)  chunked.finish();
             
             respond.flush();

source/de/anomic/server/serverFileUtils.java

@@ -85,7 +85,7 @@ public final class serverFileUtils {
      * 
      * @param source InputStream
      * @param dest OutputStream
-     * @param count the total amount of bytes to copy
+     * @param count the total amount of bytes to copy (-1 for all)
      * @return Total number of bytes copied.
      * @throws IOException 
      * 
@@ -511,40 +511,46 @@ public final class serverFileUtils {
     }
 
     /**
-     * copies the input stream to all output streams (byte per byte)
+     * copies the input stream to one output stream (byte per byte)
      * @param in
-     * @param outs
+     * @param out
-     * @return
+     * @return number of copies bytes
      * @throws IOException
      */
-    public static int copyToStreams(InputStream in, final OutputStream[] outs) throws IOException {
+    public static int copyToStream(BufferedInputStream in, final BufferedOutputStream out) throws IOException {
-        if(!(in instanceof BufferedInputStream)) {
+        int count = 0;
-            // add buffer
+        // copy bytes
-            in = new BufferedInputStream(in);
+        int b;
-        }
+        while ((b = in.read()) != -1) {
-        
+            count++;
-        // check if buffer is used
+            out.write(b);
-        int i = 0;
-        for(final OutputStream output: outs) {
-            if (!(output instanceof BufferedOutputStream)) {
-                // add buffer
-                outs[i] = new BufferedOutputStream(output);
         }
-            i++;
+        out.flush();
+        return count;
     }
     
+    /**
+     * copies the input stream to both output streams (byte per byte)
+     * @param in
+     * @param out0
+     * @param out1
+     * @return number of copies bytes
+     * @throws IOException
+     */
+    public static int copyToStreams(BufferedInputStream in, final BufferedOutputStream out0, final BufferedOutputStream out1) throws IOException {
+        assert out0 != null;
+        assert out1 != null;
+        
         int count = 0;
         // copy bytes
         int b;
         while((b = in.read()) != -1) {
             count++;
-            for(final OutputStream out: outs) {
+            out0.write(b);
-                out.write(b);
+            out1.write(b);
-            }
-        }
-        for(final OutputStream out: outs) {
-            out.flush();
         }
+        out0.flush();
+        out1.flush();
         return count;
     }
 

source/de/anomic/yacy/yacyVersion.java

@@ -27,6 +27,7 @@
 
 package de.anomic.yacy;
 
+import java.io.BufferedOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -333,7 +334,7 @@ public final class yacyVersion implements Comparator<yacyVersion>, Comparable<ya
         HttpResponse res = null;
         try {
             res = client.GET(release.url.toString());
-            Saver.writeContent(res, new FileOutputStream(download), null);
+            Saver.writeContent(res, new BufferedOutputStream(new FileOutputStream(download)), null);
             if ((!download.exists()) || (download.length() == 0)) throw new IOException("wget of url " + release.url + " failed");
         } catch (IOException e) {
             serverLog.logSevere("yacyVersion", "download of " + release.name + " failed: " + e.getMessage());

source/yacy.java

@@ -539,7 +539,7 @@ public final class yacy {
                 serverLog.logConfig("REMOTE-SHUTDOWN", "YACY accepted shutdown command.");
                 serverLog.logConfig("REMOTE-SHUTDOWN", "Stand by for termination, which may last some seconds.");
                 ByteArrayOutputStream bos = new ByteArrayOutputStream();
-                Saver.writeContent(res, bos, null);
+                Saver.writeContent(res, new BufferedOutputStream(bos), null);
             } else {
                 serverLog.logSevere("REMOTE-SHUTDOWN", "error response from YACY socket: " + res.getStatusLine());
                 System.exit(-1);