diff --git a/source/de/anomic/data/robotsParser.java b/source/de/anomic/data/robotsParser.java index 645be0fce..5bd9be078 100644 --- a/source/de/anomic/data/robotsParser.java +++ b/source/de/anomic/data/robotsParser.java @@ -43,6 +43,7 @@ package de.anomic.data; +import java.io.BufferedOutputStream; import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.File; @@ -432,7 +433,7 @@ public final class robotsParser{ // downloading the content serverByteBuffer sbb = new serverByteBuffer(); - Saver.writeContent(res, sbb, null); + Saver.writeContent(res, new BufferedOutputStream(sbb), null); robotsTxt = sbb.getBytes(); downloadEnd = System.currentTimeMillis(); diff --git a/source/de/anomic/http/HttpResponse.java b/source/de/anomic/http/HttpResponse.java index 331649766..6fb230b00 100644 --- a/source/de/anomic/http/HttpResponse.java +++ b/source/de/anomic/http/HttpResponse.java @@ -25,6 +25,8 @@ // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA package de.anomic.http; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -101,13 +103,16 @@ public interface HttpResponse { * @throws IOException * @throws UnsupportedEncodingException */ - public static void writeContent(HttpResponse res, OutputStream hfos, OutputStream byteStream) throws IOException, + public static void writeContent(HttpResponse res, BufferedOutputStream hfos, BufferedOutputStream byteStream) throws IOException, UnsupportedEncodingException { try { InputStream data = res.getDataAsStream(); - OutputStream[] streams = (byteStream == null ? new OutputStream[] { (OutputStream) hfos } - : new OutputStream[] { (OutputStream) hfos, byteStream }); - serverFileUtils.copyToStreams(data, streams); + if (byteStream == null) { + serverFileUtils.copyToStream(new BufferedInputStream(data), hfos); + } else { + serverFileUtils.copyToStreams(new BufferedInputStream(data), hfos, byteStream); + } + } finally { res.closeStream(); diff --git a/source/de/anomic/http/JakartaCommonsHttpClient.java b/source/de/anomic/http/JakartaCommonsHttpClient.java index e9b7283b6..673168f61 100644 --- a/source/de/anomic/http/JakartaCommonsHttpClient.java +++ b/source/de/anomic/http/JakartaCommonsHttpClient.java @@ -25,12 +25,13 @@ // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA package de.anomic.http; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; import java.util.Date; import java.util.HashMap; import java.util.Map; @@ -88,10 +89,29 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient { */ // conManager.getParams().setDefaultMaxConnectionsPerHost(4); // default 2 conManager.getParams().setMaxTotalConnections(50); // default 20 + conManager.getParams().setConnectionTimeout(60000); // set a default timeout + conManager.getParams().setDefaultMaxConnectionsPerHost(10); // prevent DoS by mistake // TODO should this be configurable? // accept self-signed or untrusted certificates Protocol.registerProtocol("https", new Protocol("https", (ProtocolSocketFactory)new EasySSLProtocolSocketFactory(), 443)); + + + /** + * set network timeout properties. + * see: http://java.sun.com/j2se/1.5.0/docs/guide/net/properties.html + * These properties specify the default connect and read timeout (resp.) + * for the protocol handler used by java.net.URLConnection. + * the java.net.URLConnection is also used by JakartaCommons HttpClient, see + * http://hc.apache.org/httpclient-3.x/apidocs/org/apache/commons/httpclient/util/HttpURLConnection.html + */ + // specify the timeout, in milliseconds, to establish the connection to the host. + // For HTTP connections, it is the timeout when establishing the connection to the HTTP server. + System.setProperty("sun.net.client.defaultConnectTimeout","10000"); + + // specify the response timeout, in milliseconds, when reading from an input stream + // after a connection is established with a resource + System.setProperty("sun.net.client.defaultReadTimeout","60000"); } private final Map openStreams = new HashMap(); @@ -219,7 +239,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient { if (file.isFile() && file.canRead()) { // read file final ByteArrayOutputStream fileData = new ByteArrayOutputStream(); - serverFileUtils.copyToStreams(new FileInputStream(file), new OutputStream[] { fileData }); + serverFileUtils.copyToStream(new BufferedInputStream(new FileInputStream(file)), new BufferedOutputStream(fileData)); value = fileData.toByteArray(); } } diff --git a/source/de/anomic/http/httpdProxyHandler.java b/source/de/anomic/http/httpdProxyHandler.java index e0f4d39f1..b3bd6d2a9 100644 --- a/source/de/anomic/http/httpdProxyHandler.java +++ b/source/de/anomic/http/httpdProxyHandler.java @@ -62,6 +62,7 @@ package de.anomic.http; +import java.io.BufferedOutputStream; import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -1102,7 +1103,7 @@ public final class httpdProxyHandler { responseHeader); // respondHeader(respond, res.status, res.responseHeader); - Saver.writeContent(res, (chunked != null) ? chunked : respond, null); + Saver.writeContent(res, (chunked != null) ? new BufferedOutputStream(chunked) : new BufferedOutputStream(respond), null); if (chunked != null) chunked.finish(); respond.flush(); diff --git a/source/de/anomic/server/serverFileUtils.java b/source/de/anomic/server/serverFileUtils.java index f3ac76f4d..e68949e65 100644 --- a/source/de/anomic/server/serverFileUtils.java +++ b/source/de/anomic/server/serverFileUtils.java @@ -85,7 +85,7 @@ public final class serverFileUtils { * * @param source InputStream * @param dest OutputStream - * @param count the total amount of bytes to copy + * @param count the total amount of bytes to copy (-1 for all) * @return Total number of bytes copied. * @throws IOException * @@ -511,40 +511,46 @@ public final class serverFileUtils { } /** - * copies the input stream to all output streams (byte per byte) + * copies the input stream to one output stream (byte per byte) * @param in - * @param outs - * @return + * @param out + * @return number of copies bytes * @throws IOException */ - public static int copyToStreams(InputStream in, final OutputStream[] outs) throws IOException { - if(!(in instanceof BufferedInputStream)) { - // add buffer - in = new BufferedInputStream(in); - } - - // check if buffer is used - int i = 0; - for(final OutputStream output: outs) { - if (!(output instanceof BufferedOutputStream)) { - // add buffer - outs[i] = new BufferedOutputStream(output); - } - i++; + public static int copyToStream(BufferedInputStream in, final BufferedOutputStream out) throws IOException { + int count = 0; + // copy bytes + int b; + while ((b = in.read()) != -1) { + count++; + out.write(b); } + out.flush(); + return count; + } + + /** + * copies the input stream to both output streams (byte per byte) + * @param in + * @param out0 + * @param out1 + * @return number of copies bytes + * @throws IOException + */ + public static int copyToStreams(BufferedInputStream in, final BufferedOutputStream out0, final BufferedOutputStream out1) throws IOException { + assert out0 != null; + assert out1 != null; int count = 0; // copy bytes int b; while((b = in.read()) != -1) { count++; - for(final OutputStream out: outs) { - out.write(b); - } - } - for(final OutputStream out: outs) { - out.flush(); + out0.write(b); + out1.write(b); } + out0.flush(); + out1.flush(); return count; } diff --git a/source/de/anomic/yacy/yacyVersion.java b/source/de/anomic/yacy/yacyVersion.java index 5c09fec44..99c344795 100644 --- a/source/de/anomic/yacy/yacyVersion.java +++ b/source/de/anomic/yacy/yacyVersion.java @@ -27,6 +27,7 @@ package de.anomic.yacy; +import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -333,7 +334,7 @@ public final class yacyVersion implements Comparator, Comparable