From 1dc94e7753e687db93ee0878f90767c6cf256d53 Mon Sep 17 00:00:00 2001 From: theli Date: Thu, 22 Sep 2005 10:30:55 +0000 Subject: [PATCH] *) Adding support for gzip content-encoding of http post requests used to transferRWIs and transferURLs. See: http://www.yacy-forum.de/viewtopic.php?t=1167#10020 *) adding yacyVersion.java containing constants defining yacy versions that support a given feature. Needed to determine if a remote peer is able to decode gzip content-encoded http post bodies properly. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@772 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/yacy/hello.java | 3 +- .../anomic/http/httpChunkedInputStream.java | 8 +-- source/de/anomic/http/httpc.java | 58 +++++++++++++------ source/de/anomic/http/httpd.java | 46 +++++++++++---- source/de/anomic/http/httpdFileHandler.java | 18 ++++-- source/de/anomic/icap/icapd.java | 2 +- source/de/anomic/yacy/yacyClient.java | 14 ++++- source/de/anomic/yacy/yacyVersion.java | 6 ++ 8 files changed, 115 insertions(+), 40 deletions(-) create mode 100644 source/de/anomic/yacy/yacyVersion.java diff --git a/htroot/yacy/hello.java b/htroot/yacy/hello.java index 04dc1806a..ba71bcf55 100644 --- a/htroot/yacy/hello.java +++ b/htroot/yacy/hello.java @@ -55,6 +55,7 @@ import de.anomic.server.serverSwitch; import de.anomic.yacy.yacyClient; import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacySeed; +import de.anomic.yacy.yacyVersion; public class hello { @@ -99,7 +100,7 @@ public class hello { // if the remote client has reported its own IP address and the client supports // the port forwarding feature (if client version >= 0.383) then we try to // connect to the reported IP address first - if (reportedip.length() > 0 && !clientip.equals(reportedip) && clientversion >= (float)0.383) { + if (reportedip.length() > 0 && !clientip.equals(reportedip) && clientversion >= yacyVersion.YACY_SUPPORTS_PORT_FORWARDING) { // try first the reportedip, since this may be a connect from a port-forwarding host prop.put(STR_YOURIP, reportedip); remoteSeed.put(STR_IP, reportedip); diff --git a/source/de/anomic/http/httpChunkedInputStream.java b/source/de/anomic/http/httpChunkedInputStream.java index 91f538db1..9c4c2addc 100644 --- a/source/de/anomic/http/httpChunkedInputStream.java +++ b/source/de/anomic/http/httpChunkedInputStream.java @@ -54,7 +54,7 @@ import java.io.InputStreamReader; * Some parts of this class code was copied from Apache httpclient Project. * @author theli */ -public class httpChunkedInputStream extends InputStream { +public final class httpChunkedInputStream extends InputStream { private static final int READ_CHUNK_STATE_NORMAL = 0; private static final int READ_CHUNK_STATE_CR_READ = 1; @@ -64,7 +64,7 @@ public class httpChunkedInputStream extends InputStream { private static final char CR = '\r'; private static final char LF = '\n'; - private InputStream inputStream; + private final InputStream inputStream; private int currPos; private int currChunkSize; private httpHeader httpTrailer; @@ -97,9 +97,6 @@ public class httpChunkedInputStream extends InputStream { public int read (byte[] b, int off, int len) throws IOException { - - if (b == null) throw new IllegalArgumentException("bytearry parameter must not be null"); - if (this.isClosed) throw new IOException("Inputstream already closed."); if (this.isEOF) return -1; @@ -114,7 +111,6 @@ public class httpChunkedInputStream extends InputStream { } public int read (byte[] b) throws IOException { - if (b == null) throw new IllegalArgumentException("bytearry parameter must not be null"); return read(b, 0, b.length); } diff --git a/source/de/anomic/http/httpc.java b/source/de/anomic/http/httpc.java index 144b92700..8b4907e72 100644 --- a/source/de/anomic/http/httpc.java +++ b/source/de/anomic/http/httpc.java @@ -66,6 +66,7 @@ import java.util.Locale; import java.util.TimeZone; import java.util.Vector; import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; import javax.net.ssl.SSLSocketFactory; @@ -87,6 +88,14 @@ import org.apache.commons.pool.impl.GenericObjectPool; */ public final class httpc { + // some constants + /** + * Specifies that the httpc is allowed to use gzip content encoding for + * http post requests + * @see #POST(String, httpHeader, serverObjects, Hashtable) + */ + public static final String GZIP_POST_BODY = "GZIP_POST_BODY"; + // statics private static final String vDATE = "20040602"; private static String userAgent; @@ -757,7 +766,18 @@ public final class httpc { } boundary = "--" + boundary.substring(pos + "boundary=".length()); + boolean zipContent = args.containsKey(GZIP_POST_BODY); + + OutputStream out; + GZIPOutputStream zippedOut; ByteArrayOutputStream buf = new ByteArrayOutputStream(); + if (zipContent) { + zippedOut = new GZIPOutputStream(buf); + out = zippedOut; + } else { + out = buf; + } + // in contrast to GET and HEAD, this method also transports a message body // the body consists of repeated boundaries and values in between if (args.size() != 0) { @@ -766,38 +786,42 @@ public final class httpc { Enumeration e = args.keys(); while (e.hasMoreElements()) { // start with a boundary - buf.write(boundary.getBytes()); - buf.write(serverCore.crlf); + out.write(boundary.getBytes()); + out.write(serverCore.crlf); // write value key = (String) e.nextElement(); value = (String) args.get(key, ""); if ((files != null) && (files.containsKey(key))) { // we are about to write a file - buf.write(("Content-Disposition: form-data; name=" + '"' + key + '"' + "; filename=" + '"' + value + '"').getBytes()); - buf.write(serverCore.crlf); - buf.write(serverCore.crlf); - buf.write((byte[]) files.get(key)); - buf.write(serverCore.crlf); + out.write(("Content-Disposition: form-data; name=" + '"' + key + '"' + "; filename=" + '"' + value + '"').getBytes()); + out.write(serverCore.crlf); + out.write(serverCore.crlf); + out.write((byte[]) files.get(key)); + out.write(serverCore.crlf); } else { // write a single value - buf.write(("Content-Disposition: form-data; name=" + '"' + key + '"').getBytes()); - buf.write(serverCore.crlf); - buf.write(serverCore.crlf); - buf.write(value.getBytes()); - buf.write(serverCore.crlf); + out.write(("Content-Disposition: form-data; name=" + '"' + key + '"').getBytes()); + out.write(serverCore.crlf); + out.write(serverCore.crlf); + out.write(value.getBytes()); + out.write(serverCore.crlf); } } // finish with a boundary - buf.write(boundary.getBytes()); - buf.write(serverCore.crlf); + out.write(boundary.getBytes()); + out.write(serverCore.crlf); //buf.write("" + serverCore.crlfString); } // create body array - buf.close(); + out.close(); byte[] body = buf.toByteArray(); //System.out.println("DEBUG: PUT BODY=" + new String(body)); - // size of that body - requestHeader.put(httpHeader.CONTENT_LENGTH, Integer.toString(body.length)); + if (zipContent) { + requestHeader.put(httpHeader.CONTENT_ENCODING, "gzip"); + } else { + // size of that body + requestHeader.put(httpHeader.CONTENT_LENGTH, Integer.toString(body.length)); + } // send the header //System.out.println("header=" + requestHeader); send(httpHeader.METHOD_POST, path, requestHeader, false); diff --git a/source/de/anomic/http/httpd.java b/source/de/anomic/http/httpd.java index 74b62bee0..966ed0b19 100644 --- a/source/de/anomic/http/httpd.java +++ b/source/de/anomic/http/httpd.java @@ -46,6 +46,7 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.io.PrintStream; import java.io.PushbackInputStream; @@ -666,11 +667,23 @@ public final class httpd implements serverHandler { // but this belongs to the protocol handler, this class. - public static int parseArgs(serverObjects args, PushbackInputStream in, int length) throws IOException { + public static int parseArgs(serverObjects args, InputStream in, int length) throws IOException { // this is a quick hack using a previously coded parseMultipart based on a buffer // should be replaced sometime by a 'right' implementation - byte[] buffer = new byte[length]; - in.read(buffer); + byte[] buffer = null; + + // parsing post request bodies with a given length + if (length != -1) { + buffer = new byte[length]; + in.read(buffer); + // parsing post request bodies which are gzip content-encoded + } else { + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + serverFileUtils.copy(in,bout); + buffer = bout.toByteArray(); + bout.close(); bout = null; + } + int argc = parseArgs(args, new String(buffer)); buffer = null; return argc; @@ -722,16 +735,29 @@ public final class httpd implements serverHandler { } - public static HashMap parseMultipart(httpHeader header, serverObjects args, PushbackInputStream in, int length) throws IOException { + public static HashMap parseMultipart(httpHeader header, serverObjects args, InputStream in, int length) throws IOException { // this is a quick hack using a previously coded parseMultipart based on a buffer // should be replaced sometime by a 'right' implementation - byte[] buffer = new byte[length]; - int c, a = 0; - while (a < length) { - c = in.read(buffer, a, length - a); - if (c <= 0) break; - a += c; + + byte[] buffer = null; + + // parsing post request bodies with a given length + if (length != -1) { + buffer = new byte[length]; + int c, a = 0; + while (a < length) { + c = in.read(buffer, a, length - a); + if (c <= 0) break; + a += c; + } + // parsing post request bodies which are gzip content-encoded + } else { + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + serverFileUtils.copy(in,bout); + buffer = bout.toByteArray(); + bout.close(); bout = null; } + //System.out.println("MULTIPART-BUFFER=" + new String(buffer)); HashMap files = parseMultipart(header, args, buffer); buffer = null; diff --git a/source/de/anomic/http/httpdFileHandler.java b/source/de/anomic/http/httpdFileHandler.java index 428884a25..849ede99b 100644 --- a/source/de/anomic/http/httpdFileHandler.java +++ b/source/de/anomic/http/httpdFileHandler.java @@ -96,6 +96,7 @@ import java.util.Iterator; import java.util.Map; import java.util.Properties; import java.util.logging.Level; +import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; import de.anomic.plasma.plasmaSwitchboard; @@ -240,7 +241,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http doResponse(conProp, requestHeader, response, body); } - public void doResponse(Properties conProp, httpHeader requestHeader, OutputStream out, PushbackInputStream body) throws IOException { + public void doResponse(Properties conProp, httpHeader requestHeader, OutputStream out, InputStream body) throws IOException { this.connectionProperties = conProp; @@ -303,10 +304,19 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http // no args here, maybe a POST with multipart extension int length; //System.out.println("HEADER: " + requestHeader.toString()); // DEBUG - if ((method.equals(httpHeader.METHOD_POST)) && - (requestHeader.containsKey(httpHeader.CONTENT_LENGTH))) { + if (method.equals(httpHeader.METHOD_POST)) { + + if (requestHeader.containsKey(httpHeader.CONTENT_LENGTH)) { + length = Integer.parseInt((String) requestHeader.get(httpHeader.CONTENT_LENGTH)); + } else if (requestHeader.gzip()) { + length = -1; + body = new GZIPInputStream(body); + } else { + httpd.sendRespondError(conProp,out,4,403,null,"bad post values",null); + return; + } + // if its a POST, it can be either multipart or as args in the body - length = Integer.parseInt((String) requestHeader.get(httpHeader.CONTENT_LENGTH)); if ((requestHeader.containsKey(httpHeader.CONTENT_TYPE)) && (((String) requestHeader.get(httpHeader.CONTENT_TYPE)).toLowerCase().startsWith("multipart"))) { // parse multipart diff --git a/source/de/anomic/icap/icapd.java b/source/de/anomic/icap/icapd.java index 769b660b7..c6af96b4e 100644 --- a/source/de/anomic/icap/icapd.java +++ b/source/de/anomic/icap/icapd.java @@ -286,7 +286,7 @@ public class icapd implements serverHandler { httpChunkedInputStream chunkedIn = new httpChunkedInputStream(in); ByteArrayOutputStream bout = new ByteArrayOutputStream(); int l = 0,len = 0; - byte[] buffer = new byte[256]; + byte[] buffer = new byte[2048]; while ((l = chunkedIn.read(buffer)) >= 0) { len += l; bout.write(buffer,0,l); diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index 69e422221..e204ed2ee 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -63,6 +63,7 @@ import de.anomic.server.serverObjects; import de.anomic.tools.crypt; import de.anomic.tools.nxTools; import de.anomic.yacy.yacySeed; +import de.anomic.yacy.yacyVersion; public class yacyClient { @@ -156,7 +157,7 @@ public class yacyClient { * * @see serverCore#portForwardingEnabled */ - if (!serverCore.portForwardingEnabled || otherPeerVersion >= (float)0.383) { + if (!serverCore.portForwardingEnabled || otherPeerVersion >= yacyVersion.YACY_SUPPORTS_PORT_FORWARDING) { String mytype = (String) result.get("yourtype"); if (mytype == null) { mytype = yacySeed.PEERTYPE_JUNIOR; } if ( @@ -583,6 +584,11 @@ public class yacyClient { // prepare post values final serverObjects post = new serverObjects(); final String key = crypt.randomSalt(); + + // enabling gzip compression for post request body + if (targetSeed.getVersion() >= yacyVersion.YACY_SUPPORTS_GZIP_POST_REQUESTS) { + post.put(httpc.GZIP_POST_BODY,"true"); + } post.put("key", key); post.put("iam", yacyCore.seedDB.mySeed.hash); post.put("youare", targetSeed.hash); @@ -635,6 +641,12 @@ public class yacyClient { // prepare post values final serverObjects post = new serverObjects(); final String key = crypt.randomSalt(); + + // enabling gzip compression for post request body + if (targetSeed.getVersion() >= yacyVersion.YACY_SUPPORTS_GZIP_POST_REQUESTS) { + post.put(httpc.GZIP_POST_BODY,"true"); + } + post.put("key", key); post.put("iam", yacyCore.seedDB.mySeed.hash); post.put("youare", targetSeed.hash); diff --git a/source/de/anomic/yacy/yacyVersion.java b/source/de/anomic/yacy/yacyVersion.java new file mode 100644 index 000000000..8ee1c898b --- /dev/null +++ b/source/de/anomic/yacy/yacyVersion.java @@ -0,0 +1,6 @@ +package de.anomic.yacy; + +public final class yacyVersion { + public static final float YACY_SUPPORTS_PORT_FORWARDING = (float) 0.383; + public static final float YACY_SUPPORTS_GZIP_POST_REQUESTS = (float) 0.40300772; +}