*) Adding support for gzip content-encoding of http post requests

used to transferRWIs and transferURLs.
   See: http://www.yacy-forum.de/viewtopic.php?t=1167#10020

*) adding yacyVersion.java containing constants defining yacy versions
   that support a given feature.
   Needed to determine if a remote peer is able to decode gzip 
   content-encoded http post bodies properly.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@772 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/yacy/hello.java

@@ -55,6 +55,7 @@ import de.anomic.server.serverSwitch;
 import de.anomic.yacy.yacyClient;
 import de.anomic.yacy.yacyCore;
 import de.anomic.yacy.yacySeed;
+import de.anomic.yacy.yacyVersion;
 
 public class hello {
 
@@ -99,7 +100,7 @@ public class hello {
         // if the remote client has reported its own IP address and the client supports
         // the port forwarding feature (if client version >= 0.383) then we try to 
         // connect to the reported IP address first
-        if (reportedip.length() > 0 && !clientip.equals(reportedip) && clientversion >= (float)0.383) {
+        if (reportedip.length() > 0 && !clientip.equals(reportedip) && clientversion >= yacyVersion.YACY_SUPPORTS_PORT_FORWARDING) {
             // try first the reportedip, since this may be a connect from a port-forwarding host
             prop.put(STR_YOURIP, reportedip);
             remoteSeed.put(STR_IP, reportedip);

source/de/anomic/http/httpChunkedInputStream.java

@@ -54,7 +54,7 @@ import java.io.InputStreamReader;
  * Some parts of this class code was copied from <a href="http://www.devdaily.com/java/jwarehouse/commons-httpclient-2.0/src/java/org/apache/commons/httpclient/ChunkedInputStream.shtml">Apache httpclient Project.</a>
  * @author theli
  */
-public class httpChunkedInputStream extends InputStream {
+public final class httpChunkedInputStream extends InputStream {
     
     private static final int READ_CHUNK_STATE_NORMAL = 0;
     private static final int READ_CHUNK_STATE_CR_READ = 1;
@@ -64,7 +64,7 @@ public class httpChunkedInputStream extends InputStream {
     private static final char CR = '\r';
     private static final char LF = '\n';
     
-    private InputStream inputStream;
+    private final InputStream inputStream;
     private int currPos;
     private int currChunkSize;
     private httpHeader httpTrailer;
@@ -97,9 +97,6 @@ public class httpChunkedInputStream extends InputStream {
     
     
     public int read (byte[] b, int off, int len) throws IOException {
-        
-        if (b == null) throw new IllegalArgumentException("bytearry parameter must not be null");
-        
         if (this.isClosed) throw new IOException("Inputstream already closed.");
         if (this.isEOF) return -1;
         
@@ -114,7 +111,6 @@ public class httpChunkedInputStream extends InputStream {
     }
     
     public int read (byte[] b) throws IOException {
-        if (b == null) throw new IllegalArgumentException("bytearry parameter must not be null");
         return read(b, 0, b.length);
     }
     

source/de/anomic/http/httpc.java

@@ -66,6 +66,7 @@ import java.util.Locale;
 import java.util.TimeZone;
 import java.util.Vector;
 import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
 
 import javax.net.ssl.SSLSocketFactory;
 
@@ -87,6 +88,14 @@ import org.apache.commons.pool.impl.GenericObjectPool;
 */
 public final class httpc {
 
+    // some constants
+    /** 
+     * Specifies that the httpc is allowed to use gzip content encoding for
+     * http post requests 
+     * @see #POST(String, httpHeader, serverObjects, Hashtable)
+     */
+    public static final String GZIP_POST_BODY = "GZIP_POST_BODY";
+    
     // statics
     private static final String vDATE = "20040602";
     private static String userAgent;
@@ -757,7 +766,18 @@ public final class httpc {
         }
         boundary = "--" + boundary.substring(pos + "boundary=".length());
 
+        boolean zipContent = args.containsKey(GZIP_POST_BODY);
+        
+        OutputStream out;
+        GZIPOutputStream zippedOut;
         ByteArrayOutputStream buf = new ByteArrayOutputStream();
+        if (zipContent) {
+            zippedOut = new GZIPOutputStream(buf);
+            out = zippedOut;
+        } else {
+            out = buf;
+        }
+        
         // in contrast to GET and HEAD, this method also transports a message body
         // the body consists of repeated boundaries and values in between
         if (args.size() != 0) {
@@ -766,38 +786,42 @@ public final class httpc {
             Enumeration e = args.keys();
             while (e.hasMoreElements()) {
                 // start with a boundary
-                buf.write(boundary.getBytes());
-                buf.write(serverCore.crlf);
+                out.write(boundary.getBytes());
+                out.write(serverCore.crlf);
                 // write value
                 key = (String) e.nextElement();
                 value = (String) args.get(key, "");
                 if ((files != null) && (files.containsKey(key))) {
                     // we are about to write a file
-                    buf.write(("Content-Disposition: form-data; name=" + '"' + key + '"' + "; filename=" + '"' + value + '"').getBytes());
-                    buf.write(serverCore.crlf);
-                    buf.write(serverCore.crlf);
-                    buf.write((byte[]) files.get(key));
-                    buf.write(serverCore.crlf);
+                    out.write(("Content-Disposition: form-data; name=" + '"' + key + '"' + "; filename=" + '"' + value + '"').getBytes());
+                    out.write(serverCore.crlf);
+                    out.write(serverCore.crlf);
+                    out.write((byte[]) files.get(key));
+                    out.write(serverCore.crlf);
                 } else {
                     // write a single value
-                    buf.write(("Content-Disposition: form-data; name=" + '"' + key + '"').getBytes());
-                    buf.write(serverCore.crlf);
-                    buf.write(serverCore.crlf);
-                    buf.write(value.getBytes());
-                    buf.write(serverCore.crlf);
+                    out.write(("Content-Disposition: form-data; name=" + '"' + key + '"').getBytes());
+                    out.write(serverCore.crlf);
+                    out.write(serverCore.crlf);
+                    out.write(value.getBytes());
+                    out.write(serverCore.crlf);
                 }
             }
             // finish with a boundary
-            buf.write(boundary.getBytes());
-            buf.write(serverCore.crlf);
+            out.write(boundary.getBytes());
+            out.write(serverCore.crlf);
             //buf.write("" + serverCore.crlfString);
         }
         // create body array
-        buf.close();
+        out.close();
         byte[] body = buf.toByteArray();
         //System.out.println("DEBUG: PUT BODY=" + new String(body));
+        if (zipContent) {
+            requestHeader.put(httpHeader.CONTENT_ENCODING, "gzip");
+        } else {
             // size of that body            
             requestHeader.put(httpHeader.CONTENT_LENGTH, Integer.toString(body.length));
+        }
         // send the header
         //System.out.println("header=" + requestHeader);
         send(httpHeader.METHOD_POST, path, requestHeader, false);

source/de/anomic/http/httpd.java

@@ -46,6 +46,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.PrintStream;
 import java.io.PushbackInputStream;
@@ -666,11 +667,23 @@ public final class httpd implements serverHandler {
     // but this belongs to the protocol handler, this class.
     
     
-    public static int parseArgs(serverObjects args, PushbackInputStream in, int length) throws IOException {
+    public static int parseArgs(serverObjects args, InputStream in, int length) throws IOException {
         // this is a quick hack using a previously coded parseMultipart based on a buffer
         // should be replaced sometime by a 'right' implementation
-        byte[] buffer = new byte[length];
+        byte[] buffer = null;
+        
+        // parsing post request bodies with a given length
+        if (length != -1) {
+            buffer = new byte[length];
             in.read(buffer);
+        // parsing post request bodies which are gzip content-encoded
+        } else {
+            ByteArrayOutputStream bout = new ByteArrayOutputStream();
+            serverFileUtils.copy(in,bout);
+            buffer = bout.toByteArray();
+            bout.close(); bout = null;
+        }
+        
         int argc = parseArgs(args, new String(buffer));
         buffer = null;
         return argc;
@@ -722,16 +735,29 @@ public final class httpd implements serverHandler {
     }
     
     
-    public static HashMap parseMultipart(httpHeader header, serverObjects args, PushbackInputStream in, int length) throws IOException {
+    public static HashMap parseMultipart(httpHeader header, serverObjects args, InputStream in, int length) throws IOException {
         // this is a quick hack using a previously coded parseMultipart based on a buffer
         // should be replaced sometime by a 'right' implementation
-        byte[] buffer = new byte[length];
+
+        byte[] buffer = null;
+        
+        // parsing post request bodies with a given length
+        if (length != -1) { 
+            buffer = new byte[length];
             int c, a = 0;
             while (a < length) {
                 c = in.read(buffer, a, length - a);
                 if (c <= 0) break;
                 a += c;
             }
+        // parsing post request bodies which are gzip content-encoded
+        } else {
+            ByteArrayOutputStream bout = new ByteArrayOutputStream();
+            serverFileUtils.copy(in,bout);
+            buffer = bout.toByteArray();
+            bout.close(); bout = null;
+        }
+        
         //System.out.println("MULTIPART-BUFFER=" + new String(buffer));
         HashMap files = parseMultipart(header, args, buffer);
         buffer = null;

source/de/anomic/http/httpdFileHandler.java

@@ -96,6 +96,7 @@ import java.util.Iterator;
 import java.util.Map;
 import java.util.Properties;
 import java.util.logging.Level;
+import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 
 import de.anomic.plasma.plasmaSwitchboard;
@@ -240,7 +241,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
         doResponse(conProp, requestHeader, response, body);
     }
     
-    public void doResponse(Properties conProp, httpHeader requestHeader, OutputStream out, PushbackInputStream body) throws IOException {
+    public void doResponse(Properties conProp, httpHeader requestHeader, OutputStream out, InputStream body) throws IOException {
         
         this.connectionProperties = conProp;
         
@@ -303,10 +304,19 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
             // no args here, maybe a POST with multipart extension
             int length;
             //System.out.println("HEADER: " + requestHeader.toString()); // DEBUG
-            if ((method.equals(httpHeader.METHOD_POST)) &&
-                    (requestHeader.containsKey(httpHeader.CONTENT_LENGTH))) {
-                // if its a POST, it can be either multipart or as args in the body
+            if (method.equals(httpHeader.METHOD_POST)) {
+
+                if (requestHeader.containsKey(httpHeader.CONTENT_LENGTH)) {
                     length = Integer.parseInt((String) requestHeader.get(httpHeader.CONTENT_LENGTH));
+                } else if (requestHeader.gzip()) {
+                    length = -1;
+                    body = new GZIPInputStream(body);
+                } else {
+                    httpd.sendRespondError(conProp,out,4,403,null,"bad post values",null); 
+                    return;
+                }
+                
+                // if its a POST, it can be either multipart or as args in the body
                 if ((requestHeader.containsKey(httpHeader.CONTENT_TYPE)) &&
                         (((String) requestHeader.get(httpHeader.CONTENT_TYPE)).toLowerCase().startsWith("multipart"))) {
                     // parse multipart

source/de/anomic/icap/icapd.java

@@ -286,7 +286,7 @@ public class icapd implements serverHandler {
                     httpChunkedInputStream chunkedIn = new httpChunkedInputStream(in);
                     ByteArrayOutputStream bout = new ByteArrayOutputStream();
                     int l = 0,len = 0;
-                    byte[] buffer = new byte[256];
+                    byte[] buffer = new byte[2048];
                     while ((l = chunkedIn.read(buffer)) >= 0) {
                         len += l;
                         bout.write(buffer,0,l);

source/de/anomic/yacy/yacyClient.java

@@ -63,6 +63,7 @@ import de.anomic.server.serverObjects;
 import de.anomic.tools.crypt;
 import de.anomic.tools.nxTools;
 import de.anomic.yacy.yacySeed;
+import de.anomic.yacy.yacyVersion;
 
 public class yacyClient {
 
@@ -156,7 +157,7 @@ public class yacyClient {
          * 
          * @see serverCore#portForwardingEnabled 
          */
-        if (!serverCore.portForwardingEnabled || otherPeerVersion >= (float)0.383) {
+        if (!serverCore.portForwardingEnabled || otherPeerVersion >= yacyVersion.YACY_SUPPORTS_PORT_FORWARDING) {
             String mytype = (String) result.get("yourtype");
             if (mytype == null) { mytype = yacySeed.PEERTYPE_JUNIOR; }        
             if (
@@ -583,6 +584,11 @@ public class yacyClient {
         // prepare post values
         final serverObjects post = new serverObjects();
         final String key = crypt.randomSalt();
+        
+        // enabling gzip compression for post request body
+        if (targetSeed.getVersion() >= yacyVersion.YACY_SUPPORTS_GZIP_POST_REQUESTS) {
+            post.put(httpc.GZIP_POST_BODY,"true");
+        }
         post.put("key", key);
         post.put("iam", yacyCore.seedDB.mySeed.hash);
         post.put("youare", targetSeed.hash);
@@ -635,6 +641,12 @@ public class yacyClient {
         // prepare post values
         final serverObjects post = new serverObjects();
         final String key = crypt.randomSalt();
+        
+        // enabling gzip compression for post request body
+        if (targetSeed.getVersion() >= yacyVersion.YACY_SUPPORTS_GZIP_POST_REQUESTS) {
+            post.put(httpc.GZIP_POST_BODY,"true");
+        }        
+        
         post.put("key", key);
         post.put("iam", yacyCore.seedDB.mySeed.hash);
         post.put("youare", targetSeed.hash);

source/de/anomic/yacy/yacyVersion.java

@@ -0,0 +1,6 @@
+package de.anomic.yacy;
+
+public final class yacyVersion {    
+    public static final float YACY_SUPPORTS_PORT_FORWARDING = (float) 0.383;
+    public static final float YACY_SUPPORTS_GZIP_POST_REQUESTS = (float) 0.40300772;
+}