gzip-compresson @ transferRWI & transferURL back again

This reduce upload-volume to suit limited bandwidth of home-users like me :-)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7215 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/yacy/yacyClient.java

@@ -1005,7 +1005,7 @@ public final class yacyClient {
             parts.put("wordc", new StringBody(Integer.toString(indexes.size())));
             parts.put("entryc", new StringBody(Integer.toString(indexcount)));
             parts.put("indexes", new StringBody(entrypost.toString()));
-            final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/transferRWI.html"), timeout, targetSeed.getHexHash() + ".yacyh", parts);
+            final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/transferRWI.html"), timeout, targetSeed.getHexHash() + ".yacyh", parts, gzipBody);
             final Iterator<String> v = FileUtils.strings(content);
             // this should return a list of urlhashes that are unknown
             
@@ -1049,7 +1049,7 @@ public final class yacyClient {
         }
         try {
             parts.put("urlc", new StringBody(Integer.toString(urlc)));
-            final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/transferURL.html"), timeout, targetSeed.getHexHash() + ".yacyh", parts);
+            final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/transferURL.html"), timeout, targetSeed.getHexHash() + ".yacyh", parts, gzipBody);
             final Iterator<String> v = FileUtils.strings(content);
             
             final Map<String, String> result = FileUtils.table(v);
@@ -1161,7 +1161,7 @@ public final class yacyClient {
 			}
 			byte[] res;
 			try {
-				res = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(url, timeout, vhost, newpost);
+				res = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(url, timeout, vhost, newpost, true);
 				System.out.println(new String(res));
 			} catch (IOException e1) {
 				Log.logException(e1);

source/net/yacy/cora/protocol/http/GzipCompressingEntity.java

@@ -0,0 +1,69 @@
+/**
+ *  GzipCompressingEntity
+ *  Copyright 2010 by Sebastian Gaebel
+ *  First released 01.07.2010 at http://yacy.net
+ *  
+ *  $LastChangedDate: 2010-06-16 17:11:21 +0200 (Mi, 16 Jun 2010) $
+ *  $LastChangedRevision: 6922 $
+ *  $LastChangedBy: sixcooler $
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *  
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  Lesser General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU Lesser General Public License
+ *  along with this program in the file lgpl21.txt
+ *  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+package net.yacy.cora.protocol.http;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.zip.GZIPOutputStream;
+
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.entity.HttpEntityWrapper;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.protocol.HTTP;
+
+public class GzipCompressingEntity extends HttpEntityWrapper {
+
+	private static final String GZIP_CODEC = "gzip";
+//	private static final int DEFAULT_BUFFER_SIZE = 1024; // this is also the maximum chunk size
+
+	public GzipCompressingEntity(final HttpEntity entity) {
+		super(entity);
+	}
+
+	public Header getContentEncoding() {
+		return new BasicHeader(HTTP.CONTENT_ENCODING, GZIP_CODEC);
+	}
+
+	public long getContentLength() {
+		return -1;
+	}
+
+	public boolean isChunked() {
+		// force content chunking
+		return true;
+	}
+
+	public void writeTo(final OutputStream outstream) throws IOException {
+		if (outstream == null) {
+			throw new IllegalArgumentException("Output stream may not be null");
+		}
+		GZIPOutputStream gzip = new GZIPOutputStream(outstream);
+		wrappedEntity.writeTo(gzip);
+		gzip.finish();
+	}
+
+}

source/net/yacy/cora/protocol/http/HTTPClient.java

@@ -337,7 +337,7 @@ public class HTTPClient {
      * @return content bytes
      * @throws IOException 
      */
-	public byte[] POSTbytes(final String uri, final LinkedHashMap<String, ContentBody> parts) throws IOException {
+	public byte[] POSTbytes(final String uri, final LinkedHashMap<String, ContentBody> parts, final boolean usegzip) throws IOException {
     	final HttpPost httpPost = new HttpPost(uri);
 
     	final MultipartEntity multipartEntity = new MultipartEntity();
@@ -346,7 +346,11 @@ public class HTTPClient {
     	// statistics
     	upbytes = multipartEntity.getContentLength();
 
-    	httpPost.setEntity(multipartEntity);
+    	if (usegzip) {
+    		httpPost.setEntity(new GzipCompressingEntity(multipartEntity));
+    	} else {
+    		httpPost.setEntity(multipartEntity);
+    	}
     	
     	return getContentBytes(httpPost, Long.MAX_VALUE);
     }

source/net/yacy/cora/protocol/http/HTTPConnector.java

@@ -53,19 +53,34 @@ public class HTTPConnector {
      * send data to the server named by vhost
      * 
      * @param url address of the server
+     * @param timeout in milliseconds
      * @param vhost name of the server at address which should respond
      * @param post data to send (name-value-pairs)
-     * @param timeout in milliseconds
      * @return response body
      * @throws IOException
      */
     public byte[] post(final MultiProtocolURI url, final int timeout, final String vhost, LinkedHashMap<String, ContentBody> post) throws IOException {
+		return post(url, timeout, vhost, post, false);
+	}
+    
+    /**
+     * send data to the server named by vhost
+     * 
+     * @param url address of the server
+     * @param timeout in milliseconds
+     * @param vhost name of the server at address which should respond
+     * @param post data to send (name-value-pairs)
+     * @param usegzip if the body should be gzipped
+     * @return response body
+     * @throws IOException
+     */
+    public byte[] post(final MultiProtocolURI url, final int timeout, final String vhost, LinkedHashMap<String, ContentBody> post, final boolean usegzip) throws IOException {
 		final HTTPClient client = new HTTPClient();
 		client.setTimout(timeout);
 		client.setUserAgent(this.userAgent);
 		client.setHost(vhost);
 		
-		return client.POSTbytes(url.toNormalform(false, false), post);
+		return client.POSTbytes(url.toNormalform(false, false), post, usegzip);
 	}
 
 }