diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index 2b5095962..1742d2572 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -1005,7 +1005,7 @@ public final class yacyClient { parts.put("wordc", new StringBody(Integer.toString(indexes.size()))); parts.put("entryc", new StringBody(Integer.toString(indexcount))); parts.put("indexes", new StringBody(entrypost.toString())); - final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/transferRWI.html"), timeout, targetSeed.getHexHash() + ".yacyh", parts); + final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/transferRWI.html"), timeout, targetSeed.getHexHash() + ".yacyh", parts, gzipBody); final Iterator v = FileUtils.strings(content); // this should return a list of urlhashes that are unknown @@ -1049,7 +1049,7 @@ public final class yacyClient { } try { parts.put("urlc", new StringBody(Integer.toString(urlc))); - final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/transferURL.html"), timeout, targetSeed.getHexHash() + ".yacyh", parts); + final byte[] content = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + address + "/yacy/transferURL.html"), timeout, targetSeed.getHexHash() + ".yacyh", parts, gzipBody); final Iterator v = FileUtils.strings(content); final Map result = FileUtils.table(v); @@ -1161,7 +1161,7 @@ public final class yacyClient { } byte[] res; try { - res = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(url, timeout, vhost, newpost); + res = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(url, timeout, vhost, newpost, true); System.out.println(new String(res)); } catch (IOException e1) { Log.logException(e1); diff --git a/source/net/yacy/cora/protocol/http/GzipCompressingEntity.java b/source/net/yacy/cora/protocol/http/GzipCompressingEntity.java new file mode 100644 index 000000000..29a4246ea --- /dev/null +++ b/source/net/yacy/cora/protocol/http/GzipCompressingEntity.java @@ -0,0 +1,69 @@ +/** + * GzipCompressingEntity + * Copyright 2010 by Sebastian Gaebel + * First released 01.07.2010 at http://yacy.net + * + * $LastChangedDate: 2010-06-16 17:11:21 +0200 (Mi, 16 Jun 2010) $ + * $LastChangedRevision: 6922 $ + * $LastChangedBy: sixcooler $ + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see . + */ + + +package net.yacy.cora.protocol.http; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.zip.GZIPOutputStream; + +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.entity.HttpEntityWrapper; +import org.apache.http.message.BasicHeader; +import org.apache.http.protocol.HTTP; + +public class GzipCompressingEntity extends HttpEntityWrapper { + + private static final String GZIP_CODEC = "gzip"; +// private static final int DEFAULT_BUFFER_SIZE = 1024; // this is also the maximum chunk size + + public GzipCompressingEntity(final HttpEntity entity) { + super(entity); + } + + public Header getContentEncoding() { + return new BasicHeader(HTTP.CONTENT_ENCODING, GZIP_CODEC); + } + + public long getContentLength() { + return -1; + } + + public boolean isChunked() { + // force content chunking + return true; + } + + public void writeTo(final OutputStream outstream) throws IOException { + if (outstream == null) { + throw new IllegalArgumentException("Output stream may not be null"); + } + GZIPOutputStream gzip = new GZIPOutputStream(outstream); + wrappedEntity.writeTo(gzip); + gzip.finish(); + } + +} diff --git a/source/net/yacy/cora/protocol/http/HTTPClient.java b/source/net/yacy/cora/protocol/http/HTTPClient.java index b1ff81734..0c26a253a 100644 --- a/source/net/yacy/cora/protocol/http/HTTPClient.java +++ b/source/net/yacy/cora/protocol/http/HTTPClient.java @@ -337,7 +337,7 @@ public class HTTPClient { * @return content bytes * @throws IOException */ - public byte[] POSTbytes(final String uri, final LinkedHashMap parts) throws IOException { + public byte[] POSTbytes(final String uri, final LinkedHashMap parts, final boolean usegzip) throws IOException { final HttpPost httpPost = new HttpPost(uri); final MultipartEntity multipartEntity = new MultipartEntity(); @@ -346,7 +346,11 @@ public class HTTPClient { // statistics upbytes = multipartEntity.getContentLength(); - httpPost.setEntity(multipartEntity); + if (usegzip) { + httpPost.setEntity(new GzipCompressingEntity(multipartEntity)); + } else { + httpPost.setEntity(multipartEntity); + } return getContentBytes(httpPost, Long.MAX_VALUE); } diff --git a/source/net/yacy/cora/protocol/http/HTTPConnector.java b/source/net/yacy/cora/protocol/http/HTTPConnector.java index 66707c620..bff6439de 100644 --- a/source/net/yacy/cora/protocol/http/HTTPConnector.java +++ b/source/net/yacy/cora/protocol/http/HTTPConnector.java @@ -53,19 +53,34 @@ public class HTTPConnector { * send data to the server named by vhost * * @param url address of the server + * @param timeout in milliseconds * @param vhost name of the server at address which should respond * @param post data to send (name-value-pairs) - * @param timeout in milliseconds * @return response body * @throws IOException */ public byte[] post(final MultiProtocolURI url, final int timeout, final String vhost, LinkedHashMap post) throws IOException { + return post(url, timeout, vhost, post, false); + } + + /** + * send data to the server named by vhost + * + * @param url address of the server + * @param timeout in milliseconds + * @param vhost name of the server at address which should respond + * @param post data to send (name-value-pairs) + * @param usegzip if the body should be gzipped + * @return response body + * @throws IOException + */ + public byte[] post(final MultiProtocolURI url, final int timeout, final String vhost, LinkedHashMap post, final boolean usegzip) throws IOException { final HTTPClient client = new HTTPClient(); client.setTimout(timeout); client.setUserAgent(this.userAgent); client.setHost(vhost); - return client.POSTbytes(url.toNormalform(false, false), post); + return client.POSTbytes(url.toNormalform(false, false), post, usegzip); } }