boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

re-added gzip POST for transferRWI/URL (HTTP/1.1 compliant)

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4761 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
danielr 17 years ago
parent a5a1f19368
commit d7b21bc90c
5 changed files with 191 additions and 104 deletions
Show Stats Download Patch File Download Diff File

169
source/de/anomic/http/JakartaCommonsHttpClient.java
Unescape View File

@ -25,11 +25,14 @@
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.http; package de.anomic.http;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
import java.util.zip.GZIPOutputStream;
import org.apache.commons.httpclient.ConnectMethod; import org.apache.commons.httpclient.ConnectMethod;
import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler; import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
@ -44,6 +47,7 @@ import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.HeadMethod; import org.apache.commons.httpclient.methods.HeadMethod;
import org.apache.commons.httpclient.methods.InputStreamRequestEntity; import org.apache.commons.httpclient.methods.InputStreamRequestEntity;
import org.apache.commons.httpclient.methods.PostMethod; import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.RequestEntity;
import org.apache.commons.httpclient.methods.multipart.ByteArrayPartSource; import org.apache.commons.httpclient.methods.multipart.ByteArrayPartSource;
import org.apache.commons.httpclient.methods.multipart.FilePart; import org.apache.commons.httpclient.methods.multipart.FilePart;
import org.apache.commons.httpclient.methods.multipart.MultipartRequestEntity; import org.apache.commons.httpclient.methods.multipart.MultipartRequestEntity;
@ -77,14 +81,11 @@ public class JakartaCommonsHttpClient {
*/ */
// set user-agent // set user-agent
final yacyVersion thisversion = yacyVersion.thisVersion(); final yacyVersion thisversion = yacyVersion.thisVersion();
apacheHttpClient.getParams().setParameter( final String userAgent = "yacy/" + ((thisversion == null) ? "0.0" : thisversion.releaseNr) +
HttpMethodParams.USER_AGENT, " (www.yacy.net; " + de.anomic.http.HttpClient.getSystemOST() + ") " +
"yacy/" + ((thisversion == null) ? "0.0" : thisversion.releaseNr) + // last ; must be before location (this is parsed)
" (www.yacy.net; " + getCurrentUserAgent().replace(';', ':');
de.anomic.http.HttpClient.getSystemOST() + ") " + apacheHttpClient.getParams().setParameter(HttpMethodParams.USER_AGENT, userAgent);
getCurrentUserAgent().replace(';', ':')); // last ; must be
// before location
// (this is parsed)
// only one retry // only one retry
apacheHttpClient.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, apacheHttpClient.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
new DefaultHttpMethodRetryHandler(1, false)); new DefaultHttpMethodRetryHandler(1, false));
@ -116,15 +117,15 @@ public class JakartaCommonsHttpClient {
// after a connection is established with a resource // after a connection is established with a resource
System.setProperty("sun.net.client.defaultReadTimeout", "60000"); System.setProperty("sun.net.client.defaultReadTimeout", "60000");
} }
/** /**
* every x milliseconds do a cleanup (close old connections) * every x milliseconds do a cleanup (close old connections)
*/ */
private static final int cleanupIntervall = 60000; private static final int cleanupIntervall = 60000;
/** /**
* close connections when they are not used for this time * close connections when they are not used for this time
* or otherwise: *
* hold connections this time open to reuse them * or otherwise: hold connections this time open to reuse them
*/ */
private static final long closeConnectionsAfterMillis = 120000; private static final long closeConnectionsAfterMillis = 120000;
/** /**
@ -167,7 +168,7 @@ public class JakartaCommonsHttpClient {
* @see de.anomic.http.HttpClient#setHeader(de.anomic.http.httpHeader) * @see de.anomic.http.HttpClient#setHeader(de.anomic.http.httpHeader)
*/ */
public void setHeader(final httpHeader header) { public void setHeader(final httpHeader header) {
headers = convertHeaders(header); this.headers = convertHeaders(header);
} }
/* /*
@ -185,7 +186,7 @@ public class JakartaCommonsHttpClient {
* @param follow * @param follow
*/ */
public void setFollowRedirects(final boolean follow) { public void setFollowRedirects(final boolean follow) {
followRedirects = follow; this.followRedirects = follow;
} }
/* /*
@ -206,7 +207,7 @@ public class JakartaCommonsHttpClient {
*/ */
public JakartaCommonsHttpResponse GET(final String uri) throws IOException { public JakartaCommonsHttpResponse GET(final String uri) throws IOException {
final HttpMethod get = new GetMethod(uri); final HttpMethod get = new GetMethod(uri);
get.setFollowRedirects(followRedirects); get.setFollowRedirects(this.followRedirects);
return execute(get); return execute(get);
} }
@ -221,7 +222,7 @@ public class JakartaCommonsHttpClient {
public JakartaCommonsHttpResponse HEAD(final String uri) throws IOException { public JakartaCommonsHttpResponse HEAD(final String uri) throws IOException {
assert uri != null : "precondition violated: uri != null"; assert uri != null : "precondition violated: uri != null";
final HttpMethod head = new HeadMethod(uri); final HttpMethod head = new HeadMethod(uri);
head.setFollowRedirects(followRedirects); head.setFollowRedirects(this.followRedirects);
return execute(head); return execute(head);
} }
@ -240,63 +241,63 @@ public class JakartaCommonsHttpClient {
assert ins != null : "precondition violated: ins != null"; assert ins != null : "precondition violated: ins != null";
final PostMethod post = new PostMethod(uri); final PostMethod post = new PostMethod(uri);
post.setRequestEntity(new InputStreamRequestEntity(ins)); post.setRequestEntity(new InputStreamRequestEntity(ins));
post.setFollowRedirects(false); // redirects in POST cause a "Entity enclosing requests cannot be redirected without user intervention" - exception // redirects in POST cause a "Entity enclosing requests cannot be redirected without user intervention" -
// exception
post.setFollowRedirects(false);
return execute(post); return execute(post);
} }
/** /**
* This method sends several files at once via a POST request. Only those files in the Hashtable files are written * This method sends several data at once via a POST request (multipart-message)
* whose names are contained in args. *
* @param uri
* @param multiparts
* @return
* @throws IOException
*/
public JakartaCommonsHttpResponse POST(final String uri, final List<Part> multiparts) throws IOException {
return POST(uri, multiparts, false);
}
/**
* This method sends several data at once via a POST request (multipart-message), maybe compressed
* *
* @param uri The URI to the page which the post is sent to. * @param uri The URI to the page which the post is sent to.
* @param files HashMap with the names of data as key and the content (currently implemented is File, byte[] and * @param multiparts {@link java.util.List} with the {@link Part}s of data
* String) of the files as value. * @param gzipBody should the body be compressed
* @return Instance of response with the content. * @return Instance of response with the content.
* @throws IOException * @throws IOException
*/ */
public JakartaCommonsHttpResponse POST(final String uri, final List<Part> files) throws IOException { public JakartaCommonsHttpResponse POST(final String uri, final List<Part> multiparts, final boolean gzipBody)
throws IOException {
assert uri != null : "precondition violated: uri != null"; assert uri != null : "precondition violated: uri != null";
final PostMethod post = new PostMethod(uri); final PostMethod post = new PostMethod(uri);
final Part[] parts; final Part[] parts;
if (files != null) { if (multiparts != null) {
parts = new Part[files.size()]; parts = multiparts.toArray(new Part[0]);
int i = 0;
for (final Part part : files) {
parts[i] = part;
/*
if (value instanceof File) {
final File file = (File) value;
if (file.isFile() && file.canRead()) {
// read file
final ByteArrayOutputStream fileData = new ByteArrayOutputStream();
serverFileUtils.copyToStream(new BufferedInputStream(new FileInputStream(file)),
new BufferedOutputStream(fileData));
value = fileData.toByteArray();
}
}
if (value instanceof byte[]) {
// file/binary data
parts[i] = new FilePart(key, new ByteArrayPartSource(key, (byte[]) value));
} else if (value instanceof String) {
// simple text value
parts[i] = new StringPart(key, (String) value);
} else {
// not supported
final String msg = "type of POST-data not supported: " + value.getClass();
serverLog.logSevere("HTTPC", msg);
throw new IOException("cannot POST data: " + msg);
// break; // post nothing is not what is expected by the caller
}
*/
i++;
}
} else { } else {
// nothing to POST // nothing to POST
parts = new Part[0]; parts = new Part[0];
} }
post.setRequestEntity(new MultipartRequestEntity(parts, post.getParams())); RequestEntity data = new MultipartRequestEntity(parts, post.getParams());
post.setFollowRedirects(false); // redirects in POST cause a "Entity enclosing requests cannot be redirected without user intervention" - exception if (gzipBody) {
// cache data and gzip it
final ByteArrayOutputStream zippedBytes = new ByteArrayOutputStream();
final GZIPOutputStream toZip = new GZIPOutputStream(zippedBytes);
data.writeRequest(toZip);
toZip.finish();
toZip.flush();
// use compressed data as body (not setting content length according to RFC 2616 HTTP/1.1, section 4.4)
data = new InputStreamRequestEntity(new ByteArrayInputStream(zippedBytes.toByteArray()), -1, data
.getContentType());
post.setRequestHeader(httpHeader.CONTENT_ENCODING, httpHeader.CONTENT_ENCODING_GZIP);
}
post.setRequestEntity(data);
// redirects in POST cause a "Entity enclosing requests cannot be redirected without user intervention" -
// exception
post.setFollowRedirects(false);
return execute(post); return execute(post);
} }
@ -375,18 +376,18 @@ public class JakartaCommonsHttpClient {
private JakartaCommonsHttpResponse execute(final HttpMethod method) throws IOException, HttpException { private JakartaCommonsHttpResponse execute(final HttpMethod method) throws IOException, HttpException {
assert method != null : "precondition violated: method != null"; assert method != null : "precondition violated: method != null";
// set header // set header
for (final Header header : headers) { for (final Header header : this.headers) {
method.setRequestHeader(header); method.setRequestHeader(header);
} }
// set proxy // set proxy
final httpRemoteProxyConfig proxyConfig = getProxyConfig(method.getURI().getHost()); final httpRemoteProxyConfig hostProxyConfig = getProxyConfig(method.getURI().getHost());
addProxyAuth(method, proxyConfig); addProxyAuth(method, hostProxyConfig);
final HostConfiguration hostConfig = getProxyHostConfig(proxyConfig); final HostConfiguration hostConfig = getProxyHostConfig(hostProxyConfig);
// statistics // statistics
HttpConnectionInfo.addConnection(generateConInfo(method)); HttpConnectionInfo.addConnection(generateConInfo(method));
// execute (send request) // execute (send request)
try { try {
if (hostConfig == null) { if (hostConfig == null) {
@ -394,12 +395,12 @@ public class JakartaCommonsHttpClient {
} else { } else {
apacheHttpClient.executeMethod(hostConfig, method); apacheHttpClient.executeMethod(hostConfig, method);
} }
} catch (IOException e) { } catch (final IOException e) {
// cleanUp statistics // cleanUp statistics
HttpConnectionInfo.removeConnection(generateConInfo(method)); HttpConnectionInfo.removeConnection(generateConInfo(method));
throw e; throw e;
} }
// return response // return response
return new JakartaCommonsHttpResponse(method); return new JakartaCommonsHttpResponse(method);
} }
@ -420,24 +421,24 @@ public class JakartaCommonsHttpClient {
// should not happen, because method is already executed // should not happen, because method is already executed
} }
final String query = (method.getQueryString() != null) ? "?" + method.getQueryString() : ""; final String query = (method.getQueryString() != null) ? "?" + method.getQueryString() : "";
return new HttpConnectionInfo(protocol, (port == -1 || port == 80) ? host : host + ":" + port, method.getName() + " " + return new HttpConnectionInfo(protocol, (port == -1 || port == 80) ? host : host + ":" + port,
method.getPath() + query, method.hashCode(), System.currentTimeMillis()); method.getName() + " " + method.getPath() + query, method.hashCode(), System.currentTimeMillis());
} }
/** /**
* if necessary adds a header for proxy-authentication * if necessary adds a header for proxy-authentication
* *
* @param method * @param method
* @param proxyConfig * @param hostProxyConfig
*/ */
private void addProxyAuth(final HttpMethod method, final httpRemoteProxyConfig proxyConfig) { private void addProxyAuth(final HttpMethod method, final httpRemoteProxyConfig hostProxyConfig) {
if (proxyConfig != null && proxyConfig.useProxy()) { if (hostProxyConfig != null && hostProxyConfig.useProxy()) {
final String remoteProxyUser = proxyConfig.getProxyUser(); final String remoteProxyUser = hostProxyConfig.getProxyUser();
if (remoteProxyUser != null && remoteProxyUser.length() > 0) { if (remoteProxyUser != null && remoteProxyUser.length() > 0) {
if (remoteProxyUser.contains(":")) { if (remoteProxyUser.contains(":")) {
serverLog.logWarning("HTTPC", "Proxy authentication contains invalid characters, trying anyway"); serverLog.logWarning("HTTPC", "Proxy authentication contains invalid characters, trying anyway");
} }
final String remoteProxyPwd = proxyConfig.getProxyPwd(); final String remoteProxyPwd = hostProxyConfig.getProxyPwd();
final String credentials = kelondroBase64Order.standardCoder.encodeString(remoteProxyUser.replace(":", final String credentials = kelondroBase64Order.standardCoder.encodeString(remoteProxyUser.replace(":",
"") + "") +
":" + remoteProxyPwd); ":" + remoteProxyPwd);
@ -452,28 +453,28 @@ public class JakartaCommonsHttpClient {
* @return * @return
*/ */
private httpRemoteProxyConfig getProxyConfig(final String hostname) { private httpRemoteProxyConfig getProxyConfig(final String hostname) {
final httpRemoteProxyConfig proxyConfig; final httpRemoteProxyConfig hostProxyConfig;
if (this.proxyConfig != null) { if (this.proxyConfig != null) {
// client specific // client specific
proxyConfig = httpdProxyHandler.getProxyConfig(hostname, this.proxyConfig); hostProxyConfig = httpdProxyHandler.getProxyConfig(hostname, this.proxyConfig);
} else { } else {
// default settings // default settings
proxyConfig = httpdProxyHandler.getProxyConfig(hostname, 0); hostProxyConfig = httpdProxyHandler.getProxyConfig(hostname, 0);
} }
return proxyConfig; return hostProxyConfig;
} }
/** /**
* @param proxyConfig * @param hostProxyConfig
* @return current host-config with additional proxy set or null if no proxy should be used * @return current host-config with additional proxy set or null if no proxy should be used
*/ */
private HostConfiguration getProxyHostConfig(final httpRemoteProxyConfig proxyConfig) { private HostConfiguration getProxyHostConfig(final httpRemoteProxyConfig hostProxyConfig) {
// generate http-configuration // generate http-configuration
if (proxyConfig != null && proxyConfig.useProxy()) { if (hostProxyConfig != null && hostProxyConfig.useProxy()) {
// new config based on client (default) // new config based on client (default)
final HostConfiguration hostConfig = new HostConfiguration(apacheHttpClient.getHostConfiguration()); final HostConfiguration hostConfig = new HostConfiguration(apacheHttpClient.getHostConfiguration());
// add proxy // add proxy
hostConfig.setProxy(proxyConfig.getProxyHost(), proxyConfig.getProxyPort()); hostConfig.setProxy(hostProxyConfig.getProxyHost(), hostProxyConfig.getProxyPort());
return hostConfig; return hostConfig;
} else { } else {
return null; return null;
@ -526,8 +527,10 @@ public class JakartaCommonsHttpClient {
if (args.length > 1 && "post".equals(args[1])) { if (args.length > 1 && "post".equals(args[1])) {
// POST // POST
final ArrayList<Part> files = new ArrayList<Part>(); final ArrayList<Part> files = new ArrayList<Part>();
files.add(new FilePart("myfile.txt", new ByteArrayPartSource("myfile.txt", "this is not a file ;)".getBytes()))); files.add(new FilePart("myfile.txt", new ByteArrayPartSource("myfile.txt", "this is not a file ;)"
files.add(new FilePart("anotherfile.raw", new ByteArrayPartSource("anotherfile.raw", "this is not a binary file ;)".getBytes()))); .getBytes())));
files.add(new FilePart("anotherfile.raw", new ByteArrayPartSource("anotherfile.raw",
"this is not a binary file ;)".getBytes())));
System.out.println("POST " + files.size() + " elements to " + url); System.out.println("POST " + files.size() + " elements to " + url);
final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(1000, null, null); final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(1000, null, null);
resp = client.POST(url, files); resp = client.POST(url, files);
@ -577,8 +580,8 @@ public class JakartaCommonsHttpClient {
*/ */
public static void cleanup() { public static void cleanup() {
// do it only once a while // do it only once a while
final long now = System.currentTimeMillis(); final long now = System.currentTimeMillis();
if(now - lastCleanup > cleanupIntervall) { if (now - lastCleanup > cleanupIntervall) {
lastCleanup = now; lastCleanup = now;
conManager.closeIdleConnections(closeConnectionsAfterMillis); conManager.closeIdleConnections(closeConnectionsAfterMillis);
conManager.deleteClosedConnections(); conManager.deleteClosedConnections();

31
source/de/anomic/http/httpd.java
Unescape View File

@ -64,6 +64,8 @@ import java.util.Iterator;
import java.util.Properties; import java.util.Properties;
import java.util.StringTokenizer; import java.util.StringTokenizer;
import org.apache.commons.httpclient.ChunkedInputStream;
import de.anomic.data.htmlTools; import de.anomic.data.htmlTools;
import de.anomic.data.userDB; import de.anomic.data.userDB;
import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroBase64Order;
@ -599,6 +601,26 @@ public final class httpd implements serverHandler {
if (httpVersion.equals(httpHeader.HTTP_VERSION_0_9)) header = new httpHeader(reverseMappingCache); if (httpVersion.equals(httpHeader.HTTP_VERSION_0_9)) header = new httpHeader(reverseMappingCache);
else header = httpHeader.readHeader(this.prop,this.session); else header = httpHeader.readHeader(this.prop,this.session);
// handle transfer-coding
final InputStream sessionIn;
final String transferEncoding = header.get(httpHeader.TRANSFER_ENCODING);
if (transferEncoding != null) {
if (!httpHeader.HTTP_VERSION_1_1.equals(httpVersion)) {
this.log.logWarning("client "+ session.getName() +" uses transfer-coding with HTTP version "+ httpVersion +"!");
}
if("chunked".equalsIgnoreCase(header.get(httpHeader.TRANSFER_ENCODING))) {
sessionIn = new ChunkedInputStream(this.session.in);
} else {
// "A server which receives an entity-body with a transfer-coding it does
// not understand SHOULD return 501 (Unimplemented), and close the
// connection." [RFC 2616, section 3.6]
session.out.write((httpVersion + " 501 transfer-encoding not implemented" + serverCore.CRLF_STRING + serverCore.CRLF_STRING + "you send a transfer-encoding to this server, which is not supported: " + transferEncoding + serverCore.CRLF_STRING).getBytes());
return serverCore.TERMINATE_CONNECTION;
}
} else {
sessionIn = this.session.in;
}
// handle transparent proxy support // handle transparent proxy support
httpHeader.handleTransparentProxySupport(header, this.prop, virtualHost, httpdProxyHandler.isTransparentProxy); httpHeader.handleTransparentProxySupport(header, this.prop, virtualHost, httpdProxyHandler.isTransparentProxy);
@ -610,7 +632,7 @@ public final class httpd implements serverHandler {
// pass to server // pass to server
if (allowServer) { if (allowServer) {
if (handleServerAuthentication(header)) { if (handleServerAuthentication(header)) {
httpdFileHandler.doPost(prop, header, this.session.out, this.session.in); httpdFileHandler.doPost(prop, header, this.session.out, sessionIn);
} }
} else { } else {
// not authorized through firewall blocking (ip does not match filter) // not authorized through firewall blocking (ip does not match filter)
@ -621,7 +643,7 @@ public final class httpd implements serverHandler {
// pass to proxy // pass to proxy
if (((this.allowYaCyHop) && (handleYaCyHopAuthentication(header))) || if (((this.allowYaCyHop) && (handleYaCyHopAuthentication(header))) ||
((this.allowProxy) && (handleProxyAuthentication(header)))) { ((this.allowProxy) && (handleProxyAuthentication(header)))) {
httpdProxyHandler.doPost(prop, header, this.session.out, this.session.in); httpdProxyHandler.doPost(prop, header, this.session.out, sessionIn);
} else { } else {
// not authorized through firewall blocking (ip does not match filter) // not authorized through firewall blocking (ip does not match filter)
session.out.write((httpVersion + " 403 refused (IP not granted)" + serverCore.CRLF_STRING + serverCore.CRLF_STRING + "you are not allowed to connect to this proxy, because you are using the non-granted IP " + clientIP + ". allowed are only connections that match with the following filter: " + switchboard.getConfig("proxyClient", "*") + serverCore.CRLF_STRING).getBytes()); session.out.write((httpVersion + " 403 refused (IP not granted)" + serverCore.CRLF_STRING + serverCore.CRLF_STRING + "you are not allowed to connect to this proxy, because you are using the non-granted IP " + clientIP + ". allowed are only connections that match with the following filter: " + switchboard.getConfig("proxyClient", "*") + serverCore.CRLF_STRING).getBytes());
@ -1183,9 +1205,6 @@ public final class httpd implements serverHandler {
tp.put("requestURL", urlString); tp.put("requestURL", urlString);
switch (errorcase) { switch (errorcase) {
case ERRORCASE_MESSAGE:
tp.put("errorMessageType_detailedErrorMsg", (detailedErrorMsgText == null) ? "" : detailedErrorMsgText.replaceAll("\n", "<br />"));
break;
case ERRORCASE_FILE: case ERRORCASE_FILE:
tp.put("errorMessageType_file", (detailedErrorMsgFile == null) ? "" : detailedErrorMsgFile.toString()); tp.put("errorMessageType_file", (detailedErrorMsgFile == null) ? "" : detailedErrorMsgFile.toString());
if ((detailedErrorMsgValues != null) && (detailedErrorMsgValues.size() > 0)) { if ((detailedErrorMsgValues != null) && (detailedErrorMsgValues.size() > 0)) {
@ -1197,7 +1216,9 @@ public final class httpd implements serverHandler {
} }
} }
break; break;
case ERRORCASE_MESSAGE:
default: default:
tp.put("errorMessageType_detailedErrorMsg", (detailedErrorMsgText == null) ? "" : detailedErrorMsgText.replaceAll("\n", "<br />"));
break; break;
} }

11
source/de/anomic/http/httpdFileHandler.java
Unescape View File

@ -263,7 +263,7 @@ public final class httpdFileHandler {
doResponse(conProp, requestHeader, response, null); doResponse(conProp, requestHeader, response, null);
} }
public static void doPost(Properties conProp, httpHeader requestHeader, OutputStream response, PushbackInputStream body) { public static void doPost(Properties conProp, httpHeader requestHeader, OutputStream response, InputStream body) {
doResponse(conProp, requestHeader, response, body); doResponse(conProp, requestHeader, response, body);
} }
@ -352,11 +352,14 @@ public final class httpdFileHandler {
if (method.equals(httpHeader.METHOD_POST)) { if (method.equals(httpHeader.METHOD_POST)) {
GZIPInputStream gzipBody = null; GZIPInputStream gzipBody = null;
if (requestHeader.containsKey(httpHeader.CONTENT_LENGTH)) { /* "If the message does include a non-identity transfer-coding, the Content-Length MUST be ignored."
length = Integer.parseInt((String) requestHeader.get(httpHeader.CONTENT_LENGTH)); * [RFC 2616 HTTP/1.1, section 4.4] TODO: full RFC compliance
} else if (requestHeader.gzip()) { */
if (requestHeader.gzip()) {
length = -1; length = -1;
gzipBody = new GZIPInputStream(body); gzipBody = new GZIPInputStream(body);
} else if (requestHeader.containsKey(httpHeader.CONTENT_LENGTH)) {
length = Integer.parseInt((String) requestHeader.get(httpHeader.CONTENT_LENGTH));
} }
// } else { // } else {
// httpd.sendRespondError(conProp,out,4,403,null,"bad post values",null); // httpd.sendRespondError(conProp,out,4,403,null,"bad post values",null);

82
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -46,6 +46,8 @@ package de.anomic.yacy;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
@ -251,7 +253,21 @@ public final class yacyClient {
* @throws IOException * @throws IOException
*/ */
private static byte[] wput(final String url, String vhost, final List<Part> post) throws IOException { private static byte[] wput(final String url, String vhost, final List<Part> post) throws IOException {
return wput(url, vhost, post, 10000); return wput(url, vhost, post, 10000, false);
}
/**
* send data to the server named by vhost
*
* @param address address of the server
* @param vhost name of the server at address which should respond
* @param post data to send (name-value-pairs)
* @param gzipBody send with content gzip encoded
* @return response body
* @throws IOException
*/
private static byte[] wput(final String url, String vhost, final List<Part> post, boolean gzipBody) throws IOException {
return wput(url, vhost, post, 10000, gzipBody);
} }
/** /**
* send data to the server named by vhost * send data to the server named by vhost
@ -264,6 +280,20 @@ public final class yacyClient {
* @throws IOException * @throws IOException
*/ */
private static byte[] wput(final String url, String vhost, final List<Part> post, final int timeout) throws IOException { private static byte[] wput(final String url, String vhost, final List<Part> post, final int timeout) throws IOException {
return wput(url, vhost, post, timeout, false);
}
/**
* send data to the server named by vhost
*
* @param address address of the server
* @param vhost name of the server at address which should respond
* @param post data to send (name-value-pairs)
* @param timeout in milliseconds
* @param gzipBody send with content gzip encoded
* @return response body
* @throws IOException
*/
private static byte[] wput(final String url, String vhost, final List<Part> post, final int timeout, boolean gzipBody) throws IOException {
JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, null, null); JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, null, null);
client.setProxy(proxyConfig()); client.setProxy(proxyConfig());
@ -276,7 +306,7 @@ public final class yacyClient {
byte[] content = null; byte[] content = null;
try { try {
// send request/data // send request/data
res = client.POST(url, post); res = client.POST(url, post, gzipBody);
content = res.getData(); content = res.getData();
} finally { } finally {
if(res != null) { if(res != null) {
@ -937,9 +967,9 @@ public final class yacyClient {
final List<Part> post = yacyNetwork.basicRequestPost(plasmaSwitchboard.getSwitchboard(), targetSeed.hash, salt); final List<Part> post = yacyNetwork.basicRequestPost(plasmaSwitchboard.getSwitchboard(), targetSeed.hash, salt);
// enabling gzip compression for post request body // enabling gzip compression for post request body
/*if ((gzipBody) && (targetSeed.getVersion() >= yacyVersion.YACY_SUPPORTS_GZIP_POST_REQUESTS)) { if (gzipBody && (targetSeed.getVersion() < yacyVersion.YACY_SUPPORTS_GZIP_POST_REQUESTS_CHUNKED)) {
// TODO generate gzip-Header (and stream?) gzipBody = false;
}*/ }
post.add(new StringPart("wordc", Integer.toString(indexes.length))); post.add(new StringPart("wordc", Integer.toString(indexes.length)));
int indexcount = 0; int indexcount = 0;
@ -968,9 +998,9 @@ public final class yacyClient {
post.add(new StringPart("entryc", Integer.toString(indexcount))); post.add(new StringPart("entryc", Integer.toString(indexcount)));
post.add(new StringPart("indexes", entrypost.toString())); post.add(new StringPart("indexes", entrypost.toString()));
try { try {
final byte[] content = wput("http://" + address + "/yacy/transferRWI.html", targetSeed.getHexHash() + ".yacyh", post); final byte[] content = wput("http://" + address + "/yacy/transferRWI.html", targetSeed.getHexHash() + ".yacyh", post, gzipBody);
final ArrayList<String> v = nxTools.strings(content, "UTF-8"); final ArrayList<String> v = nxTools.strings(content, "UTF-8");
// this should return a list of urlhashes that are unknwon // this should return a list of urlhashes that are unknown
if ((v != null) && (v.size() > 0)) { if ((v != null) && (v.size() > 0)) {
yacyCore.seedDB.mySeed().incSI(indexcount); yacyCore.seedDB.mySeed().incSI(indexcount);
} }
@ -995,9 +1025,9 @@ public final class yacyClient {
final List<Part> post = yacyNetwork.basicRequestPost(plasmaSwitchboard.getSwitchboard(), targetSeed.hash, salt); final List<Part> post = yacyNetwork.basicRequestPost(plasmaSwitchboard.getSwitchboard(), targetSeed.hash, salt);
// enabling gzip compression for post request body // enabling gzip compression for post request body
/*if ((gzipBody) && (targetSeed.getVersion() >= yacyVersion.YACY_SUPPORTS_GZIP_POST_REQUESTS)) { if (gzipBody && (targetSeed.getVersion() < yacyVersion.YACY_SUPPORTS_GZIP_POST_REQUESTS_CHUNKED)) {
// TODO generate gzip-Header (and stream?) gzipBody = false;
}*/ }
String resource = ""; String resource = "";
int urlc = 0; int urlc = 0;
@ -1014,7 +1044,7 @@ public final class yacyClient {
} }
post.add(new StringPart("urlc", Integer.toString(urlc))); post.add(new StringPart("urlc", Integer.toString(urlc)));
try { try {
final byte[] content = wput("http://" + address + "/yacy/transferURL.html", targetSeed.getHexHash() + ".yacyh", post); final byte[] content = wput("http://" + address + "/yacy/transferURL.html", targetSeed.getHexHash() + ".yacyh", post, gzipBody);
final ArrayList<String> v = nxTools.strings(content, "UTF-8"); final ArrayList<String> v = nxTools.strings(content, "UTF-8");
if ((v != null) && (v.size() > 0)) { if ((v != null) && (v.size() > 0)) {
@ -1058,6 +1088,7 @@ public final class yacyClient {
} }
public static void main(String[] args) { public static void main(String[] args) {
if(args.length > 1) {
System.out.println("yacyClient Test"); System.out.println("yacyClient Test");
try { try {
final plasmaSwitchboard sb = new plasmaSwitchboard(new File(args[0]), "httpProxy.init", "DATA/SETTINGS/yacy.conf", false); final plasmaSwitchboard sb = new plasmaSwitchboard(new File(args[0]), "httpProxy.init", "DATA/SETTINGS/yacy.conf", false);
@ -1083,6 +1114,35 @@ public final class yacyClient {
e.printStackTrace(); e.printStackTrace();
} }
System.exit(0); System.exit(0);
} else if(args.length == 1) {
System.out.println("wput Test");
// connection params
URL url = null;
try {
url = new URL(args[0]);
} catch (MalformedURLException e) {
e.printStackTrace();
}
if(url == null) {
System.exit(1);
return;
}
final String vhost = url.getHost();
final int timeout = 10000;
final boolean gzipBody = true;
// data
final List<Part> post = new ArrayList<Part>();
post.add(new StringPart("process", "permission"));
post.add(new StringPart("purpose", "crcon"));
//post.add(new FilePart("filename", new ByteArrayPartSource(filename, file)));
// do it!
try {
final byte[] response = wput(url.toString(), vhost, post, timeout, gzipBody);
System.out.println(new String(response));
} catch (IOException e) {
e.printStackTrace();
}
}
} }
} }

2
source/de/anomic/yacy/yacyVersion.java
Unescape View File

@ -55,7 +55,7 @@ public final class yacyVersion implements Comparator<yacyVersion>, Comparable<ya
// general release info // general release info
public static final float YACY_SUPPORTS_PORT_FORWARDING = (float) 0.383; public static final float YACY_SUPPORTS_PORT_FORWARDING = (float) 0.383;
public static final float YACY_SUPPORTS_GZIP_POST_REQUESTS = (float) 0.40300772; public static final float YACY_SUPPORTS_GZIP_POST_REQUESTS_CHUNKED = (float) 0.58204761;
public static final float YACY_ACCEPTS_RANKING_TRANSMISSION = (float) 0.414; public static final float YACY_ACCEPTS_RANKING_TRANSMISSION = (float) 0.414;
public static final float YACY_HANDLES_COLLECTION_INDEX = (float) 0.486; public static final float YACY_HANDLES_COLLECTION_INDEX = (float) 0.486;
public static final float YACY_POVIDES_REMOTECRAWL_LISTS = (float) 0.550; public static final float YACY_POVIDES_REMOTECRAWL_LISTS = (float) 0.550;

Write Preview
Loading…
Cancel
Save