diff --git a/.classpath b/.classpath index 355399f26..9088d90ba 100644 --- a/.classpath +++ b/.classpath @@ -41,9 +41,9 @@ - - - + + + diff --git a/addon/YaCy.app/Contents/Info.plist b/addon/YaCy.app/Contents/Info.plist index ce4104f0d..268bfeb26 100644 --- a/addon/YaCy.app/Contents/Info.plist +++ b/addon/YaCy.app/Contents/Info.plist @@ -50,9 +50,9 @@ $JAVAROOT/lib/commons-logging-1.1.1.jar $JAVAROOT/lib/fontbox-1.6.0.jar $JAVAROOT/lib/geronimo-stax-api_1.0_spec-1.0.1.jar - $JAVAROOT/lib/httpclient-4.1.3.jar - $JAVAROOT/lib/httpcore-4.1.4.jar - $JAVAROOT/lib/httpmime-4.1.3.jar + $JAVAROOT/lib/httpclient-4.2.jar + $JAVAROOT/lib/httpcore-4.2.jar + $JAVAROOT/lib/httpmime-4.2.jar $JAVAROOT/lib/icu4j-core.jar $JAVAROOT/lib/J7Zip-modified.jar $JAVAROOT/lib/jakarta-oro-2.0.8.jar diff --git a/build.xml b/build.xml index 3f83f9039..beae51d7c 100644 --- a/build.xml +++ b/build.xml @@ -168,9 +168,9 @@ - - - + + + diff --git a/lib/httpclient-4.1.3.jar b/lib/httpclient-4.1.3.jar deleted file mode 100644 index dfa8793c2..000000000 Binary files a/lib/httpclient-4.1.3.jar and /dev/null differ diff --git a/lib/httpclient-4.1.3.License b/lib/httpclient-4.2.License similarity index 100% rename from lib/httpclient-4.1.3.License rename to lib/httpclient-4.2.License diff --git a/lib/httpclient-4.2.jar b/lib/httpclient-4.2.jar new file mode 100644 index 000000000..9e0df5098 Binary files /dev/null and b/lib/httpclient-4.2.jar differ diff --git a/lib/httpcore-4.1.4.jar b/lib/httpcore-4.1.4.jar deleted file mode 100644 index 88202d2bf..000000000 Binary files a/lib/httpcore-4.1.4.jar and /dev/null differ diff --git a/lib/httpmime-4.1.3.License b/lib/httpcore-4.2.License similarity index 100% rename from lib/httpmime-4.1.3.License rename to lib/httpcore-4.2.License diff --git a/lib/httpcore-4.2.jar b/lib/httpcore-4.2.jar new file mode 100644 index 000000000..20214eabd Binary files /dev/null and b/lib/httpcore-4.2.jar differ diff --git a/lib/httpcore-4.1.4.License b/lib/httpmime-4.2.License similarity index 97% rename from lib/httpcore-4.1.4.License rename to lib/httpmime-4.2.License index 2bb9ad240..2c41ec88f 100644 --- a/lib/httpcore-4.1.4.License +++ b/lib/httpmime-4.2.License @@ -173,4 +173,10 @@ incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. - END OF TERMS AND CONDITIONS \ No newline at end of file + END OF TERMS AND CONDITIONS + +This project contains annotations derived from JCIP-ANNOTATIONS +Copyright (c) 2005 Brian Goetz and Tim Peierls. +See http://www.jcip.net and the Creative Commons Attribution License +(http://creativecommons.org/licenses/by/2.5) + diff --git a/lib/httpmime-4.1.3.jar b/lib/httpmime-4.2.jar similarity index 60% rename from lib/httpmime-4.1.3.jar rename to lib/httpmime-4.2.jar index ff2014f33..8cec2fc23 100644 Binary files a/lib/httpmime-4.1.3.jar and b/lib/httpmime-4.2.jar differ diff --git a/nbproject/project.xml b/nbproject/project.xml index 6b21148a3..c968ec0d4 100644 --- a/nbproject/project.xml +++ b/nbproject/project.xml @@ -77,7 +77,7 @@ source htroot - lib/activation.jar:lib/apache-mime4j-0.6.jar:lib/apache-solr-solrj-3.6.0.jar:lib/bcmail-jdk15-145.jar:lib/bcprov-jdk15-145.jar:lib/bzip2.jar:lib/commons-codec-1.6.jar:lib/commons-fileupload-1.2.2.jar:lib/commons-httpclient-3.1.jar:lib/commons-io-2.1.jar:lib/commons-jxpath-1.3.jar:lib/commons-logging-1.1.1.jar:lib/fontbox-1.6.0.jar:lib/geronimo-stax-api_1.0_spec-1.0.1.jar:lib/httpclient-4.1.3.jar:lib/httpcore-4.1.4.jar:lib/httpmime-4.1.3.jar:lib/icu4j-core.jar:lib/J7Zip-modified.jar:lib/jakarta-oro-2.0.8.jar:lib/jcifs-1.3.15.jar:lib/jcl-over-slf4j-1.6.1.jar:lib/jempbox-1.6.0.jar:lib/jsch-0.1.42.jar:lib/json-simple-1.1.jar:lib/log4j-1.2.16.jar:lib/metadata-extractor-2.4.0-beta-1.jar:lib/mysql-connector-java-5.1.12-bin.jar:lib/pdfbox-1.6.0.jar:lib/poi-3.6-20091214.jar:lib/poi-scratchpad-3.6-20091214.jar:lib/servlet-api.jar:lib/slf4j-api-1.6.1.jar:lib/slf4j-jdk14-1.6.1.jar:lib/webcat-0.1-swf.jar:lib/wstx-asl-3.2.7.jar:lib/xercesImpl.jar:lib/xml-apis.jar:htroot/processing/domaingraph/applet/domaingraph.jar + lib/activation.jar:lib/apache-mime4j-0.6.jar:lib/apache-solr-solrj-3.6.0.jar:lib/bcmail-jdk15-145.jar:lib/bcprov-jdk15-145.jar:lib/bzip2.jar:lib/commons-codec-1.6.jar:lib/commons-fileupload-1.2.2.jar:lib/commons-httpclient-3.1.jar:lib/commons-io-2.1.jar:lib/commons-jxpath-1.3.jar:lib/commons-logging-1.1.1.jar:lib/fontbox-1.6.0.jar:lib/geronimo-stax-api_1.0_spec-1.0.1.jar:lib/httpclient-4.2.jar:lib/httpcore-4.2.jar:lib/httpmime-4.2.jar:lib/icu4j-core.jar:lib/J7Zip-modified.jar:lib/jakarta-oro-2.0.8.jar:lib/jcifs-1.3.15.jar:lib/jcl-over-slf4j-1.6.1.jar:lib/jempbox-1.6.0.jar:lib/jsch-0.1.42.jar:lib/json-simple-1.1.jar:lib/log4j-1.2.16.jar:lib/metadata-extractor-2.4.0-beta-1.jar:lib/mysql-connector-java-5.1.12-bin.jar:lib/pdfbox-1.6.0.jar:lib/poi-3.6-20091214.jar:lib/poi-scratchpad-3.6-20091214.jar:lib/servlet-api.jar:lib/slf4j-api-1.6.1.jar:lib/slf4j-jdk14-1.6.1.jar:lib/webcat-0.1-swf.jar:lib/wstx-asl-3.2.7.jar:lib/xercesImpl.jar:lib/xml-apis.jar:htroot/processing/domaingraph/applet/domaingraph.jar 1.6 diff --git a/source/net/yacy/cora/protocol/http/HTTPClient.java b/source/net/yacy/cora/protocol/http/HTTPClient.java index cbf9f91a5..d5bfd252a 100644 --- a/source/net/yacy/cora/protocol/http/HTTPClient.java +++ b/source/net/yacy/cora/protocol/http/HTTPClient.java @@ -50,6 +50,8 @@ import net.yacy.cora.protocol.ConnectionInfo; import net.yacy.cora.protocol.HeaderFramework; import org.apache.http.Header; +import org.apache.http.HeaderElement; +import org.apache.http.HeaderElementIterator; import org.apache.http.HttpEntity; import org.apache.http.HttpEntityEnclosingRequest; import org.apache.http.HttpHost; @@ -67,6 +69,7 @@ import org.apache.http.client.params.CookiePolicy; import org.apache.http.client.params.HttpClientParams; import org.apache.http.client.protocol.ClientContext; import org.apache.http.conn.ClientConnectionManager; +import org.apache.http.conn.ConnectionKeepAliveStrategy; import org.apache.http.conn.params.ConnRouteParams; import org.apache.http.conn.routing.HttpRoute; import org.apache.http.conn.scheme.PlainSocketFactory; @@ -77,10 +80,10 @@ import org.apache.http.entity.InputStreamEntity; import org.apache.http.entity.mime.MultipartEntity; import org.apache.http.entity.mime.content.ContentBody; import org.apache.http.entity.mime.content.StringBody; -import org.apache.http.impl.client.AbstractHttpClient; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; +import org.apache.http.impl.conn.PoolingClientConnectionManager; +import org.apache.http.message.BasicHeaderElementIterator; import org.apache.http.params.BasicHttpParams; import org.apache.http.params.HttpConnectionParams; import org.apache.http.params.HttpParams; @@ -139,7 +142,7 @@ public class HTTPClient { schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); schemeRegistry.register(new Scheme("https", 443, getSSLSocketFactory())); - final ThreadSafeClientConnManager clientConnectionManager = new ThreadSafeClientConnManager(schemeRegistry); + final PoolingClientConnectionManager clientConnectionManager = new PoolingClientConnectionManager(schemeRegistry); // Create and initialize HTTP parameters final HttpParams httpParams = new BasicHttpParams(); @@ -154,7 +157,7 @@ public class HTTPClient { clientConnectionManager.setDefaultMaxPerRoute(2); // Increase max connections for localhost final HttpHost localhost = new HttpHost("localhost"); - clientConnectionManager.setMaxForRoute(new HttpRoute(localhost), maxcon); + clientConnectionManager.setMaxPerRoute(new HttpRoute(localhost), maxcon); /** * HTTP protocol settings */ @@ -188,10 +191,12 @@ public class HTTPClient { httpClient = new DefaultHttpClient(clientConnectionManager, httpParams); // disable the cookiestore, cause this may cause segfaults and is not needed ((DefaultHttpClient) httpClient).setCookieStore(null); + // add cutom keep alive strategy + addCustomKeepAliveStrategy((DefaultHttpClient) httpClient); // ask for gzip - ((AbstractHttpClient) httpClient).addRequestInterceptor(new GzipRequestInterceptor()); + ((DefaultHttpClient) httpClient).addRequestInterceptor(new GzipRequestInterceptor()); // uncompress gzip - ((AbstractHttpClient) httpClient).addResponseInterceptor(new GzipResponseInterceptor()); + ((DefaultHttpClient) httpClient).addResponseInterceptor(new GzipResponseInterceptor()); if (idledConnectionEvictor == null) { idledConnectionEvictor = new IdledConnectionEvictor(clientConnectionManager); @@ -233,7 +238,7 @@ public class HTTPClient { */ public static void setMaxRouteHost(final String host) { final HttpHost mHost = new HttpHost(host); - ((ThreadSafeClientConnManager) httpClient.getConnectionManager()).setMaxForRoute(new HttpRoute(mHost), 50); + ((PoolingClientConnectionManager) httpClient.getConnectionManager()).setMaxPerRoute(new HttpRoute(mHost), 50); } /** @@ -655,7 +660,7 @@ public class HTTPClient { if (ProxySettings.use) ConnRouteParams.setDefaultProxy(httpParams, ProxySettings.getProxyHost()); // TODO find a better way for this - ProxySettings.setProxyCreds((AbstractHttpClient) httpClient); + ProxySettings.setProxyCreds((DefaultHttpClient) httpClient); } private void storeConnectionInfo(final HttpUriRequest httpUriRequest) { @@ -703,6 +708,38 @@ public class HTTPClient { final SSLSocketFactory sslSF = new SSLSocketFactory(sslContext, SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); return sslSF; } + + /** + * If the Keep-Alive header is not present in the response, + * HttpClient assumes the connection can be kept alive indefinitely. + * Here we limit this to 5 seconds. + * + * @param defaultHttpClient + */ + private static void addCustomKeepAliveStrategy(final DefaultHttpClient defaultHttpClient) { + defaultHttpClient.setKeepAliveStrategy(new ConnectionKeepAliveStrategy() { + public long getKeepAliveDuration(HttpResponse response, HttpContext context) { + // Honor 'keep-alive' header + String param, value; + HeaderElement element; + HeaderElementIterator it = new BasicHeaderElementIterator( + response.headerIterator(HTTP.CONN_KEEP_ALIVE)); + while (it.hasNext()) { + element = it.nextElement(); + param = element.getName(); + value = element.getValue(); + if (value != null && param.equalsIgnoreCase("timeout")) { + try { + return Long.parseLong(value) * 1000; + } catch(final NumberFormatException e) { + } + } + } + // Keep alive for 5 seconds only + return 5 * 1000; + } + }); + } /** * testing