bump to httpclient- httpcore-, httpmime- 4.2

.classpath

@@ -41,9 +41,9 @@
 	<classpathentry kind="lib" path="lib/jcl-over-slf4j-1.6.1.jar"/>
 	<classpathentry kind="lib" path="lib/wstx-asl-3.2.7.jar"/>
 	<classpathentry kind="lib" path="lib/commons-codec-1.6.jar"/>
-	<classpathentry kind="lib" path="lib/httpcore-4.1.4.jar"/>
+	<classpathentry kind="lib" path="lib/httpcore-4.2.jar"/>
-	<classpathentry kind="lib" path="lib/httpclient-4.1.3.jar"/>
+	<classpathentry kind="lib" path="lib/httpclient-4.2.jar"/>
-	<classpathentry kind="lib" path="lib/httpmime-4.1.3.jar"/>
+	<classpathentry kind="lib" path="lib/httpmime-4.2.jar"/>
 	<classpathentry kind="lib" path="lib/commons-io-2.1.jar"/>
 	<classpathentry kind="lib" path="lib/apache-solr-solrj-3.6.0.jar" sourcepath="/solrj/src"/>
 	<classpathentry kind="lib" path="lib/commons-compress-1.4.1.jar"/>

addon/YaCy.app/Contents/Info.plist

@@ -50,9 +50,9 @@
 			<string>$JAVAROOT/lib/commons-logging-1.1.1.jar</string>
 			<string>$JAVAROOT/lib/fontbox-1.6.0.jar</string>
 			<string>$JAVAROOT/lib/geronimo-stax-api_1.0_spec-1.0.1.jar</string>
-			<string>$JAVAROOT/lib/httpclient-4.1.3.jar</string>
+			<string>$JAVAROOT/lib/httpclient-4.2.jar</string>
-			<string>$JAVAROOT/lib/httpcore-4.1.4.jar</string>
+			<string>$JAVAROOT/lib/httpcore-4.2.jar</string>
-			<string>$JAVAROOT/lib/httpmime-4.1.3.jar</string>
+			<string>$JAVAROOT/lib/httpmime-4.2.jar</string>
 			<string>$JAVAROOT/lib/icu4j-core.jar</string>
 			<string>$JAVAROOT/lib/J7Zip-modified.jar</string>
 			<string>$JAVAROOT/lib/jakarta-oro-2.0.8.jar</string>

build.xml

@@ -168,9 +168,9 @@
     	<pathelement location="${lib}/commons-logging-1.1.1.jar" />
     	<pathelement location="${lib}/fontbox-1.6.0.jar" />
         <pathelement location="${lib}/geronimo-stax-api_1.0_spec-1.0.1.jar" />
-        <pathelement location="${lib}/httpclient-4.1.3.jar" />
+        <pathelement location="${lib}/httpclient-4.2.jar" />
-        <pathelement location="${lib}/httpcore-4.1.4.jar" />
+        <pathelement location="${lib}/httpcore-4.2.jar" />
-        <pathelement location="${lib}/httpmime-4.1.3.jar" />
+        <pathelement location="${lib}/httpmime-4.2.jar" />
         <pathelement location="${lib}/icu4j-core.jar" />
         <pathelement location="${lib}/J7Zip-modified.jar" />
         <pathelement location="${lib}/jakarta-oro-2.0.8.jar" />

lib/httpmime-4.2.License

@@ -173,4 +173,10 @@
       incurred by, or claims asserted against, such Contributor by reason
       of your accepting any such warranty or additional liability.
 
-   END OF TERMS AND CONDITIONS
+   END OF TERMS AND CONDITIONS
+   
+This project contains annotations derived from JCIP-ANNOTATIONS
+Copyright (c) 2005 Brian Goetz and Tim Peierls.
+See http://www.jcip.net and the Creative Commons Attribution License 
+(http://creativecommons.org/licenses/by/2.5)
+

nbproject/project.xml

@@ -77,7 +77,7 @@
             <compilation-unit>
                 <package-root>source</package-root>
                 <package-root>htroot</package-root>
-                <classpath mode="compile">lib/activation.jar:lib/apache-mime4j-0.6.jar:lib/apache-solr-solrj-3.6.0.jar:lib/bcmail-jdk15-145.jar:lib/bcprov-jdk15-145.jar:lib/bzip2.jar:lib/commons-codec-1.6.jar:lib/commons-fileupload-1.2.2.jar:lib/commons-httpclient-3.1.jar:lib/commons-io-2.1.jar:lib/commons-jxpath-1.3.jar:lib/commons-logging-1.1.1.jar:lib/fontbox-1.6.0.jar:lib/geronimo-stax-api_1.0_spec-1.0.1.jar:lib/httpclient-4.1.3.jar:lib/httpcore-4.1.4.jar:lib/httpmime-4.1.3.jar:lib/icu4j-core.jar:lib/J7Zip-modified.jar:lib/jakarta-oro-2.0.8.jar:lib/jcifs-1.3.15.jar:lib/jcl-over-slf4j-1.6.1.jar:lib/jempbox-1.6.0.jar:lib/jsch-0.1.42.jar:lib/json-simple-1.1.jar:lib/log4j-1.2.16.jar:lib/metadata-extractor-2.4.0-beta-1.jar:lib/mysql-connector-java-5.1.12-bin.jar:lib/pdfbox-1.6.0.jar:lib/poi-3.6-20091214.jar:lib/poi-scratchpad-3.6-20091214.jar:lib/servlet-api.jar:lib/slf4j-api-1.6.1.jar:lib/slf4j-jdk14-1.6.1.jar:lib/webcat-0.1-swf.jar:lib/wstx-asl-3.2.7.jar:lib/xercesImpl.jar:lib/xml-apis.jar:htroot/processing/domaingraph/applet/domaingraph.jar</classpath>
+                <classpath mode="compile">lib/activation.jar:lib/apache-mime4j-0.6.jar:lib/apache-solr-solrj-3.6.0.jar:lib/bcmail-jdk15-145.jar:lib/bcprov-jdk15-145.jar:lib/bzip2.jar:lib/commons-codec-1.6.jar:lib/commons-fileupload-1.2.2.jar:lib/commons-httpclient-3.1.jar:lib/commons-io-2.1.jar:lib/commons-jxpath-1.3.jar:lib/commons-logging-1.1.1.jar:lib/fontbox-1.6.0.jar:lib/geronimo-stax-api_1.0_spec-1.0.1.jar:lib/httpclient-4.2.jar:lib/httpcore-4.2.jar:lib/httpmime-4.2.jar:lib/icu4j-core.jar:lib/J7Zip-modified.jar:lib/jakarta-oro-2.0.8.jar:lib/jcifs-1.3.15.jar:lib/jcl-over-slf4j-1.6.1.jar:lib/jempbox-1.6.0.jar:lib/jsch-0.1.42.jar:lib/json-simple-1.1.jar:lib/log4j-1.2.16.jar:lib/metadata-extractor-2.4.0-beta-1.jar:lib/mysql-connector-java-5.1.12-bin.jar:lib/pdfbox-1.6.0.jar:lib/poi-3.6-20091214.jar:lib/poi-scratchpad-3.6-20091214.jar:lib/servlet-api.jar:lib/slf4j-api-1.6.1.jar:lib/slf4j-jdk14-1.6.1.jar:lib/webcat-0.1-swf.jar:lib/wstx-asl-3.2.7.jar:lib/xercesImpl.jar:lib/xml-apis.jar:htroot/processing/domaingraph/applet/domaingraph.jar</classpath>
                 <source-level>1.6</source-level>
             </compilation-unit>
         </java-data>

source/net/yacy/cora/protocol/http/HTTPClient.java

@@ -50,6 +50,8 @@ import net.yacy.cora.protocol.ConnectionInfo;
 import net.yacy.cora.protocol.HeaderFramework;
 
 import org.apache.http.Header;
+import org.apache.http.HeaderElement;
+import org.apache.http.HeaderElementIterator;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpEntityEnclosingRequest;
 import org.apache.http.HttpHost;
@@ -67,6 +69,7 @@ import org.apache.http.client.params.CookiePolicy;
 import org.apache.http.client.params.HttpClientParams;
 import org.apache.http.client.protocol.ClientContext;
 import org.apache.http.conn.ClientConnectionManager;
+import org.apache.http.conn.ConnectionKeepAliveStrategy;
 import org.apache.http.conn.params.ConnRouteParams;
 import org.apache.http.conn.routing.HttpRoute;
 import org.apache.http.conn.scheme.PlainSocketFactory;
@@ -77,10 +80,10 @@ import org.apache.http.entity.InputStreamEntity;
 import org.apache.http.entity.mime.MultipartEntity;
 import org.apache.http.entity.mime.content.ContentBody;
 import org.apache.http.entity.mime.content.StringBody;
-import org.apache.http.impl.client.AbstractHttpClient;
 import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.impl.client.DefaultHttpClient;
-import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
+import org.apache.http.impl.conn.PoolingClientConnectionManager;
+import org.apache.http.message.BasicHeaderElementIterator;
 import org.apache.http.params.BasicHttpParams;
 import org.apache.http.params.HttpConnectionParams;
 import org.apache.http.params.HttpParams;
@@ -139,7 +142,7 @@ public class HTTPClient {
 		schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
 		schemeRegistry.register(new Scheme("https", 443, getSSLSocketFactory()));
 
-		final ThreadSafeClientConnManager clientConnectionManager = new ThreadSafeClientConnManager(schemeRegistry);
+		final PoolingClientConnectionManager clientConnectionManager = new PoolingClientConnectionManager(schemeRegistry);
 
 		// Create and initialize HTTP parameters
 		final HttpParams httpParams = new BasicHttpParams();
@@ -154,7 +157,7 @@ public class HTTPClient {
 		clientConnectionManager.setDefaultMaxPerRoute(2);
 		// Increase max connections for localhost
 		final HttpHost localhost = new HttpHost("localhost");
-		clientConnectionManager.setMaxForRoute(new HttpRoute(localhost), maxcon);
+		clientConnectionManager.setMaxPerRoute(new HttpRoute(localhost), maxcon);
 		/**
 		 * HTTP protocol settings
 		 */
@@ -188,10 +191,12 @@ public class HTTPClient {
 		httpClient = new DefaultHttpClient(clientConnectionManager, httpParams);
 		// disable the cookiestore, cause this may cause segfaults and is not needed
 		((DefaultHttpClient) httpClient).setCookieStore(null);
+		// add cutom keep alive strategy
+		addCustomKeepAliveStrategy((DefaultHttpClient) httpClient);
 		// ask for gzip
-		((AbstractHttpClient) httpClient).addRequestInterceptor(new GzipRequestInterceptor());
+		((DefaultHttpClient) httpClient).addRequestInterceptor(new GzipRequestInterceptor());
 		// uncompress gzip
-		((AbstractHttpClient) httpClient).addResponseInterceptor(new GzipResponseInterceptor());
+		((DefaultHttpClient) httpClient).addResponseInterceptor(new GzipResponseInterceptor());
 
 		if (idledConnectionEvictor == null) {
 		    idledConnectionEvictor = new IdledConnectionEvictor(clientConnectionManager);
@@ -233,7 +238,7 @@ public class HTTPClient {
      */
     public static void setMaxRouteHost(final String host) {
     	final HttpHost mHost = new HttpHost(host);
-    	((ThreadSafeClientConnManager) httpClient.getConnectionManager()).setMaxForRoute(new HttpRoute(mHost), 50);
+    	((PoolingClientConnectionManager) httpClient.getConnectionManager()).setMaxPerRoute(new HttpRoute(mHost), 50);
     }
 
     /**
@@ -655,7 +660,7 @@ public class HTTPClient {
     	if (ProxySettings.use)
     		ConnRouteParams.setDefaultProxy(httpParams, ProxySettings.getProxyHost());
     	// TODO find a better way for this
-    	ProxySettings.setProxyCreds((AbstractHttpClient) httpClient);
+    	ProxySettings.setProxyCreds((DefaultHttpClient) httpClient);
     }
 
     private void storeConnectionInfo(final HttpUriRequest httpUriRequest) {
@@ -703,6 +708,38 @@ public class HTTPClient {
         final SSLSocketFactory sslSF = new SSLSocketFactory(sslContext, SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
     	return sslSF;
     }
+    
+    /**
+     * If the Keep-Alive header is not present in the response,
+     * HttpClient assumes the connection can be kept alive indefinitely.
+     * Here we limit this to 5 seconds. 
+     * 
+     * @param defaultHttpClient
+     */
+    private static void addCustomKeepAliveStrategy(final DefaultHttpClient defaultHttpClient) {
+    	defaultHttpClient.setKeepAliveStrategy(new ConnectionKeepAliveStrategy() {
+			public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
+		        // Honor 'keep-alive' header
+				String param, value;
+				HeaderElement element;
+		        HeaderElementIterator it = new BasicHeaderElementIterator(
+		                response.headerIterator(HTTP.CONN_KEEP_ALIVE));
+		        while (it.hasNext()) {
+		            element = it.nextElement();
+		            param = element.getName(); 
+		            value = element.getValue();
+		            if (value != null && param.equalsIgnoreCase("timeout")) {
+		                try {
+		                    return Long.parseLong(value) * 1000;
+		                } catch(final NumberFormatException e) {
+		                }
+		            }
+		        }
+		        // Keep alive for 5 seconds only
+		        return 5 * 1000;
+			}
+    	});
+    }
 
     /**
      * testing