|
|
|
|
@ -50,6 +50,8 @@ import net.yacy.cora.protocol.ConnectionInfo;
|
|
|
|
|
import net.yacy.cora.protocol.HeaderFramework;
|
|
|
|
|
|
|
|
|
|
import org.apache.http.Header;
|
|
|
|
|
import org.apache.http.HeaderElement;
|
|
|
|
|
import org.apache.http.HeaderElementIterator;
|
|
|
|
|
import org.apache.http.HttpEntity;
|
|
|
|
|
import org.apache.http.HttpEntityEnclosingRequest;
|
|
|
|
|
import org.apache.http.HttpHost;
|
|
|
|
|
@ -67,6 +69,7 @@ import org.apache.http.client.params.CookiePolicy;
|
|
|
|
|
import org.apache.http.client.params.HttpClientParams;
|
|
|
|
|
import org.apache.http.client.protocol.ClientContext;
|
|
|
|
|
import org.apache.http.conn.ClientConnectionManager;
|
|
|
|
|
import org.apache.http.conn.ConnectionKeepAliveStrategy;
|
|
|
|
|
import org.apache.http.conn.params.ConnRouteParams;
|
|
|
|
|
import org.apache.http.conn.routing.HttpRoute;
|
|
|
|
|
import org.apache.http.conn.scheme.PlainSocketFactory;
|
|
|
|
|
@ -77,10 +80,10 @@ import org.apache.http.entity.InputStreamEntity;
|
|
|
|
|
import org.apache.http.entity.mime.MultipartEntity;
|
|
|
|
|
import org.apache.http.entity.mime.content.ContentBody;
|
|
|
|
|
import org.apache.http.entity.mime.content.StringBody;
|
|
|
|
|
import org.apache.http.impl.client.AbstractHttpClient;
|
|
|
|
|
import org.apache.http.impl.client.BasicCredentialsProvider;
|
|
|
|
|
import org.apache.http.impl.client.DefaultHttpClient;
|
|
|
|
|
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
|
|
|
|
|
import org.apache.http.impl.conn.PoolingClientConnectionManager;
|
|
|
|
|
import org.apache.http.message.BasicHeaderElementIterator;
|
|
|
|
|
import org.apache.http.params.BasicHttpParams;
|
|
|
|
|
import org.apache.http.params.HttpConnectionParams;
|
|
|
|
|
import org.apache.http.params.HttpParams;
|
|
|
|
|
@ -139,7 +142,7 @@ public class HTTPClient {
|
|
|
|
|
schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
|
|
|
|
|
schemeRegistry.register(new Scheme("https", 443, getSSLSocketFactory()));
|
|
|
|
|
|
|
|
|
|
final ThreadSafeClientConnManager clientConnectionManager = new ThreadSafeClientConnManager(schemeRegistry);
|
|
|
|
|
final PoolingClientConnectionManager clientConnectionManager = new PoolingClientConnectionManager(schemeRegistry);
|
|
|
|
|
|
|
|
|
|
// Create and initialize HTTP parameters
|
|
|
|
|
final HttpParams httpParams = new BasicHttpParams();
|
|
|
|
|
@ -154,7 +157,7 @@ public class HTTPClient {
|
|
|
|
|
clientConnectionManager.setDefaultMaxPerRoute(2);
|
|
|
|
|
// Increase max connections for localhost
|
|
|
|
|
final HttpHost localhost = new HttpHost("localhost");
|
|
|
|
|
clientConnectionManager.setMaxForRoute(new HttpRoute(localhost), maxcon);
|
|
|
|
|
clientConnectionManager.setMaxPerRoute(new HttpRoute(localhost), maxcon);
|
|
|
|
|
/**
|
|
|
|
|
* HTTP protocol settings
|
|
|
|
|
*/
|
|
|
|
|
@ -188,10 +191,12 @@ public class HTTPClient {
|
|
|
|
|
httpClient = new DefaultHttpClient(clientConnectionManager, httpParams);
|
|
|
|
|
// disable the cookiestore, cause this may cause segfaults and is not needed
|
|
|
|
|
((DefaultHttpClient) httpClient).setCookieStore(null);
|
|
|
|
|
// add cutom keep alive strategy
|
|
|
|
|
addCustomKeepAliveStrategy((DefaultHttpClient) httpClient);
|
|
|
|
|
// ask for gzip
|
|
|
|
|
((AbstractHttpClient) httpClient).addRequestInterceptor(new GzipRequestInterceptor());
|
|
|
|
|
((DefaultHttpClient) httpClient).addRequestInterceptor(new GzipRequestInterceptor());
|
|
|
|
|
// uncompress gzip
|
|
|
|
|
((AbstractHttpClient) httpClient).addResponseInterceptor(new GzipResponseInterceptor());
|
|
|
|
|
((DefaultHttpClient) httpClient).addResponseInterceptor(new GzipResponseInterceptor());
|
|
|
|
|
|
|
|
|
|
if (idledConnectionEvictor == null) {
|
|
|
|
|
idledConnectionEvictor = new IdledConnectionEvictor(clientConnectionManager);
|
|
|
|
|
@ -233,7 +238,7 @@ public class HTTPClient {
|
|
|
|
|
*/
|
|
|
|
|
public static void setMaxRouteHost(final String host) {
|
|
|
|
|
final HttpHost mHost = new HttpHost(host);
|
|
|
|
|
((ThreadSafeClientConnManager) httpClient.getConnectionManager()).setMaxForRoute(new HttpRoute(mHost), 50);
|
|
|
|
|
((PoolingClientConnectionManager) httpClient.getConnectionManager()).setMaxPerRoute(new HttpRoute(mHost), 50);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
@ -655,7 +660,7 @@ public class HTTPClient {
|
|
|
|
|
if (ProxySettings.use)
|
|
|
|
|
ConnRouteParams.setDefaultProxy(httpParams, ProxySettings.getProxyHost());
|
|
|
|
|
// TODO find a better way for this
|
|
|
|
|
ProxySettings.setProxyCreds((AbstractHttpClient) httpClient);
|
|
|
|
|
ProxySettings.setProxyCreds((DefaultHttpClient) httpClient);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private void storeConnectionInfo(final HttpUriRequest httpUriRequest) {
|
|
|
|
|
@ -704,6 +709,38 @@ public class HTTPClient {
|
|
|
|
|
return sslSF;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* If the Keep-Alive header is not present in the response,
|
|
|
|
|
* HttpClient assumes the connection can be kept alive indefinitely.
|
|
|
|
|
* Here we limit this to 5 seconds.
|
|
|
|
|
*
|
|
|
|
|
* @param defaultHttpClient
|
|
|
|
|
*/
|
|
|
|
|
private static void addCustomKeepAliveStrategy(final DefaultHttpClient defaultHttpClient) {
|
|
|
|
|
defaultHttpClient.setKeepAliveStrategy(new ConnectionKeepAliveStrategy() {
|
|
|
|
|
public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
|
|
|
|
|
// Honor 'keep-alive' header
|
|
|
|
|
String param, value;
|
|
|
|
|
HeaderElement element;
|
|
|
|
|
HeaderElementIterator it = new BasicHeaderElementIterator(
|
|
|
|
|
response.headerIterator(HTTP.CONN_KEEP_ALIVE));
|
|
|
|
|
while (it.hasNext()) {
|
|
|
|
|
element = it.nextElement();
|
|
|
|
|
param = element.getName();
|
|
|
|
|
value = element.getValue();
|
|
|
|
|
if (value != null && param.equalsIgnoreCase("timeout")) {
|
|
|
|
|
try {
|
|
|
|
|
return Long.parseLong(value) * 1000;
|
|
|
|
|
} catch(final NumberFormatException e) {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// Keep alive for 5 seconds only
|
|
|
|
|
return 5 * 1000;
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* testing
|
|
|
|
|
*
|
|
|
|
|
|
sixcooler
13 years ago