boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of ssh://git@gitorious.org/yacy/rc1.git

Browse Source
pull/1/head
Michael Peter Christen 12 years ago
parent 91a875dff5 3c48fc65fd
commit a496313248
1 changed files with 93 additions and 83 deletions
Show Stats Download Patch File Download Diff File

174
source/net/yacy/cora/federate/solr/instance/RemoteInstance.java
Unescape View File

@ -37,27 +37,34 @@ import net.yacy.search.schema.WebgraphSchema;
import org.apache.http.Header; import org.apache.http.Header;
import org.apache.http.HeaderElement; import org.apache.http.HeaderElement;
import org.apache.http.HttpEntity; import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest; import org.apache.http.HttpRequest;
import org.apache.http.HttpRequestInterceptor; import org.apache.http.HttpRequestInterceptor;
import org.apache.http.HttpResponse; import org.apache.http.HttpResponse;
import org.apache.http.HttpResponseInterceptor; import org.apache.http.HttpResponseInterceptor;
import org.apache.http.auth.AuthScope; import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.config.RequestConfig; import org.apache.http.client.AuthCache;
import org.apache.http.client.entity.GzipDecompressingEntity; import org.apache.http.client.entity.GzipDecompressingEntity;
import org.apache.http.client.protocol.ClientContext;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.impl.conn.PoolingClientConnectionManager;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.apache.http.protocol.HttpContext; import org.apache.http.protocol.HttpContext;
import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.apache.solr.client.solrj.impl.HttpSolrServer;
@SuppressWarnings("deprecation") //TODO: switch to 4.3-Stuff
public class RemoteInstance implements SolrInstance { public class RemoteInstance implements SolrInstance {
private String solrurl; private String solrurl;
// old Stuff: private final DefaultHttpClient client; private final DefaultHttpClient client;
private final CloseableHttpClient client; // 4.3 private final CloseableHttpClient client;
private final String defaultCoreName; private final String defaultCoreName;
private final HttpSolrServer defaultServer; private final HttpSolrServer defaultServer;
private final Collection<String> coreNames; private final Collection<String> coreNames;
@ -127,73 +134,25 @@ public class RemoteInstance implements SolrInstance {
} }
} }
if (solraccount.length() > 0) { if (solraccount.length() > 0) {
final PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); // 4.3:
cm.setMaxTotal(100); // final PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
final RequestConfig.Builder reqBuilder = RequestConfig.custom();
reqBuilder.setSocketTimeout(timeout);
reqBuilder.setConnectTimeout(timeout);
reqBuilder.setConnectionRequestTimeout(timeout);
final BasicCredentialsProvider credsProvider = new BasicCredentialsProvider();
credsProvider.setCredentials(new AuthScope(host, AuthScope.ANY_PORT), new UsernamePasswordCredentials(solraccount, solrpw));
final HttpClientBuilder builder = HttpClientBuilder.create();
builder.setConnectionManager(cm);
builder.setDefaultRequestConfig(reqBuilder.build());
builder.setDefaultCredentialsProvider(credsProvider);
builder.disableAutomaticRetries(); // no retries needed; we expect connections to fail; therefore we should not retry
// ask for gzip - why not using net.yacy.cora.protocol.http.GzipRequestInterceptor?
builder.addInterceptorLast(new HttpRequestInterceptor() {
@Override
public void process(final HttpRequest request, final HttpContext context) throws IOException {
if (!request.containsHeader("Accept-Encoding")) request.addHeader("Accept-Encoding", "gzip");
if (!request.containsHeader("Connection")) request.addHeader("Connection", "close"); // prevent CLOSE_WAIT
}
});
// uncompress gzip - why not using net.yacy.cora.protocol.http.GzipResponseInterceptor?
builder.addInterceptorLast(new HttpResponseInterceptor() {
@Override
public void process(final HttpResponse response, final HttpContext context) throws IOException {
HttpEntity entity = response.getEntity();
if (entity != null) {
Header ceheader = entity.getContentEncoding();
if (ceheader != null) {
HeaderElement[] codecs = ceheader.getElements();
for (HeaderElement codec : codecs) {
if (codec.getName().equalsIgnoreCase("gzip")) {
response.setEntity(new GzipDecompressingEntity(response.getEntity()));
return;
}
}
}
}
}
});
this.client = builder.build();
// old Stuff:
// PoolingClientConnectionManager cm = new PoolingClientConnectionManager(); // try also: ThreadSafeClientConnManager
// cm.setMaxTotal(100); // cm.setMaxTotal(100);
// //
// this.client = new DefaultHttpClient(cm) { // final RequestConfig.Builder reqBuilder = RequestConfig.custom();
// @Override // reqBuilder.setSocketTimeout(timeout);
// protected HttpContext createHttpContext() { // reqBuilder.setConnectTimeout(timeout);
// Auth-caching HttpContext context = super.createHttpContext(); // reqBuilder.setConnectionRequestTimeout(timeout);
// is done AuthCache authCache = new BasicAuthCache(); //
// by default BasicScheme basicAuth = new BasicScheme(); // final BasicCredentialsProvider credsProvider = new BasicCredentialsProvider();
// since 4.1 HttpHost targetHost = new HttpHost(u.getHost(), u.getPort(), u.getProtocol()); // credsProvider.setCredentials(new AuthScope(host, AuthScope.ANY_PORT), new UsernamePasswordCredentials(solraccount, solrpw));
// authCache.put(targetHost, basicAuth); //
// context.setAttribute(ClientContext.AUTH_CACHE, authCache); // final HttpClientBuilder builder = HttpClientBuilder.create();
// this.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(0, false)); // no retries needed; we expect connections to fail; therefore we should not retry // builder.setConnectionManager(cm);
// return context; // builder.setDefaultRequestConfig(reqBuilder.build());
// } // builder.setDefaultCredentialsProvider(credsProvider);
// }; // builder.disableAutomaticRetries(); // no retries needed; we expect connections to fail; therefore we should not retry
// HttpParams params = this.client.getParams(); // // ask for gzip - why not using net.yacy.cora.protocol.http.GzipRequestInterceptor?
// HttpConnectionParams.setConnectionTimeout(params, timeout); // builder.addInterceptorLast(new HttpRequestInterceptor() {
// HttpConnectionParams.setSoTimeout(params, timeout);
// this.client.addRequestInterceptor(new HttpRequestInterceptor() {
// @Override // @Override
// public void process(final HttpRequest request, final HttpContext context) throws IOException { // public void process(final HttpRequest request, final HttpContext context) throws IOException {
// if (!request.containsHeader("Accept-Encoding")) request.addHeader("Accept-Encoding", "gzip"); // if (!request.containsHeader("Accept-Encoding")) request.addHeader("Accept-Encoding", "gzip");
@ -201,7 +160,8 @@ public class RemoteInstance implements SolrInstance {
// } // }
// //
// }); // });
// this.client.addResponseInterceptor(new HttpResponseInterceptor() { // // uncompress gzip - why not using net.yacy.cora.protocol.http.GzipResponseInterceptor?
// builder.addInterceptorLast(new HttpResponseInterceptor() {
// @Override // @Override
// public void process(final HttpResponse response, final HttpContext context) throws IOException { // public void process(final HttpResponse response, final HttpContext context) throws IOException {
// HttpEntity entity = response.getEntity(); // HttpEntity entity = response.getEntity();
@ -219,9 +179,58 @@ public class RemoteInstance implements SolrInstance {
// } // }
// } // }
// }); // });
// BasicCredentialsProvider credsProvider = new BasicCredentialsProvider(); // this.client = builder.build();
// credsProvider.setCredentials(new AuthScope(host, AuthScope.ANY_PORT), new UsernamePasswordCredentials(solraccount, solrpw));
// this.client.setCredentialsProvider(credsProvider); // old Stuff START
PoolingClientConnectionManager cm = new PoolingClientConnectionManager(); // try also: ThreadSafeClientConnManager
cm.setMaxTotal(100);
this.client = new DefaultHttpClient(cm) {
@Override
protected HttpContext createHttpContext() {
HttpContext context = super.createHttpContext();
AuthCache authCache = new BasicAuthCache();
BasicScheme basicAuth = new BasicScheme();
HttpHost targetHost = new HttpHost(u.getHost(), u.getPort(), u.getProtocol());
authCache.put(targetHost, basicAuth);
context.setAttribute(ClientContext.AUTH_CACHE, authCache);
this.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(0, false)); // no retries needed; we expect connections to fail; therefore we should not retry
return context;
}
};
HttpParams params = this.client.getParams();
HttpConnectionParams.setConnectionTimeout(params, timeout);
HttpConnectionParams.setSoTimeout(params, timeout);
this.client.addRequestInterceptor(new HttpRequestInterceptor() {
@Override
public void process(final HttpRequest request, final HttpContext context) throws IOException {
if (!request.containsHeader("Accept-Encoding")) request.addHeader("Accept-Encoding", "gzip");
if (!request.containsHeader("Connection")) request.addHeader("Connection", "close"); // prevent CLOSE_WAIT
}
});
this.client.addResponseInterceptor(new HttpResponseInterceptor() {
@Override
public void process(final HttpResponse response, final HttpContext context) throws IOException {
HttpEntity entity = response.getEntity();
if (entity != null) {
Header ceheader = entity.getContentEncoding();
if (ceheader != null) {
HeaderElement[] codecs = ceheader.getElements();
for (HeaderElement codec : codecs) {
if (codec.getName().equalsIgnoreCase("gzip")) {
response.setEntity(new GzipDecompressingEntity(response.getEntity()));
return;
}
}
}
}
}
});
BasicCredentialsProvider credsProvider = new BasicCredentialsProvider();
credsProvider.setCredentials(new AuthScope(host, AuthScope.ANY_PORT), new UsernamePasswordCredentials(solraccount, solrpw));
this.client.setCredentialsProvider(credsProvider);
// old Stuff END
} else { } else {
this.client = null; this.client = null;
} }
@ -289,13 +298,14 @@ public class RemoteInstance implements SolrInstance {
@Override @Override
public void close() { public void close() {
// old Stuff: if (this.client != null) this.client.getConnectionManager().shutdown(); if (this.client != null) this.client.getConnectionManager().shutdown();
if (this.client != null) // 4.3
try { // if (this.client != null)
this.client.close(); // try {
} catch (final IOException e) { // this.client.close();
// TODO Auto-generated catch block // } catch (final IOException e) {
} // // TODO Auto-generated catch block
// }
} }
} }

Write Preview
Loading…
Cancel
Save