boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

update to httpclient-4.1 - sorry forgot some

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7474 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
sixcooler 14 years ago
parent 45dcfa3460
commit 3e8b72be50
2 changed files with 33 additions and 38 deletions
Show Stats Download Patch File Download Diff File

24
source/net/yacy/cora/protocol/http/GzipDecompressingEntity.java
Unescape View File

@ -36,17 +36,17 @@ import org.apache.http.entity.HttpEntityWrapper;
public class GzipDecompressingEntity extends HttpEntityWrapper {
private static final int DEFAULT_BUFFER_SIZE = 1024; // this is also the maximum chunk size
private GZIPInputStream gzipInputStream = null;
public GzipDecompressingEntity(final HttpEntity entity) {
super(entity);
}
public InputStream getContent() throws IOException, IllegalStateException {
// the wrapped entity's getContent() decides about repeatability
InputStream wrappedin = wrappedEntity.getContent();
return new GZIPInputStream(wrappedin);
if (gzipInputStream == null) {
gzipInputStream = new GZIPInputStream(wrappedEntity.getContent());
}
return gzipInputStream;
}
public void writeTo(OutputStream outstream) throws IOException {
@ -54,11 +54,15 @@ public class GzipDecompressingEntity extends HttpEntityWrapper {
throw new IllegalArgumentException("Output stream may not be null");
}
InputStream instream = this.getContent();
int l;
byte[] tmp = new byte[DEFAULT_BUFFER_SIZE];
while ((l = instream.read(tmp)) != -1) {
outstream.write(tmp, 0, l);
}
try {
int l;
byte[] tmp = new byte[DEFAULT_BUFFER_SIZE];
while ((l = instream.read(tmp)) != -1) {
outstream.write(tmp, 0, l);
}
} finally {
instream.close();
}
}
public boolean isChunked() {

47
source/net/yacy/cora/protocol/http/HTTPClient.java
Unescape View File

@ -59,8 +59,6 @@ import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.params.HttpClientParams;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.params.ConnManagerParams;
import org.apache.http.conn.params.ConnPerRouteBean;
import org.apache.http.conn.params.ConnRouteParams;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.conn.scheme.PlainSocketFactory;
@ -115,23 +113,27 @@ public class HTTPClient {
}
public static HttpClient initConnectionManager() {
// Create and initialize scheme registry
final SchemeRegistry schemeRegistry = new SchemeRegistry();
schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
schemeRegistry.register(new Scheme("https", 443, getSSLSocketFactory()));
ThreadSafeClientConnManager clientConnectionManager = new ThreadSafeClientConnManager(schemeRegistry);
// Create and initialize HTTP parameters
final HttpParams httpParams = new BasicHttpParams();
/**
* ConnectionManager settings
*/
// TODO: how much connections do we need? - default: 20
ConnManagerParams.setMaxTotalConnections(httpParams, maxcon);
// how much connections do we need? - default: 20
clientConnectionManager.setMaxTotal(maxcon);
// for statistics same value should also be set here
ConnectionInfo.setMaxcount(maxcon);
// connections per host (2 default)
final ConnPerRouteBean connPerRoute = new ConnPerRouteBean(2);
clientConnectionManager.setDefaultMaxPerRoute(2);
// Increase max connections for localhost
HttpHost localhost = new HttpHost("locahost");
connPerRoute.setMaxForRoute(new HttpRoute(localhost), maxcon);
ConnManagerParams.setMaxConnectionsPerRoute(httpParams, connPerRoute);
// how long to wait for getting a connection from manager in milliseconds
ConnManagerParams.setTimeout(httpParams, 9000L);
clientConnectionManager.setMaxForRoute(new HttpRoute(localhost), maxcon);
/**
* HTTP protocol settings
*/
@ -156,13 +158,6 @@ public class HTTPClient {
HttpConnectionParams.setTcpNoDelay(httpParams, false);
// TODO: testing noreuse - there will be HttpConnectionParams.setSoReuseaddr(HttpParams params, boolean reuseaddr) in core-4.1
// Create and initialize scheme registry
final SchemeRegistry schemeRegistry = new SchemeRegistry();
schemeRegistry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80));
schemeRegistry.register(new Scheme("https", getSSLSocketFactory(), 443));
ClientConnectionManager clientConnectionManager = new ThreadSafeClientConnManager(httpParams, schemeRegistry);
httpClient = new DefaultHttpClient(clientConnectionManager, httpParams);
// ask for gzip
((AbstractHttpClient) httpClient).addRequestInterceptor(new GzipRequestInterceptor());
@ -408,9 +403,8 @@ public class HTTPClient {
if (httpEntity != null) try {
httpEntity.writeTo(outputStream);
outputStream.flush();
// TODO: The name of this method is misnomer.
// It will be renamed to #finish() in the next major release of httpcore
httpEntity.consumeContent();
// Ensures that the entity content is fully consumed and the content stream, if exists, is closed.
EntityUtils.consume(httpEntity);
ConnectionInfo.removeConnection(currentRequest.hashCode());
currentRequest = null;
} catch (final IOException e) {
@ -432,9 +426,8 @@ public class HTTPClient {
if (httpResponse != null) {
final HttpEntity httpEntity = httpResponse.getEntity();
if (httpEntity != null && httpEntity.isStreaming()) {
// TODO: The name of this method is misnomer.
// It will be renamed to #finish() in the next major release of httpcore
httpEntity.consumeContent();
// Ensures that the entity content is fully consumed and the content stream, if exists, is closed.
EntityUtils.consume(httpEntity);
}
}
if (currentRequest != null) {
@ -454,10 +447,9 @@ public class HTTPClient {
if (httpEntity != null) {
if (getStatusCode() == 200 && httpEntity.getContentLength() < maxBytes) {
content = EntityUtils.toByteArray(httpEntity);
}
// TODO: The name of this method is misnomer.
// It will be renamed to #finish() in the next major release of httpcore
httpEntity.consumeContent();
}
// Ensures that the entity content is fully consumed and the content stream, if exists, is closed.
EntityUtils.consume(httpEntity);
}
} catch (final IOException e) {
ConnectionInfo.removeConnection(httpUriRequest.hashCode());
@ -560,8 +552,7 @@ public class HTTPClient {
// e.printStackTrace();
}
final SSLSocketFactory sslSF = new SSLSocketFactory(sslContext);
sslSF.setHostnameVerifier(SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
final SSLSocketFactory sslSF = new SSLSocketFactory(sslContext, SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
return sslSF;
}

Write Preview
Loading…
Cancel
Save