boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

handle all references for client, response, request to be able to close

Browse Source 
them
pull/436/head
sgaebel 3 years ago
parent 1cdc55a425
commit fc4275f901
1 changed files with 59 additions and 61 deletions
Show Stats Download Patch File Download Diff File

120
source/net/yacy/cora/protocol/http/HTTPClient.java
Unescape View File

@ -170,14 +170,18 @@ public class HTTPClient {
private final static HttpClientBuilder clientBuilder = initClientBuilder();
private final RequestConfig.Builder reqConfBuilder;
private Set<Entry<String, String>> headers = null;
private CloseableHttpResponse httpResponse = null;
private HttpUriRequest currentRequest = null;
private long upbytes = 0L;
private String host = null;
private final long timeout;
private static ExecutorService executor = Executors
.newCachedThreadPool(new NamePrefixThreadFactory(HTTPClient.class.getSimpleName() + ".execute"));
/** these are the main variable to hold information and to take care of closing: */
private CloseableHttpClient client = null;
private CloseableHttpResponse httpResponse = null;
private HttpUriRequest currentRequest = null;
public HTTPClient(final ClientIdentification.Agent agent) {
super();
this.timeout = agent.clientTimeout;
@ -214,6 +218,8 @@ public class HTTPClient {
final HttpClientBuilder builder = HttpClientBuilder.create();
builder.setConnectionManager(CONNECTION_MANAGER);
builder.setConnectionManagerShared(true);
builder.setDefaultRequestConfig(DFLTREQUESTCONFIG);
// UserAgent
@ -429,15 +435,15 @@ public class HTTPClient {
public byte[] GETbytes(final MultiProtocolURL url, final String username, final String pass, final int maxBytes, final boolean concurrent) throws IOException {
final boolean localhost = Domains.isLocalhost(url.getHost());
final String urix = url.toNormalform(true);
HttpGet httpGet = null;
try {
httpGet = new HttpGet(urix);
this.currentRequest = new HttpGet(urix);
} catch (IllegalArgumentException e) {
throw new IOException(e.getMessage()); // can be caused at java.net.URI.create()
}
if (!localhost) setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
if (!localhost || pass == null) {
return getContentBytes(httpGet, maxBytes, concurrent);
return getContentBytes(maxBytes, concurrent);
}
CredentialsProvider credsProvider = new BasicCredentialsProvider();
@ -447,7 +453,7 @@ public class HTTPClient {
try (final CloseableHttpClient httpclient = HttpClients.custom().setDefaultCredentialsProvider(credsProvider)
.setDefaultAuthSchemeRegistry(AUTHSCHEMEREGISTRY).build()) {
this.httpResponse = httpclient.execute(httpGet);
this.httpResponse = httpclient.execute(this.currentRequest);
try {
HttpEntity httpEntity = this.httpResponse.getEntity();
if (httpEntity != null) {
@ -461,11 +467,8 @@ public class HTTPClient {
return getByteArray(httpEntity, maxBytes);
}
}
} catch (final IOException e) {
httpGet.abort();
throw e;
} finally {
this.httpResponse.close();
finish();
}
}
return null;
@ -494,15 +497,15 @@ public class HTTPClient {
public void GET(final MultiProtocolURL url, final boolean concurrent) throws IOException {
if (this.currentRequest != null) throw new IOException("Client is in use!");
final String urix = url.toNormalform(true);
HttpGet httpGet = null;
try {
httpGet = new HttpGet(urix);
this.currentRequest = new HttpGet(urix);
} catch (IllegalArgumentException e) {
throw new IOException(e.getMessage()); // can be caused at java.net.URI.create()
}
setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
this.currentRequest = httpGet;
execute(httpGet, concurrent);
execute(concurrent);
}
/**
@ -524,11 +527,9 @@ public class HTTPClient {
* @throws IOException
*/
public HttpResponse HEADResponse(final MultiProtocolURL url, final boolean concurrent) throws IOException {
final HttpHead httpHead = new HttpHead(url.toNormalform(true));
this.currentRequest = new HttpHead(url.toNormalform(true));
setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
execute(httpHead, concurrent);
finish();
ConnectionInfo.removeConnection(httpHead.hashCode());
execute(concurrent);
return this.httpResponse;
}
@ -560,16 +561,15 @@ public class HTTPClient {
*/
public void POST(final MultiProtocolURL url, final InputStream instream, final long length, final boolean concurrent) throws IOException {
if (this.currentRequest != null) throw new IOException("Client is in use!");
final HttpPost httpPost = new HttpPost(url.toNormalform(true));
this.currentRequest = new HttpPost(url.toNormalform(true));
String host = url.getHost();
if (host == null) host = Domains.LOCALHOST;
setHost(host); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
final NonClosingInputStreamEntity inputStreamEntity = new NonClosingInputStreamEntity(instream, length);
// statistics
this.upbytes = length;
httpPost.setEntity(inputStreamEntity);
this.currentRequest = httpPost;
execute(httpPost, concurrent);
((HttpPost) this.currentRequest).setEntity(inputStreamEntity);
execute(concurrent);
}
/**
@ -615,7 +615,7 @@ public class HTTPClient {
*/
public byte[] POSTbytes(final MultiProtocolURL url, final String vhost, final Map<String, ContentBody> post,
final String userName, final String password, final boolean usegzip, final boolean concurrent) throws IOException {
final HttpPost httpPost = new HttpPost(url.toNormalform(true));
this.currentRequest = new HttpPost(url.toNormalform(true));
final boolean localhost = Domains.isLocalhost(url.getHost());
if (!localhost) setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
if (vhost == null) setHost(Domains.LOCALHOST);
@ -627,13 +627,13 @@ public class HTTPClient {
this.upbytes = multipartEntity.getContentLength();
if (usegzip) {
httpPost.setEntity(new GzipCompressingEntity(multipartEntity));
((HttpPost) this.currentRequest).setEntity(new GzipCompressingEntity(multipartEntity));
} else {
httpPost.setEntity(multipartEntity);
((HttpPost) this.currentRequest).setEntity(multipartEntity);
}
if (!localhost || password == null) {
return getContentBytes(httpPost, Integer.MAX_VALUE, concurrent);
return getContentBytes(Integer.MAX_VALUE, concurrent);
}
final CredentialsProvider credsProvider = new BasicCredentialsProvider();
@ -643,7 +643,7 @@ public class HTTPClient {
try (final CloseableHttpClient httpclient = HttpClients.custom().setDefaultCredentialsProvider(credsProvider)
.setDefaultAuthSchemeRegistry(AUTHSCHEMEREGISTRY).build()) {
this.httpResponse = httpclient.execute(httpPost);
this.httpResponse = httpclient.execute(this.currentRequest);
try {
HttpEntity httpEntity = this.httpResponse.getEntity();
if (httpEntity != null) {
@ -652,7 +652,7 @@ public class HTTPClient {
}
}
} finally {
this.httpResponse.close();
finish();
}
}
return null;
@ -820,6 +820,9 @@ public class HTTPClient {
EntityUtils.consumeQuietly(this.httpResponse.getEntity());
this.httpResponse.close();
}
if (this.client != null) {
client.close();
}
} finally {
if (this.currentRequest != null) {
ConnectionInfo.removeConnection(this.currentRequest.hashCode());
@ -829,9 +832,9 @@ public class HTTPClient {
}
}
private byte[] getContentBytes(final HttpUriRequest httpUriRequest, final int maxBytes, final boolean concurrent) throws IOException {
private byte[] getContentBytes(final int maxBytes, final boolean concurrent) throws IOException {
try {
execute(httpUriRequest, concurrent);
execute(concurrent);
if (this.httpResponse == null) return null;
// get the response body
final HttpEntity httpEntity = this.httpResponse.getEntity();
@ -846,28 +849,24 @@ public class HTTPClient {
return getByteArray(httpEntity, maxBytes);
}
}
} catch (final IOException e) {
httpUriRequest.abort();
throw e;
} finally {
if (this.httpResponse != null) this.httpResponse.close();
ConnectionInfo.removeConnection(httpUriRequest.hashCode());
finish();
}
return null;
}
private void execute(final HttpUriRequest httpUriRequest, final boolean concurrent) throws IOException {
private void execute(final boolean concurrent) throws IOException {
final HttpClientContext context = HttpClientContext.create();
context.setRequestConfig(reqConfBuilder.build());
if (this.host != null)
context.setTargetHost(new HttpHost(this.host));
setHeaders(httpUriRequest);
setHeaders();
// statistics
storeConnectionInfo(httpUriRequest);
storeConnectionInfo();
// execute the method; some asserts confirm that that the request can be send with Content-Length and is therefore not terminated by EOF
if (httpUriRequest instanceof HttpEntityEnclosingRequest) {
final HttpEntityEnclosingRequest hrequest = (HttpEntityEnclosingRequest) httpUriRequest;
if (this.currentRequest instanceof HttpEntityEnclosingRequest) {
final HttpEntityEnclosingRequest hrequest = (HttpEntityEnclosingRequest) this.currentRequest;
final HttpEntity entity = hrequest.getEntity();
assert entity != null;
//assert !entity.isChunked();
@ -876,16 +875,17 @@ public class HTTPClient {
}
final String initialThreadName = Thread.currentThread().getName();
Thread.currentThread().setName("HTTPClient-" + httpUriRequest.getURI());
final String uri = this.currentRequest.getURI().toString();
Thread.currentThread().setName("HTTPClient-" + uri);
final long time = System.currentTimeMillis();
try {
this.client = clientBuilder.build();
if (concurrent) {
FutureTask<CloseableHttpResponse> t = new FutureTask<CloseableHttpResponse>(new Callable<CloseableHttpResponse>() {
@Override
public CloseableHttpResponse call() throws ClientProtocolException, IOException {
final CloseableHttpClient client = clientBuilder.build();
CloseableHttpResponse response = client.execute(httpUriRequest, context);
CloseableHttpResponse response = client.execute(currentRequest, context);
return response;
}
});
@ -896,20 +896,18 @@ public class HTTPClient {
throw e.getCause();
} catch (Throwable e) {}
try {t.cancel(true);} catch (Throwable e) {}
if (this.httpResponse == null) throw new IOException("timout to client after " + this.timeout + "ms" + " for url " + httpUriRequest.getURI().toString());
if (this.httpResponse == null) {
throw new IOException("timout to client after " + this.timeout + "ms" + " for url " + uri);
}
} else {
final CloseableHttpClient client = clientBuilder.build();
this.httpResponse = client.execute(httpUriRequest, context);
this.httpResponse = client.execute(this.currentRequest, context);
}
this.httpResponse.setHeader(HeaderFramework.RESPONSE_TIME_MILLIS, Long.toString(System.currentTimeMillis() - time));
} catch (final Throwable e) {
ConnectionInfo.removeConnection(httpUriRequest.hashCode());
httpUriRequest.abort();
if (this.httpResponse != null) this.httpResponse.close();
//e.printStackTrace();
finish();
throw new IOException("Client can't execute: "
+ (e.getCause() == null ? e.getMessage() : e.getCause().getMessage())
+ " duration=" + Long.toString(System.currentTimeMillis() - time) + " for url " + httpUriRequest.getURI().toString());
+ " duration=" + Long.toString(System.currentTimeMillis() - time) + " for url " + uri);
} finally {
/* Restore the thread initial name */
Thread.currentThread().setName(initialThreadName);
@ -973,25 +971,25 @@ public class HTTPClient {
}
}
private void setHeaders(final HttpUriRequest httpUriRequest) {
private void setHeaders() {
if (this.headers != null) {
for (final Entry<String, String> entry : this.headers) {
httpUriRequest.setHeader(entry.getKey(),entry.getValue());
this.currentRequest.setHeader(entry.getKey(),entry.getValue());
}
}
if (this.host != null) httpUriRequest.setHeader(HTTP.TARGET_HOST, this.host);
httpUriRequest.setHeader(HTTP.CONN_DIRECTIVE, "close"); // don't keep alive, prevent CLOSE_WAIT state
if (this.host != null) this.currentRequest.setHeader(HTTP.TARGET_HOST, this.host);
this.currentRequest.setHeader(HTTP.CONN_DIRECTIVE, "close"); // don't keep alive, prevent CLOSE_WAIT state
}
private void storeConnectionInfo(final HttpUriRequest httpUriRequest) {
final int port = httpUriRequest.getURI().getPort();
final String thost = httpUriRequest.getURI().getHost();
private void storeConnectionInfo() {
final int port = this.currentRequest.getURI().getPort();
final String thost = this.currentRequest.getURI().getHost();
//assert thost != null : "uri = " + httpUriRequest.getURI().toString();
ConnectionInfo.addConnection(new ConnectionInfo(
httpUriRequest.getURI().getScheme(),
this.currentRequest.getURI().getScheme(),
port == -1 ? thost : thost + ":" + port,
httpUriRequest.getMethod() + " " + httpUriRequest.getURI().getPath(),
httpUriRequest.hashCode(),
this.currentRequest.getMethod() + " " + this.currentRequest.getURI().getPath(),
this.currentRequest.hashCode(),
System.currentTimeMillis(),
this.upbytes));
}

Write Preview
Loading…
Cancel
Save