|
|
@ -452,10 +452,8 @@ public class HTTPClient {
|
|
|
|
.register(AuthSchemes.DIGEST, YACY_DIGEST_SCHEME_FACTORY)
|
|
|
|
.register(AuthSchemes.DIGEST, YACY_DIGEST_SCHEME_FACTORY)
|
|
|
|
.build();
|
|
|
|
.build();
|
|
|
|
|
|
|
|
|
|
|
|
CloseableHttpClient httpclient = HttpClients.custom().setDefaultCredentialsProvider(credsProvider)
|
|
|
|
try (final CloseableHttpClient httpclient = HttpClients.custom().setDefaultCredentialsProvider(credsProvider)
|
|
|
|
.setDefaultAuthSchemeRegistry(authSchemeRegistry).build();
|
|
|
|
.setDefaultAuthSchemeRegistry(authSchemeRegistry).build()) {
|
|
|
|
byte[] content = null;
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
|
|
|
this.httpResponse = httpclient.execute(httpGet);
|
|
|
|
this.httpResponse = httpclient.execute(httpGet);
|
|
|
|
try {
|
|
|
|
try {
|
|
|
|
HttpEntity httpEntity = this.httpResponse.getEntity();
|
|
|
|
HttpEntity httpEntity = this.httpResponse.getEntity();
|
|
|
@ -467,10 +465,8 @@ public class HTTPClient {
|
|
|
|
* Otherwise returning null and consuming fully the entity can be very long on large resources */
|
|
|
|
* Otherwise returning null and consuming fully the entity can be very long on large resources */
|
|
|
|
throw new IOException("Content to download exceed maximum value of " + Formatter.bytesToString(maxBytes));
|
|
|
|
throw new IOException("Content to download exceed maximum value of " + Formatter.bytesToString(maxBytes));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
content = getByteArray(httpEntity, maxBytes);
|
|
|
|
return getByteArray(httpEntity, maxBytes);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// Ensures that the entity content is fully consumed and the content stream, if exists, is closed.
|
|
|
|
|
|
|
|
EntityUtils.consume(httpEntity);
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} catch (final IOException e) {
|
|
|
|
} catch (final IOException e) {
|
|
|
|
httpGet.abort();
|
|
|
|
httpGet.abort();
|
|
|
@ -478,10 +474,8 @@ public class HTTPClient {
|
|
|
|
} finally {
|
|
|
|
} finally {
|
|
|
|
this.httpResponse.close();
|
|
|
|
this.httpResponse.close();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} finally {
|
|
|
|
|
|
|
|
httpclient.close();
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return content;
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
/**
|
|
|
@ -649,8 +643,6 @@ public class HTTPClient {
|
|
|
|
return getContentBytes(httpPost, Integer.MAX_VALUE, concurrent);
|
|
|
|
return getContentBytes(httpPost, Integer.MAX_VALUE, concurrent);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
byte[] content = null;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
final CredentialsProvider credsProvider = new BasicCredentialsProvider();
|
|
|
|
final CredentialsProvider credsProvider = new BasicCredentialsProvider();
|
|
|
|
credsProvider.setCredentials(
|
|
|
|
credsProvider.setCredentials(
|
|
|
|
new AuthScope("localhost", url.getPort()),
|
|
|
|
new AuthScope("localhost", url.getPort()),
|
|
|
@ -662,27 +654,21 @@ public class HTTPClient {
|
|
|
|
.register(AuthSchemes.DIGEST, YACY_DIGEST_SCHEME_FACTORY)
|
|
|
|
.register(AuthSchemes.DIGEST, YACY_DIGEST_SCHEME_FACTORY)
|
|
|
|
.build();
|
|
|
|
.build();
|
|
|
|
|
|
|
|
|
|
|
|
CloseableHttpClient httpclient = HttpClients.custom().setDefaultCredentialsProvider(credsProvider)
|
|
|
|
try (final CloseableHttpClient httpclient = HttpClients.custom().setDefaultCredentialsProvider(credsProvider)
|
|
|
|
.setDefaultAuthSchemeRegistry(authSchemeRegistry).build();
|
|
|
|
.setDefaultAuthSchemeRegistry(authSchemeRegistry).build()) {
|
|
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
|
|
|
this.httpResponse = httpclient.execute(httpPost);
|
|
|
|
this.httpResponse = httpclient.execute(httpPost);
|
|
|
|
try {
|
|
|
|
try {
|
|
|
|
HttpEntity httpEntity = this.httpResponse.getEntity();
|
|
|
|
HttpEntity httpEntity = this.httpResponse.getEntity();
|
|
|
|
if (httpEntity != null) {
|
|
|
|
if (httpEntity != null) {
|
|
|
|
if (getStatusCode() == HttpStatus.SC_OK) {
|
|
|
|
if (getStatusCode() == HttpStatus.SC_OK) {
|
|
|
|
content = getByteArray(httpEntity, Integer.MAX_VALUE);
|
|
|
|
return getByteArray(httpEntity, Integer.MAX_VALUE);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// Ensures that the entity content is fully consumed and the content stream, if exists, is closed.
|
|
|
|
|
|
|
|
EntityUtils.consume(httpEntity);
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} finally {
|
|
|
|
} finally {
|
|
|
|
this.httpResponse.close();
|
|
|
|
this.httpResponse.close();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} finally {
|
|
|
|
|
|
|
|
httpclient.close();
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return content;
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
/**
|
|
|
@ -920,8 +906,6 @@ public class HTTPClient {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
content = getByteArray(httpEntity, maxBytes);
|
|
|
|
content = getByteArray(httpEntity, maxBytes);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// Ensures that the entity content is fully consumed and the content stream, if exists, is closed.
|
|
|
|
|
|
|
|
EntityUtils.consume(httpEntity);
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} catch (final IOException e) {
|
|
|
|
} catch (final IOException e) {
|
|
|
|
httpUriRequest.abort();
|
|
|
|
httpUriRequest.abort();
|
|
|
@ -1001,11 +985,10 @@ public class HTTPClient {
|
|
|
|
* @throws IOException when a read error occured or content length is over maxBytes
|
|
|
|
* @throws IOException when a read error occured or content length is over maxBytes
|
|
|
|
*/
|
|
|
|
*/
|
|
|
|
public static byte[] getByteArray(final HttpEntity entity, int maxBytes) throws IOException {
|
|
|
|
public static byte[] getByteArray(final HttpEntity entity, int maxBytes) throws IOException {
|
|
|
|
final InputStream instream = entity.getContent();
|
|
|
|
try (final InputStream instream = entity.getContent()) {
|
|
|
|
if (instream == null) {
|
|
|
|
if (instream == null) {
|
|
|
|
return null;
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
try {
|
|
|
|
|
|
|
|
long contentLength = entity.getContentLength();
|
|
|
|
long contentLength = entity.getContentLength();
|
|
|
|
/*
|
|
|
|
/*
|
|
|
|
* When no maxBytes is specified, the default limit is
|
|
|
|
* When no maxBytes is specified, the default limit is
|
|
|
@ -1046,7 +1029,8 @@ public class HTTPClient {
|
|
|
|
} catch (final OutOfMemoryError e) {
|
|
|
|
} catch (final OutOfMemoryError e) {
|
|
|
|
throw new IOException(e.toString());
|
|
|
|
throw new IOException(e.toString());
|
|
|
|
} finally {
|
|
|
|
} finally {
|
|
|
|
instream.close();
|
|
|
|
// Ensures that the entity content is fully consumed and the content stream, if exists, is closed.
|
|
|
|
|
|
|
|
EntityUtils.consume(entity);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|