From 69adaa9f55d78abf0cc01dab078d2dd1851eb3ef Mon Sep 17 00:00:00 2001 From: sgaebel Date: Sun, 31 Oct 2021 15:07:17 +0100 Subject: [PATCH] makes our HTTPClient closable --- .../yacy/cora/protocol/http/HTTPClient.java | 32 ++++++++++--------- .../net/yacy/cora/util/HTTPInputStream.java | 2 +- .../yacy/crawler/retrieval/HTTPLoader.java | 6 ++-- .../xml/opensearchdescriptionReader.java | 4 +-- source/net/yacy/http/ProxyHandler.java | 2 +- .../yacy/server/http/HTTPDProxyHandler.java | 4 +-- 6 files changed, 26 insertions(+), 24 deletions(-) diff --git a/source/net/yacy/cora/protocol/http/HTTPClient.java b/source/net/yacy/cora/protocol/http/HTTPClient.java index 834f4fdf1..31230b166 100644 --- a/source/net/yacy/cora/protocol/http/HTTPClient.java +++ b/source/net/yacy/cora/protocol/http/HTTPClient.java @@ -25,6 +25,7 @@ package net.yacy.cora.protocol.http; +import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -119,7 +120,7 @@ import net.yacy.kelondro.util.NamePrefixThreadFactory; * @author sixcooler * */ -public class HTTPClient { +public class HTTPClient implements Closeable { private static final int default_timeout = 6000; @@ -468,7 +469,7 @@ public class HTTPClient { } } } finally { - finish(); + close(); } } return null; @@ -477,7 +478,7 @@ public class HTTPClient { /** * This method GETs a page from the server. * to be used for streaming out - * Please take care to call finish()! + * Please take care to call close()! * * @param uri the url to get * @throws IOException @@ -489,7 +490,7 @@ public class HTTPClient { /** * This method GETs a page from the server. * to be used for streaming out - * Please take care to call finish()! + * Please take care to call close()! * * @param url the url to get * @throws IOException @@ -536,7 +537,7 @@ public class HTTPClient { /** * This method POSTs a page from the server. * to be used for streaming out - * Please take care to call finish()! + * Please take care to call close()! * * @param uri the url to post * @param instream the input to post @@ -552,7 +553,7 @@ public class HTTPClient { /** * This method POSTs a page from the server. * to be used for streaming out - * Please take care to call finish()! + * Please take care to call close()! * * @param url the url to post * @param instream the input to post @@ -652,7 +653,7 @@ public class HTTPClient { } } } finally { - finish(); + close(); } } return null; @@ -770,7 +771,7 @@ public class HTTPClient { /** * This method gets direct access to the content-stream * Since this way is uncontrolled by the Client think of using 'writeTo' instead! - * Please take care to call finish()! + * Please take care to call close()! * * @return the content as InputStream * @throws IOException @@ -781,7 +782,7 @@ public class HTTPClient { if (httpEntity != null) try { return httpEntity.getContent(); } catch (final IOException e) { - finish(); + close(); throw e; } } @@ -790,7 +791,7 @@ public class HTTPClient { /** * This method streams the content to the outputStream - * Please take care to call finish()! + * Please take care to call close()! * * @param outputStream * @throws IOException @@ -802,18 +803,19 @@ public class HTTPClient { httpEntity.writeTo(outputStream); outputStream.flush(); } finally { - finish(); + close(); } } } /** - * This method ensures correct finish of client-connections + * This method ensures correct close of client-connections * This method should be used after every use of GET or POST and writeTo or getContentstream! * * @throws IOException */ - public void finish() throws IOException { + @Override + public void close() throws IOException { try { if (this.httpResponse != null) { // Ensures that the entity content Stream is closed. @@ -850,7 +852,7 @@ public class HTTPClient { } } } finally { - finish(); + close(); } return null; } @@ -904,7 +906,7 @@ public class HTTPClient { } this.httpResponse.setHeader(HeaderFramework.RESPONSE_TIME_MILLIS, Long.toString(System.currentTimeMillis() - time)); } catch (final Throwable e) { - finish(); + close(); throw new IOException("Client can't execute: " + (e.getCause() == null ? e.getMessage() : e.getCause().getMessage()) + " duration=" + Long.toString(System.currentTimeMillis() - time) + " for url " + uri); diff --git a/source/net/yacy/cora/util/HTTPInputStream.java b/source/net/yacy/cora/util/HTTPInputStream.java index 035edfad9..94eb9e957 100755 --- a/source/net/yacy/cora/util/HTTPInputStream.java +++ b/source/net/yacy/cora/util/HTTPInputStream.java @@ -60,7 +60,7 @@ public class HTTPInputStream extends InputStream { */ @Override public void close() throws IOException { - httpClient.finish(); + httpClient.close(); } diff --git a/source/net/yacy/crawler/retrieval/HTTPLoader.java b/source/net/yacy/crawler/retrieval/HTTPLoader.java index 62edd0fbf..9c4497293 100644 --- a/source/net/yacy/crawler/retrieval/HTTPLoader.java +++ b/source/net/yacy/crawler/retrieval/HTTPLoader.java @@ -156,7 +156,7 @@ public final class HTTPLoader { // check redirection if (statusCode > 299 && statusCode < 310) { - client.finish(); + client.close(); final DigestURL redirectionUrl = extractRedirectURL(request, profile, url, statusline, responseHeader, requestURLString); @@ -225,7 +225,7 @@ public final class HTTPLoader { } catch (final IOException e) { this.log.warn("cannot write " + url + " to Cache (3): " + e.getMessage(), e); } finally { - client.finish(); + client.close(); } contentStream = new ByteArrayInputStream(content); @@ -251,7 +251,7 @@ public final class HTTPLoader { return new StreamResponse(new Response(request, requestHeader, responseHeader, profile, false, null), contentStream); } else { - client.finish(); + client.close(); // if the response has not the right response type then reject file this.sb.crawlQueues.errorURL.push(request.url(), request.depth(), profile, FailCategory.TEMPORARY_NETWORK_FAILURE, "wrong http status code", statusCode); diff --git a/source/net/yacy/document/parser/xml/opensearchdescriptionReader.java b/source/net/yacy/document/parser/xml/opensearchdescriptionReader.java index e6a20a064..54bc8dbf9 100644 --- a/source/net/yacy/document/parser/xml/opensearchdescriptionReader.java +++ b/source/net/yacy/document/parser/xml/opensearchdescriptionReader.java @@ -156,7 +156,7 @@ public class opensearchdescriptionReader extends DefaultHandler { ConcurrentLog.logException(e); } finally { try { - www.finish(); + www.close(); } catch (final IOException e) { ConcurrentLog.logException(e); } @@ -187,7 +187,7 @@ public class opensearchdescriptionReader extends DefaultHandler { return false; } finally { try { - www.finish(); + www.close(); } catch (final IOException e) { ConcurrentLog.logException(e); } diff --git a/source/net/yacy/http/ProxyHandler.java b/source/net/yacy/http/ProxyHandler.java index 7dcfc438b..5e1b482f2 100644 --- a/source/net/yacy/http/ProxyHandler.java +++ b/source/net/yacy/http/ProxyHandler.java @@ -220,7 +220,7 @@ public class ProxyHandler extends AbstractRemoteHandler implements Handler { } catch (final SocketException se) { throw new ServletException("Socket Exception: " + se.getMessage()); } finally { - client.finish(); + client.close(); } // we handled this request, break out of handler chain diff --git a/source/net/yacy/server/http/HTTPDProxyHandler.java b/source/net/yacy/server/http/HTTPDProxyHandler.java index c63b4ee9c..2d7ab3ebd 100644 --- a/source/net/yacy/server/http/HTTPDProxyHandler.java +++ b/source/net/yacy/server/http/HTTPDProxyHandler.java @@ -601,7 +601,7 @@ public final class HTTPDProxyHandler { // // client cut proxy connection, abort download // res.abort(); // } - client.finish(); + client.close(); handleProxyException(se,conProp,respond,url); } finally { // if opened ... @@ -609,7 +609,7 @@ public final class HTTPDProxyHandler { // // ... close connection // res.closeStream(); // } - client.finish(); + client.close(); } } catch (final Exception e) { handleProxyException(e,conProp,respond,url);