From 202a3adb3ed1b98777ce9a0b6f01c0f731fa9c13 Mon Sep 17 00:00:00 2001 From: orbiter Date: Thu, 10 Apr 2008 22:47:05 +0000 Subject: [PATCH] refactoring of HttpClient Writer processes git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4678 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/Network.java | 2 +- source/de/anomic/data/SitemapParser.java | 7 +- source/de/anomic/data/robotsParser.java | 15 +- source/de/anomic/http/HttpClient.java | 78 +--------- source/de/anomic/http/HttpResponse.java | 135 ------------------ .../anomic/http/JakartaCommonsHttpClient.java | 20 +-- .../http/JakartaCommonsHttpResponse.java | 3 +- source/de/anomic/http/httpdProxyHandler.java | 65 ++++++--- .../index/indexRepositoryReference.java | 7 +- .../plasma/crawler/plasmaHTTPLoader.java | 5 +- source/de/anomic/plasma/plasmaParser.java | 7 +- source/de/anomic/yacy/yacyClient.java | 6 +- source/de/anomic/yacy/yacySeedDB.java | 7 +- source/de/anomic/yacy/yacyVersion.java | 16 ++- source/yacy.java | 14 +- 15 files changed, 113 insertions(+), 274 deletions(-) delete mode 100644 source/de/anomic/http/HttpResponse.java diff --git a/htroot/Network.java b/htroot/Network.java index c51863038..7ef2fde04 100644 --- a/htroot/Network.java +++ b/htroot/Network.java @@ -375,7 +375,7 @@ public class Network { prop.putHTML(STR_TABLE_LIST + conCount + "_fullname", seed.get(yacySeed.NAME, "deadlink")); userAgent = null; if (seed.hash.equals(yacyCore.seedDB.mySeed().hash)) { - final HttpClient httpClient = new JakartaCommonsHttpClient(10000, null, null); + final JakartaCommonsHttpClient httpClient = new JakartaCommonsHttpClient(10000, null, null); userAgent = httpClient.getUserAgent(); location = HttpClient.generateLocation(); } else { diff --git a/source/de/anomic/data/SitemapParser.java b/source/de/anomic/data/SitemapParser.java index 8f511c64c..724db0a07 100644 --- a/source/de/anomic/data/SitemapParser.java +++ b/source/de/anomic/data/SitemapParser.java @@ -58,9 +58,8 @@ import org.xml.sax.Attributes; import org.xml.sax.SAXException; import org.xml.sax.helpers.DefaultHandler; -import de.anomic.http.HttpClient; -import de.anomic.http.HttpResponse; import de.anomic.http.JakartaCommonsHttpClient; +import de.anomic.http.JakartaCommonsHttpResponse; import de.anomic.http.httpdByteCountInputStream; import de.anomic.index.indexURLReference; import de.anomic.plasma.plasmaCrawlProfile; @@ -178,8 +177,8 @@ public class SitemapParser extends DefaultHandler { */ public void parse() { // download document - HttpClient client = new JakartaCommonsHttpClient(5000, null, null); - HttpResponse res = null; + JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(5000, null, null); + JakartaCommonsHttpResponse res = null; try { res = client.GET(siteMapURL.toString()); if (res.getStatusCode() != 200) { diff --git a/source/de/anomic/data/robotsParser.java b/source/de/anomic/data/robotsParser.java index 5bd9be078..949e713de 100644 --- a/source/de/anomic/data/robotsParser.java +++ b/source/de/anomic/data/robotsParser.java @@ -43,6 +43,7 @@ package de.anomic.data; +import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.BufferedReader; import java.io.ByteArrayInputStream; @@ -56,13 +57,13 @@ import java.util.ArrayList; import java.util.Date; import de.anomic.http.HttpClient; -import de.anomic.http.HttpResponse; import de.anomic.http.JakartaCommonsHttpClient; +import de.anomic.http.JakartaCommonsHttpResponse; import de.anomic.http.httpHeader; -import de.anomic.http.HttpResponse.Saver; import de.anomic.plasma.plasmaCrawlRobotsTxt; import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverByteBuffer; +import de.anomic.server.serverFileUtils; import de.anomic.server.logging.serverLog; import de.anomic.yacy.yacyURL; @@ -405,8 +406,8 @@ public final class robotsParser{ // setup http-client //TODO: adding Traffic statistic for robots download? - HttpClient client = new JakartaCommonsHttpClient(10000, reqHeaders, null); - HttpResponse res = null; + JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(10000, reqHeaders, null); + JakartaCommonsHttpResponse res = null; try { // sending the get request res = client.GET(robotsURL.toString()); @@ -433,7 +434,11 @@ public final class robotsParser{ // downloading the content serverByteBuffer sbb = new serverByteBuffer(); - Saver.writeContent(res, new BufferedOutputStream(sbb), null); + try { + serverFileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(sbb)); + } finally { + res.closeStream(); + } robotsTxt = sbb.getBytes(); downloadEnd = System.currentTimeMillis(); diff --git a/source/de/anomic/http/HttpClient.java b/source/de/anomic/http/HttpClient.java index 454645f7b..6b4d5161f 100644 --- a/source/de/anomic/http/HttpClient.java +++ b/source/de/anomic/http/HttpClient.java @@ -23,11 +23,11 @@ // You should have received a copy of the GNU General Public License // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + package de.anomic.http; import java.io.IOException; import java.util.Date; -import java.util.Map; import de.anomic.server.logging.serverLog; @@ -37,6 +37,7 @@ import de.anomic.server.logging.serverLog; * some methods must be implemented (the "socket-layer") */ public abstract class HttpClient { + private static final String systemOST; static { // provide system information for client identification @@ -68,73 +69,6 @@ public abstract class HttpClient { return systemOST; } - /** - * "This method can be used for obtaining metainformation about the entity implied by the request without - * transferring the entity-body itself" [RFC 2616, sect. 9.4] - * - * @param uri - * @return response header - * @throws IOException if data cannot be read - */ - public abstract HttpResponse HEAD(String uri) throws IOException; - - /** - * "The GET method means retrieve whatever information [...] is identified by the Request-URI" [RFC 2616, sect. 9.3] - * - * @param uri - * @return response body - * @throws IOException if data cannot be read - */ - public abstract HttpResponse GET(String uri) throws IOException; - - /** - * "Providing a block of data [...] to a data-handling process;" [RFC 2616, sect. 9.5] - * - * @param uri - * @param nameDataPairs - * @return response body - * @throws IOException if data cannot be read - */ - public abstract HttpResponse POST(String uri, Map nameDataPairs) throws IOException; - - /** - * "for use with a proxy that can dynamically switch to being a tunnel" [RFC 2616, sect. 9.9] - * - * @param host - * @param port - * @return - * @throws IOException - */ - public abstract HttpResponse CONNECT(String host, int port) throws IOException; - - /** - * use proxyConfig to establish the connection - * - * @param proxyConfig - */ - public abstract void setProxy(httpRemoteProxyConfig proxyConfig); - - /** - * sets the header for all coming requests - * - * @param header may be null - */ - public abstract void setHeader(httpHeader header); - - /** - * sets the timeout in milliseconds - * - * @param timeout - */ - public abstract void setTimeout(int timeout); - - /** - * gives the user-agent used by this http-client - * - * @return - */ - public abstract String getUserAgent(); - /** * for easy access * @@ -213,7 +147,7 @@ public abstract class HttpClient { */ public static byte[] wget(final String uri, httpHeader header, final String vhost, int timeout) { assert uri != null : "precondition violated: uri != null"; - final HttpClient client = new JakartaCommonsHttpClient(timeout, null, null); + final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, null, null); // set header header = addHostHeader(header, vhost); @@ -221,7 +155,7 @@ public abstract class HttpClient { // do the request try { - final HttpResponse response = client.GET(uri); + final JakartaCommonsHttpResponse response = client.GET(uri); return response.getData(); } catch (final IOException e) { serverLog.logWarning("HTTPC", "wget(" + uri + ") failed: " + e.getMessage()); @@ -265,9 +199,9 @@ public abstract class HttpClient { * @return */ public static httpHeader whead(final String uri, final httpHeader header) { - final de.anomic.http.HttpClient client = new JakartaCommonsHttpClient(10000, header, null); + final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(10000, header, null); try { - final HttpResponse response = client.HEAD(uri); + final JakartaCommonsHttpResponse response = client.HEAD(uri); return response.getResponseHeader(); } catch (final IOException e) { serverLog.logWarning("HTTPC", "whead(" + uri + ") failed: " + e.getMessage()); diff --git a/source/de/anomic/http/HttpResponse.java b/source/de/anomic/http/HttpResponse.java deleted file mode 100644 index 8a3ea03f3..000000000 --- a/source/de/anomic/http/HttpResponse.java +++ /dev/null @@ -1,135 +0,0 @@ -// HttpResponse.java -// (C) 2008 by Daniel Raap; danielr@users.berlios.de -// first published 2.4.2008 on http://yacy.net -// -// This is a part of YaCy, a peer-to-peer based web search engine -// -// $LastChangedDate: 2008-03-14 01:16:04 +0100 (Fr, 14 Mrz 2008) $ -// $LastChangedRevision: 4558 $ -// $LastChangedBy: orbiter $ -// -// LICENSE -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -package de.anomic.http; - -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.BufferedWriter; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.UnsupportedEncodingException; - -import de.anomic.server.serverFileUtils; - -/** - * @author daniel - * - */ -public interface HttpResponse { - - /** - * returns the header as {@link de.anomic.http.httpHeader} - * @return - */ - public abstract httpHeader getResponseHeader(); - - /** - * get the body of the response - * @return the data - * @throws IOException - */ - public abstract byte[] getData() throws IOException; - - /** - * returns a stream to read the body of the response - * - * Important: When done call closeStream()! - * @return - * @throws IOException - */ - public abstract InputStream getDataAsStream() throws IOException; - - /** - * closes the input stream - */ - public abstract void closeStream(); - - /** - * "Statuscode SPACE Statusmessage" - * @return the statusLine - */ - public abstract String getStatusLine(); - - /** - * the status code of this response - * @return - * @ensure result >= 100 && result <= 999 (is a 3-digit integer) - */ - public abstract int getStatusCode(); - - /** - * HTTP version of the response - * @return - */ - public abstract String getHttpVer(); - - /** - * save a response to some storage - * @author daniel - * - */ - public static class Saver { - - /** - * copies the body of the response accordingly to hfos to the destination - * @param res - * @param hfos OutputStream (raw bytes) or Writer (decode to characters) - * @param byteStream additional OutputStream where data is stored - * @throws IOException - * @throws UnsupportedEncodingException - */ - public static void writeContent(HttpResponse res, BufferedOutputStream hfos, BufferedOutputStream byteStream) throws IOException, UnsupportedEncodingException { - try { - InputStream data = res.getDataAsStream(); - if (byteStream == null) { - serverFileUtils.copyToStream(new BufferedInputStream(data), hfos); - } else { - serverFileUtils.copyToStreams(new BufferedInputStream(data), hfos, byteStream); - } - } finally { - res.closeStream(); - } - } - - public static void writeContent(HttpResponse res, BufferedWriter hfos, OutputStream byteStream) throws IOException, UnsupportedEncodingException { - try { - InputStream data = res.getDataAsStream(); - String charSet = httpHeader.getCharSet(res.getResponseHeader()); - if (byteStream == null) { - serverFileUtils.copyToWriter(new BufferedInputStream(data), hfos, charSet); - } else { - serverFileUtils.copyToWriters(new BufferedInputStream(data), hfos, new BufferedWriter(new OutputStreamWriter(byteStream, charSet)) , charSet); - } - } finally { - res.closeStream(); - } - } - - } - -} \ No newline at end of file diff --git a/source/de/anomic/http/JakartaCommonsHttpClient.java b/source/de/anomic/http/JakartaCommonsHttpClient.java index 9d5de0e22..9f6f55041 100644 --- a/source/de/anomic/http/JakartaCommonsHttpClient.java +++ b/source/de/anomic/http/JakartaCommonsHttpClient.java @@ -69,7 +69,7 @@ import de.anomic.yacy.yacyVersion; * @author danielr * */ -public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient { +public class JakartaCommonsHttpClient { /** * "the HttpClient instance and connection manager should be shared among all threads for maximum efficiency." * (Concurrent execution of HTTP methods, http://hc.apache.org/httpclient-3.x/performance.html) @@ -95,7 +95,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient { * set options for connection manager */ // conManager.getParams().setDefaultMaxConnectionsPerHost(4); // default 2 - conManager.getParams().setMaxTotalConnections(50); // default 20 + conManager.getParams().setMaxTotalConnections(200); // Proxy may need many connections conManager.getParams().setConnectionTimeout(60000); // set a default timeout conManager.getParams().setDefaultMaxConnectionsPerHost(20); // prevent DoS by mistake // TODO should this be configurable? @@ -184,7 +184,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient { * @return InputStream of content (body) * @throws IOException */ - public HttpResponse GET(final String uri) throws IOException { + public JakartaCommonsHttpResponse GET(final String uri) throws IOException { final HttpMethod get = new GetMethod(uri); return execute(get); } @@ -197,7 +197,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient { * @return Instance of response with the content. * @throws IOException */ - public HttpResponse HEAD(final String uri) throws IOException { + public JakartaCommonsHttpResponse HEAD(final String uri) throws IOException { assert uri != null : "precondition violated: uri != null"; final HttpMethod head = new HeadMethod(uri); return execute(head); @@ -213,7 +213,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient { * @return Instance of response with the content. * @throws IOException */ - public HttpResponse POST(final String uri, final InputStream ins) throws IOException { + public JakartaCommonsHttpResponse POST(final String uri, final InputStream ins) throws IOException { assert uri != null : "precondition violated: uri != null"; assert ins != null : "precondition violated: ins != null"; final PostMethod post = new PostMethod(uri); @@ -231,7 +231,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient { * @return Instance of response with the content. * @throws IOException */ - public HttpResponse POST(final String uri, final Map files) throws IOException { + public JakartaCommonsHttpResponse POST(final String uri, final Map files) throws IOException { assert uri != null : "precondition violated: uri != null"; final PostMethod post = new PostMethod(uri); @@ -278,7 +278,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient { * * @see de.anomic.http.HttpClient#CONNECT(java.lang.String, int, de.anomic.http.httpHeader) */ - public HttpResponse CONNECT(final String host, final int port) throws IOException { + public JakartaCommonsHttpResponse CONNECT(final String host, final int port) throws IOException { final HostConfiguration hostConfig = new HostConfiguration(); hostConfig.setHost(host, port); final HttpMethod connect = new ConnectMethod(hostConfig); @@ -369,7 +369,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient { * @throws IOException * @throws HttpException */ - private HttpResponse execute(final HttpMethod method) throws IOException, HttpException { + private JakartaCommonsHttpResponse execute(final HttpMethod method) throws IOException, HttpException { assert method != null : "precondition violated: method != null"; // set header for (final Header header : headers) { @@ -480,7 +480,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient { * @param args */ public static void main(final String[] args) { - HttpResponse resp = null; + JakartaCommonsHttpResponse resp = null; String url = args[0]; if (!(url.toUpperCase().startsWith("HTTP://"))) { url = "http://" + url; @@ -492,7 +492,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient { files.put("myfile.txt", "this is not a file ;)".getBytes()); files.put("anotherfile.raw", "this is not a binary file ;)".getBytes()); System.out.println("POST " + files.size() + " elements to " + url); - final de.anomic.http.HttpClient client = new JakartaCommonsHttpClient(1000, null, null); + final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(1000, null, null); resp = client.POST(url, files); System.out.println("----- Header: -----"); System.out.println(new String(resp.getResponseHeader().toString())); diff --git a/source/de/anomic/http/JakartaCommonsHttpResponse.java b/source/de/anomic/http/JakartaCommonsHttpResponse.java index 6f7d2c431..944463837 100644 --- a/source/de/anomic/http/JakartaCommonsHttpResponse.java +++ b/source/de/anomic/http/JakartaCommonsHttpResponse.java @@ -23,6 +23,7 @@ // You should have received a copy of the GNU General Public License // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + package de.anomic.http; import java.io.ByteArrayOutputStream; @@ -39,7 +40,7 @@ import org.apache.commons.httpclient.HttpMethod; * @author daniel * @since 21.03.2008 */ -public class JakartaCommonsHttpResponse implements HttpResponse { +public class JakartaCommonsHttpResponse { private final HttpMethod method; private String incomingAccountingName = null; diff --git a/source/de/anomic/http/httpdProxyHandler.java b/source/de/anomic/http/httpdProxyHandler.java index 6ad26e510..8e294620e 100644 --- a/source/de/anomic/http/httpdProxyHandler.java +++ b/source/de/anomic/http/httpdProxyHandler.java @@ -62,7 +62,7 @@ package de.anomic.http; -import java.io.BufferedOutputStream; +import java.io.BufferedInputStream; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.ByteArrayInputStream; @@ -75,6 +75,7 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.PrintWriter; +import java.io.UnsupportedEncodingException; import java.io.Writer; import java.net.BindException; import java.net.ConnectException; @@ -98,7 +99,6 @@ import java.util.zip.GZIPOutputStream; import de.anomic.htmlFilter.htmlFilterContentTransformer; import de.anomic.htmlFilter.htmlFilterTransformer; import de.anomic.htmlFilter.htmlFilterWriter; -import de.anomic.http.HttpResponse.Saver; import de.anomic.index.indexReferenceBlacklist; import de.anomic.plasma.plasmaHTCache; import de.anomic.plasma.plasmaParser; @@ -144,7 +144,7 @@ public final class httpdProxyHandler { * Do logging configuration for special proxy access log file */ static { - // Doing logger initialisation + // Doing logger initialization try { serverLog.logInfo("PROXY","Configuring proxy access logging ..."); @@ -395,7 +395,7 @@ public final class httpdProxyHandler { addXForwardedForHeader(conProp, requestHeader); - // decide wether to use a cache entry or connect to the network + // decide whether to use a cache entry or connect to the network File cacheFile = plasmaHTCache.getCachePath(url); httpHeader cachedResponseHeader = null; @@ -411,8 +411,8 @@ public final class httpdProxyHandler { // why are files unzipped upon arrival? why not zip all files in cache? // This follows from the following premises - // (a) no file shall be unzip-ed more than once to prevent unnessesary computing time - // (b) old cache entries shall be comparable with refill-entries to detect/distiguish case 3+4 + // (a) no file shall be unzip-ed more than once to prevent unnecessary computing time + // (b) old cache entries shall be comparable with refill-entries to detect/distinguish case 3+4 // (c) the indexing mechanism needs files unzip-ed, a schedule could do that later // case b and c contradicts, if we use a scheduler, because files in a stale cache would be unzipped // and the newly arrival would be zipped and would have to be unzipped upon load. But then the @@ -487,7 +487,7 @@ public final class httpdProxyHandler { httpChunkedOutputStream chunkedOut = null; Writer hfos = null; - HttpResponse res = null; + JakartaCommonsHttpResponse res = null; try { String host = conProp.getProperty(httpHeader.CONNECTION_PROP_HOST); @@ -519,7 +519,7 @@ public final class httpdProxyHandler { prepareRequestHeader(requestHeader, httpVer); // setup HTTP-client - final HttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader, null); + final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader, null); final String connectHost = hostPart(host, port, yAddress); final String getUrl = "http://"+ connectHost + remotePath; @@ -635,7 +635,7 @@ public final class httpdProxyHandler { { // ok, we don't write actually into a file, only to RAM, and schedule writing the file. ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); - Saver.writeContent(res, new BufferedWriter(hfos), byteStream); + writeContent(res, new BufferedWriter(hfos), byteStream); // cached bytes byte[] cacheArray; if(byteStream.size() > 0) { @@ -670,7 +670,7 @@ public final class httpdProxyHandler { // the file is too big to cache it in the ram, or the size is unknown // write to file right here. cacheFile.getParentFile().mkdirs(); - Saver.writeContent(res, new BufferedWriter(hfos), new FileOutputStream(cacheFile)); + writeContent(res, new BufferedWriter(hfos), new FileOutputStream(cacheFile)); if (hfos instanceof htmlFilterWriter) ((htmlFilterWriter) hfos).finalize(); theLogger.logFine("for write-file of " + url + ": contentLength = " + contentLength + ", sizeBeforeDelete = " + sizeBeforeDelete); plasmaHTCache.writeFileAnnouncement(cacheFile); @@ -700,7 +700,7 @@ public final class httpdProxyHandler { " StoreHTCache=" + storeHTCache + " SupportetContent=" + isSupportedContent); - Saver.writeContent(res, new BufferedWriter(hfos), null); + writeContent(res, new BufferedWriter(hfos)); if (hfos instanceof htmlFilterWriter) ((htmlFilterWriter) hfos).finalize(); if (sizeBeforeDelete == -1) { // no old file and no load. just data passing @@ -834,6 +834,28 @@ public final class httpdProxyHandler { return; } + public static void writeContent(JakartaCommonsHttpResponse res, BufferedWriter hfos) throws IOException, UnsupportedEncodingException { + try { + InputStream data = res.getDataAsStream(); + if (data == null) return; + String charSet = httpHeader.getCharSet(res.getResponseHeader()); + serverFileUtils.copyToWriter(new BufferedInputStream(data), hfos, charSet); + } finally { + res.closeStream(); + } + } + + public static void writeContent(JakartaCommonsHttpResponse res, BufferedWriter hfos, OutputStream byteStream) throws IOException, UnsupportedEncodingException { + assert byteStream != null; + try { + InputStream data = res.getDataAsStream(); + if (data == null) return; + String charSet = httpHeader.getCharSet(res.getResponseHeader()); + serverFileUtils.copyToWriters(new BufferedInputStream(data), hfos, new BufferedWriter(new OutputStreamWriter(byteStream, charSet)) , charSet); + } finally { + res.closeStream(); + } + } private static void removeHopByHopHeaders(httpHeader headers) { /* @@ -868,7 +890,7 @@ public final class httpdProxyHandler { public static void doHead(Properties conProp, httpHeader requestHeader, OutputStream respond) { - HttpResponse res = null; + JakartaCommonsHttpResponse res = null; yacyURL url = null; try { // remembering the starting time of the request @@ -935,7 +957,7 @@ public final class httpdProxyHandler { prepareRequestHeader(requestHeader, httpVer); // setup HTTP-client - HttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader, null); + JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader, null); // generate request-url final String connectHost = hostPart(host, port, yAddress); @@ -1057,7 +1079,7 @@ public final class httpdProxyHandler { serverFileUtils.copy(body, buffer, requestLength); body = new ByteArrayInputStream(buffer.toByteArray()); } - HttpResponse res = null; + JakartaCommonsHttpResponse res = null; try { // sending the request res = client.POST(getUrl, body); @@ -1104,8 +1126,17 @@ public final class httpdProxyHandler { responseHeader); // respondHeader(respond, res.status, res.responseHeader); - Saver.writeContent(res, (chunked != null) ? new BufferedOutputStream(chunked) : new BufferedOutputStream(respond), null); + // Saver.writeContent(res, (chunked != null) ? new BufferedOutputStream(chunked) : new BufferedOutputStream(respond)); + /* + // *** (†bernommen aus Saver-Klasse: warum ist dies hier die einzige Methode, die einen OutputStream statt einen Writer benutzt?) + try { + serverFileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), (chunked != null) ? new BufferedOutputStream(chunked) : new BufferedOutputStream(respond)); + } finally { + res.closeStream(); + } if (chunked != null) chunked.finish(); + */ + writeContent(res, new BufferedWriter(new OutputStreamWriter((chunked != null) ? chunked : respond))); respond.flush(); } finally { @@ -1208,9 +1239,9 @@ public final class httpdProxyHandler { (proxyConfig.useProxy()) && (proxyConfig.useProxy4SSL()) ) { - HttpClient remoteProxy = new JakartaCommonsHttpClient(timeout, requestHeader, proxyConfig); + JakartaCommonsHttpClient remoteProxy = new JakartaCommonsHttpClient(timeout, requestHeader, proxyConfig); - HttpResponse response = null; + JakartaCommonsHttpResponse response = null; try { response = remoteProxy.CONNECT(host, port); // outputs a logline to the serverlog with the current status diff --git a/source/de/anomic/index/indexRepositoryReference.java b/source/de/anomic/index/indexRepositoryReference.java index e76555a32..f2c1b9d33 100644 --- a/source/de/anomic/index/indexRepositoryReference.java +++ b/source/de/anomic/index/indexRepositoryReference.java @@ -36,9 +36,8 @@ import java.util.HashSet; import java.util.Iterator; import de.anomic.data.htmlTools; -import de.anomic.http.HttpClient; -import de.anomic.http.HttpResponse; import de.anomic.http.JakartaCommonsHttpClient; +import de.anomic.http.JakartaCommonsHttpResponse; import de.anomic.http.httpRemoteProxyConfig; import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroCache; @@ -257,9 +256,9 @@ public final class indexRepositoryReference { yacyURL newUrl = new yacyURL(newUrlStr, null); // doing a http head request to test if the url is correct - HttpClient client = new JakartaCommonsHttpClient(10000, null, null); + JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(10000, null, null); client.setProxy(proxyConfig); - HttpResponse res = null; + JakartaCommonsHttpResponse res = null; try { res = client.HEAD(newUrl.toString()); } finally { diff --git a/source/de/anomic/plasma/crawler/plasmaHTTPLoader.java b/source/de/anomic/plasma/crawler/plasmaHTTPLoader.java index 564dd82bc..8e06ef3f1 100644 --- a/source/de/anomic/plasma/crawler/plasmaHTTPLoader.java +++ b/source/de/anomic/plasma/crawler/plasmaHTTPLoader.java @@ -55,7 +55,6 @@ import java.net.UnknownHostException; import java.util.Date; import de.anomic.http.HttpClient; -import de.anomic.http.HttpResponse; import de.anomic.http.JakartaCommonsHttpClient; import de.anomic.http.JakartaCommonsHttpResponse; import de.anomic.http.httpHeader; @@ -177,9 +176,9 @@ public final class plasmaHTTPLoader { requestHeader.put(httpHeader.ACCEPT_ENCODING, this.acceptEncoding); // HTTP-Client - HttpClient client = new JakartaCommonsHttpClient(socketTimeout, requestHeader, null); + JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(socketTimeout, requestHeader, null); - HttpResponse res = null; + JakartaCommonsHttpResponse res = null; try { // send request res = client.GET(entry.url().toString()); diff --git a/source/de/anomic/plasma/plasmaParser.java b/source/de/anomic/plasma/plasmaParser.java index c31a85e20..fdd7de853 100644 --- a/source/de/anomic/plasma/plasmaParser.java +++ b/source/de/anomic/plasma/plasmaParser.java @@ -54,9 +54,8 @@ import de.anomic.htmlFilter.htmlFilterContentScraper; import de.anomic.htmlFilter.htmlFilterImageEntry; import de.anomic.htmlFilter.htmlFilterInputStream; import de.anomic.htmlFilter.htmlFilterWriter; -import de.anomic.http.HttpClient; -import de.anomic.http.HttpResponse; import de.anomic.http.JakartaCommonsHttpClient; +import de.anomic.http.JakartaCommonsHttpResponse; import de.anomic.plasma.parser.Parser; import de.anomic.plasma.parser.ParserException; import de.anomic.plasma.parser.ParserInfo; @@ -847,7 +846,7 @@ public final class plasmaParser { } String mode = args[0]; - HttpResponse res = null; + JakartaCommonsHttpResponse res = null; plasmaParserDocument document = null; try { // close InputStream when done if (mode.equalsIgnoreCase("-f")) { @@ -857,7 +856,7 @@ public final class plasmaParser { contentURL = new yacyURL(args[1], null); // downloading the document content - HttpClient client = new JakartaCommonsHttpClient(5000, null, null); + JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(5000, null, null); res = client.GET(args[1]); if (res.getStatusCode() != 200) { diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index 8b1ec69fb..5e6640771 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -53,8 +53,8 @@ import java.util.Map; import java.util.TreeMap; import de.anomic.http.HttpClient; -import de.anomic.http.HttpResponse; import de.anomic.http.JakartaCommonsHttpClient; +import de.anomic.http.JakartaCommonsHttpResponse; import de.anomic.http.httpHeader; import de.anomic.http.httpRemoteProxyConfig; import de.anomic.http.httpdProxyHandler; @@ -257,7 +257,7 @@ public final class yacyClient { * @throws IOException */ private static byte[] wput(final String url, String vhost, final Map post, final int timeout) throws IOException { - HttpClient client = new JakartaCommonsHttpClient(timeout, null, null); + JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, null, null); client.setProxy(proxyConfig()); // address vhost @@ -265,7 +265,7 @@ public final class yacyClient { header.add(httpHeader.HOST, vhost); client.setHeader(header); - HttpResponse res = null; + JakartaCommonsHttpResponse res = null; byte[] content = null; try { // send request/data diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index 74f6fb794..bc2b4768e 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -60,9 +60,8 @@ import java.util.Iterator; import java.util.Map; import java.util.TreeMap; -import de.anomic.http.HttpClient; -import de.anomic.http.HttpResponse; import de.anomic.http.JakartaCommonsHttpClient; +import de.anomic.http.JakartaCommonsHttpResponse; import de.anomic.http.httpHeader; import de.anomic.http.httpd; import de.anomic.kelondro.kelondroBase64Order; @@ -817,9 +816,9 @@ public final class yacySeedDB { reqHeader.put(httpHeader.CACHE_CONTROL, "no-cache"); // httpc uses HTTP/1.0 is this necessary? // init http-client - HttpClient client = new JakartaCommonsHttpClient(10000, reqHeader, null); + JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(10000, reqHeader, null); byte[] content = null; - HttpResponse res = null; + JakartaCommonsHttpResponse res = null; try { // send request res = client.GET(seedURL.toString()); diff --git a/source/de/anomic/yacy/yacyVersion.java b/source/de/anomic/yacy/yacyVersion.java index 99c344795..f712872ee 100644 --- a/source/de/anomic/yacy/yacyVersion.java +++ b/source/de/anomic/yacy/yacyVersion.java @@ -27,6 +27,7 @@ package de.anomic.yacy; +import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; @@ -41,12 +42,11 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import de.anomic.htmlFilter.htmlFilterContentScraper; -import de.anomic.http.HttpClient; -import de.anomic.http.HttpResponse; import de.anomic.http.JakartaCommonsHttpClient; -import de.anomic.http.HttpResponse.Saver; +import de.anomic.http.JakartaCommonsHttpResponse; import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverCore; +import de.anomic.server.serverFileUtils; import de.anomic.server.serverSystem; import de.anomic.server.logging.serverLog; @@ -330,11 +330,15 @@ public final class yacyVersion implements Comparator, Comparable