From c3342e117852267af631276c318a2aeebfe7eab3 Mon Sep 17 00:00:00 2001 From: orbiter Date: Wed, 9 Apr 2008 23:35:20 +0000 Subject: [PATCH] - removed class with only one static method - removed connection method with too long time-out git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4672 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/Network.java | 4 +- source/de/anomic/data/SitemapParser.java | 5 +- source/de/anomic/data/robotsParser.java | 4 +- source/de/anomic/http/HttpClient.java | 17 +---- source/de/anomic/http/HttpFactory.java | 62 ------------------- .../anomic/http/JakartaCommonsHttpClient.java | 13 +--- source/de/anomic/http/httpdProxyHandler.java | 15 ++--- .../index/indexRepositoryReference.java | 4 +- .../plasma/crawler/plasmaHTTPLoader.java | 4 +- source/de/anomic/plasma/plasmaParser.java | 5 +- source/de/anomic/yacy/yacyClient.java | 4 +- source/de/anomic/yacy/yacySeedDB.java | 4 +- source/de/anomic/yacy/yacyVersion.java | 4 +- source/yacy.java | 3 +- 14 files changed, 28 insertions(+), 120 deletions(-) delete mode 100644 source/de/anomic/http/HttpFactory.java diff --git a/htroot/Network.java b/htroot/Network.java index a1cb1a607..c51863038 100644 --- a/htroot/Network.java +++ b/htroot/Network.java @@ -55,7 +55,7 @@ import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; import de.anomic.http.HttpClient; -import de.anomic.http.HttpFactory; +import de.anomic.http.JakartaCommonsHttpClient; import de.anomic.http.httpHeader; import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverCodings; @@ -375,7 +375,7 @@ public class Network { prop.putHTML(STR_TABLE_LIST + conCount + "_fullname", seed.get(yacySeed.NAME, "deadlink")); userAgent = null; if (seed.hash.equals(yacyCore.seedDB.mySeed().hash)) { - final HttpClient httpClient = HttpFactory.newClient(); + final HttpClient httpClient = new JakartaCommonsHttpClient(10000, null, null); userAgent = httpClient.getUserAgent(); location = HttpClient.generateLocation(); } else { diff --git a/source/de/anomic/data/SitemapParser.java b/source/de/anomic/data/SitemapParser.java index 31a65eea1..8f511c64c 100644 --- a/source/de/anomic/data/SitemapParser.java +++ b/source/de/anomic/data/SitemapParser.java @@ -59,8 +59,8 @@ import org.xml.sax.SAXException; import org.xml.sax.helpers.DefaultHandler; import de.anomic.http.HttpClient; -import de.anomic.http.HttpFactory; import de.anomic.http.HttpResponse; +import de.anomic.http.JakartaCommonsHttpClient; import de.anomic.http.httpdByteCountInputStream; import de.anomic.index.indexURLReference; import de.anomic.plasma.plasmaCrawlProfile; @@ -178,8 +178,7 @@ public class SitemapParser extends DefaultHandler { */ public void parse() { // download document - HttpClient client = HttpFactory.newClient(); - client.setTimeout(5000); + HttpClient client = new JakartaCommonsHttpClient(5000, null, null); HttpResponse res = null; try { res = client.GET(siteMapURL.toString()); diff --git a/source/de/anomic/data/robotsParser.java b/source/de/anomic/data/robotsParser.java index ff6d84c96..645be0fce 100644 --- a/source/de/anomic/data/robotsParser.java +++ b/source/de/anomic/data/robotsParser.java @@ -55,8 +55,8 @@ import java.util.ArrayList; import java.util.Date; import de.anomic.http.HttpClient; -import de.anomic.http.HttpFactory; import de.anomic.http.HttpResponse; +import de.anomic.http.JakartaCommonsHttpClient; import de.anomic.http.httpHeader; import de.anomic.http.HttpResponse.Saver; import de.anomic.plasma.plasmaCrawlRobotsTxt; @@ -404,7 +404,7 @@ public final class robotsParser{ // setup http-client //TODO: adding Traffic statistic for robots download? - HttpClient client = HttpFactory.newClient(reqHeaders, 10000); + HttpClient client = new JakartaCommonsHttpClient(10000, reqHeaders, null); HttpResponse res = null; try { // sending the get request diff --git a/source/de/anomic/http/HttpClient.java b/source/de/anomic/http/HttpClient.java index 5703745c0..454645f7b 100644 --- a/source/de/anomic/http/HttpClient.java +++ b/source/de/anomic/http/HttpClient.java @@ -135,14 +135,6 @@ public abstract class HttpClient { */ public abstract String getUserAgent(); - /** - * Returns the given date in an HTTP-usable format. (according to RFC1123/RFC822) - * - * @param date The Date-Object to be converted. - * @return String with the date. - */ - public abstract String date2String(Date date); - /** * for easy access * @@ -151,7 +143,7 @@ public abstract class HttpClient { * @return */ public static String dateString(final Date date) { - return HttpFactory.newClient().date2String(date); + return JakartaCommonsHttpClient.date2String(date); } /** @@ -221,7 +213,7 @@ public abstract class HttpClient { */ public static byte[] wget(final String uri, httpHeader header, final String vhost, int timeout) { assert uri != null : "precondition violated: uri != null"; - final HttpClient client = HttpFactory.newClient(null, timeout); + final HttpClient client = new JakartaCommonsHttpClient(timeout, null, null); // set header header = addHostHeader(header, vhost); @@ -273,10 +265,7 @@ public abstract class HttpClient { * @return */ public static httpHeader whead(final String uri, final httpHeader header) { - final de.anomic.http.HttpClient client = HttpFactory.newClient(); - if (header != null) { - client.setHeader(header); - } + final de.anomic.http.HttpClient client = new JakartaCommonsHttpClient(10000, header, null); try { final HttpResponse response = client.HEAD(uri); return response.getResponseHeader(); diff --git a/source/de/anomic/http/HttpFactory.java b/source/de/anomic/http/HttpFactory.java deleted file mode 100644 index 05ebdeb44..000000000 --- a/source/de/anomic/http/HttpFactory.java +++ /dev/null @@ -1,62 +0,0 @@ -// HttpFactory.java -// (C) 2008 by Daniel Raap; danielr@users.berlios.de -// first published 03.04.2008 on http://yacy.net -// -// This is a part of YaCy, a peer-to-peer based web search engine -// -// $LastChangedDate: 2008-03-14 01:16:04 +0100 (Fr, 14 Mrz 2008) $ -// $LastChangedRevision: 4558 $ -// $LastChangedBy: orbiter $ -// -// LICENSE -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -package de.anomic.http; - -/** - * generates generic Objects used in this package - * - * @author danielr - */ -public class HttpFactory { - /** - * no instances - */ - private HttpFactory() { - } - - /** - * generates a new HttpClient object - * - * @return - */ - public static HttpClient newClient() { - return new JakartaCommonsHttpClient(); - } - - /** - * generates a new HttpClient object with given header and timeout - * - * @param header used for all HTTP-requests (unless another one is set) - * @param timeout in milliseconds - * @return - */ - public static HttpClient newClient(httpHeader header, int timeout) { - HttpClient client = new JakartaCommonsHttpClient(timeout, header, null); - return client; - } - -} diff --git a/source/de/anomic/http/JakartaCommonsHttpClient.java b/source/de/anomic/http/JakartaCommonsHttpClient.java index 284abb827..e9b7283b6 100644 --- a/source/de/anomic/http/JakartaCommonsHttpClient.java +++ b/source/de/anomic/http/JakartaCommonsHttpClient.java @@ -98,15 +98,6 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient { private Header[] headers = new Header[0]; private httpRemoteProxyConfig proxyConfig = null; - /** - * constructor - * - * with half-hour timeout - */ - public JakartaCommonsHttpClient() { - this(1800000, null, null); - } - /** * constructs a new Client with given parameters * @@ -432,7 +423,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient { * @param date The Date-Object to be converted. * @return String with the date. */ - public String date2String(Date date) { + public static String date2String(Date date) { if (date == null) return ""; @@ -474,7 +465,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient { files.put("myfile.txt", "this is not a file ;)".getBytes()); files.put("anotherfile.raw", "this is not a binary file ;)".getBytes()); System.out.println("POST " + files.size() + " elements to " + url); - final de.anomic.http.HttpClient client = HttpFactory.newClient(); + final de.anomic.http.HttpClient client = new JakartaCommonsHttpClient(1000, null, null); resp = client.POST(url, files); System.out.println("----- Header: -----"); System.out.println(new String(resp.getResponseHeader().toString())); diff --git a/source/de/anomic/http/httpdProxyHandler.java b/source/de/anomic/http/httpdProxyHandler.java index 8067ce274..bab659938 100644 --- a/source/de/anomic/http/httpdProxyHandler.java +++ b/source/de/anomic/http/httpdProxyHandler.java @@ -517,7 +517,7 @@ public final class httpdProxyHandler { prepareRequestHeader(requestHeader, httpVer); // setup HTTP-client - final HttpClient client = HttpFactory.newClient(requestHeader, timeout); + final HttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader, null); final String connectHost = hostPart(host, port, yAddress); final String getUrl = "http://"+ connectHost + remotePath; @@ -933,7 +933,7 @@ public final class httpdProxyHandler { prepareRequestHeader(requestHeader, httpVer); // setup HTTP-client - HttpClient client = HttpFactory.newClient(requestHeader, timeout); + HttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader, null); // generate request-url final String connectHost = hostPart(host, port, yAddress); @@ -1035,11 +1035,7 @@ public final class httpdProxyHandler { prepareRequestHeader(requestHeader, httpVer); // setup HTTP-client - JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(); - client.setTimeout(timeout); - - // set header for connection - client.setHeader(requestHeader); + JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader, null); final String connectHost = hostPart(host, port, yAddress); client.setProxy(getProxyConfig(connectHost)); @@ -1210,10 +1206,7 @@ public final class httpdProxyHandler { (proxyConfig.useProxy()) && (proxyConfig.useProxy4SSL()) ) { - HttpClient remoteProxy = HttpFactory.newClient(); - remoteProxy.setTimeout(timeout); - remoteProxy.setProxy(proxyConfig); - remoteProxy.setHeader(requestHeader); + HttpClient remoteProxy = new JakartaCommonsHttpClient(timeout, requestHeader, proxyConfig); HttpResponse response = null; try { diff --git a/source/de/anomic/index/indexRepositoryReference.java b/source/de/anomic/index/indexRepositoryReference.java index 41d8a4893..e76555a32 100644 --- a/source/de/anomic/index/indexRepositoryReference.java +++ b/source/de/anomic/index/indexRepositoryReference.java @@ -37,8 +37,8 @@ import java.util.Iterator; import de.anomic.data.htmlTools; import de.anomic.http.HttpClient; -import de.anomic.http.HttpFactory; import de.anomic.http.HttpResponse; +import de.anomic.http.JakartaCommonsHttpClient; import de.anomic.http.httpRemoteProxyConfig; import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroCache; @@ -257,7 +257,7 @@ public final class indexRepositoryReference { yacyURL newUrl = new yacyURL(newUrlStr, null); // doing a http head request to test if the url is correct - HttpClient client = HttpFactory.newClient(null, 30000); + HttpClient client = new JakartaCommonsHttpClient(10000, null, null); client.setProxy(proxyConfig); HttpResponse res = null; try { diff --git a/source/de/anomic/plasma/crawler/plasmaHTTPLoader.java b/source/de/anomic/plasma/crawler/plasmaHTTPLoader.java index 634a0ab8a..564dd82bc 100644 --- a/source/de/anomic/plasma/crawler/plasmaHTTPLoader.java +++ b/source/de/anomic/plasma/crawler/plasmaHTTPLoader.java @@ -55,8 +55,8 @@ import java.net.UnknownHostException; import java.util.Date; import de.anomic.http.HttpClient; -import de.anomic.http.HttpFactory; import de.anomic.http.HttpResponse; +import de.anomic.http.JakartaCommonsHttpClient; import de.anomic.http.JakartaCommonsHttpResponse; import de.anomic.http.httpHeader; import de.anomic.http.httpdBoundedSizeOutputStream; @@ -177,7 +177,7 @@ public final class plasmaHTTPLoader { requestHeader.put(httpHeader.ACCEPT_ENCODING, this.acceptEncoding); // HTTP-Client - HttpClient client = HttpFactory.newClient(requestHeader, socketTimeout); + HttpClient client = new JakartaCommonsHttpClient(socketTimeout, requestHeader, null); HttpResponse res = null; try { diff --git a/source/de/anomic/plasma/plasmaParser.java b/source/de/anomic/plasma/plasmaParser.java index ba7e47224..c31a85e20 100644 --- a/source/de/anomic/plasma/plasmaParser.java +++ b/source/de/anomic/plasma/plasmaParser.java @@ -55,8 +55,8 @@ import de.anomic.htmlFilter.htmlFilterImageEntry; import de.anomic.htmlFilter.htmlFilterInputStream; import de.anomic.htmlFilter.htmlFilterWriter; import de.anomic.http.HttpClient; -import de.anomic.http.HttpFactory; import de.anomic.http.HttpResponse; +import de.anomic.http.JakartaCommonsHttpClient; import de.anomic.plasma.parser.Parser; import de.anomic.plasma.parser.ParserException; import de.anomic.plasma.parser.ParserInfo; @@ -857,8 +857,7 @@ public final class plasmaParser { contentURL = new yacyURL(args[1], null); // downloading the document content - HttpClient client = HttpFactory.newClient(); - client.setTimeout(5000); + HttpClient client = new JakartaCommonsHttpClient(5000, null, null); res = client.GET(args[1]); if (res.getStatusCode() != 200) { diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index bd2e50508..8b1ec69fb 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -53,8 +53,8 @@ import java.util.Map; import java.util.TreeMap; import de.anomic.http.HttpClient; -import de.anomic.http.HttpFactory; import de.anomic.http.HttpResponse; +import de.anomic.http.JakartaCommonsHttpClient; import de.anomic.http.httpHeader; import de.anomic.http.httpRemoteProxyConfig; import de.anomic.http.httpdProxyHandler; @@ -257,7 +257,7 @@ public final class yacyClient { * @throws IOException */ private static byte[] wput(final String url, String vhost, final Map post, final int timeout) throws IOException { - HttpClient client = HttpFactory.newClient(null, timeout); + HttpClient client = new JakartaCommonsHttpClient(timeout, null, null); client.setProxy(proxyConfig()); // address vhost diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index 1b6b06919..74f6fb794 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -61,8 +61,8 @@ import java.util.Map; import java.util.TreeMap; import de.anomic.http.HttpClient; -import de.anomic.http.HttpFactory; import de.anomic.http.HttpResponse; +import de.anomic.http.JakartaCommonsHttpClient; import de.anomic.http.httpHeader; import de.anomic.http.httpd; import de.anomic.kelondro.kelondroBase64Order; @@ -817,7 +817,7 @@ public final class yacySeedDB { reqHeader.put(httpHeader.CACHE_CONTROL, "no-cache"); // httpc uses HTTP/1.0 is this necessary? // init http-client - HttpClient client = HttpFactory.newClient(reqHeader, 10000); + HttpClient client = new JakartaCommonsHttpClient(10000, reqHeader, null); byte[] content = null; HttpResponse res = null; try { diff --git a/source/de/anomic/yacy/yacyVersion.java b/source/de/anomic/yacy/yacyVersion.java index f834bd3ac..5c09fec44 100644 --- a/source/de/anomic/yacy/yacyVersion.java +++ b/source/de/anomic/yacy/yacyVersion.java @@ -41,8 +41,8 @@ import java.util.regex.Pattern; import de.anomic.htmlFilter.htmlFilterContentScraper; import de.anomic.http.HttpClient; -import de.anomic.http.HttpFactory; import de.anomic.http.HttpResponse; +import de.anomic.http.JakartaCommonsHttpClient; import de.anomic.http.HttpResponse.Saver; import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverCore; @@ -329,7 +329,7 @@ public final class yacyVersion implements Comparator, Comparable