diff --git a/source/de/anomic/crawler/HTTPLoader.java b/source/de/anomic/crawler/HTTPLoader.java index 7c1a28b98..c7dd43592 100644 --- a/source/de/anomic/crawler/HTTPLoader.java +++ b/source/de/anomic/crawler/HTTPLoader.java @@ -151,7 +151,7 @@ public final class HTTPLoader { requestHeader.put(httpHeader.ACCEPT_ENCODING, sb.getConfig("crawler.http.acceptEncoding", DEFAULT_ENCODING)); // HTTP-Client - final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(socketTimeout, requestHeader, null); + final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(socketTimeout, requestHeader); JakartaCommonsHttpResponse res = null; try { diff --git a/source/de/anomic/crawler/RobotsTxt.java b/source/de/anomic/crawler/RobotsTxt.java index e3ce528ba..1e6c45d7f 100644 --- a/source/de/anomic/crawler/RobotsTxt.java +++ b/source/de/anomic/crawler/RobotsTxt.java @@ -514,7 +514,7 @@ public class RobotsTxt { // setup http-client //TODO: adding Traffic statistic for robots download? - final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(10000, reqHeaders, null); + final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(10000, reqHeaders); JakartaCommonsHttpResponse res = null; try { // sending the get request diff --git a/source/de/anomic/data/SitemapParser.java b/source/de/anomic/data/SitemapParser.java index 8c2ea27bd..a25202c5c 100644 --- a/source/de/anomic/data/SitemapParser.java +++ b/source/de/anomic/data/SitemapParser.java @@ -154,7 +154,7 @@ public class SitemapParser extends DefaultHandler { // download document final httpHeader header = new httpHeader(); header.put(httpHeader.USER_AGENT, HTTPLoader.crawlerUserAgent); - final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(5000, header, null); + final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(5000, header); JakartaCommonsHttpResponse res = null; try { res = client.GET(siteMapURL.toString()); diff --git a/source/de/anomic/http/HttpClient.java b/source/de/anomic/http/HttpClient.java index 9781b2221..e1b3de354 100644 --- a/source/de/anomic/http/HttpClient.java +++ b/source/de/anomic/http/HttpClient.java @@ -94,7 +94,7 @@ public abstract class HttpClient { public static byte[] wget(final String uri, final httpHeader header, final int timeout, final String vhost) { assert uri != null : "precondition violated: uri != null"; addHostHeader(header, vhost); - final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, header, null); + final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, header); // do the request try { @@ -141,7 +141,7 @@ public abstract class HttpClient { * @return null on error */ public static httpHeader whead(final String uri, final httpHeader header) { - final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(10000, header, null); + final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(10000, header); JakartaCommonsHttpResponse response = null; try { response = client.HEAD(uri); diff --git a/source/de/anomic/http/JakartaCommonsHttpClient.java b/source/de/anomic/http/JakartaCommonsHttpClient.java index 1b2005a76..a9b227695 100644 --- a/source/de/anomic/http/JakartaCommonsHttpClient.java +++ b/source/de/anomic/http/JakartaCommonsHttpClient.java @@ -32,6 +32,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.List; +import java.util.Map.Entry; import java.util.zip.GZIPOutputStream; import org.apache.commons.httpclient.ConnectMethod; @@ -139,14 +140,39 @@ public class JakartaCommonsHttpClient { private Header[] headers = new Header[0]; private httpRemoteProxyConfig proxyConfig = null; + private boolean useGlobalProxyConfig = true; private boolean followRedirects = true; private boolean ignoreCookies = false; + + /** + * creates a new JakartaCommonsHttpClient with given timeout using global remoteProxyConfig + * + * @param timeout in milliseconds + */ + public JakartaCommonsHttpClient(final int timeout) { + this(timeout, null); + } + /** - * constructs a new Client with given parameters + * creates a new JakartaCommonsHttpClient with given timeout and requestHeader using global remoteProxyConfig + * + * @param timeout in milliseconds + * @param header header options to send + */ + public JakartaCommonsHttpClient(final int timeout, final httpHeader header) { + super(); + setTimeout(timeout); + setHeader(header); + } + + /** + * creates a new JakartaCommonsHttpClient with given timeout and requestHeader using given remoteProxyConfig + * + * if proxyConfig is null, then no proxy is used * * @param timeout in milliseconds - * @param header + * @param header header options to send * @param proxyConfig */ public JakartaCommonsHttpClient(final int timeout, final httpHeader header, final httpRemoteProxyConfig proxyConfig) { @@ -161,9 +187,8 @@ public class JakartaCommonsHttpClient { * @see de.anomic.http.HttpClient#setProxy(de.anomic.http.httpRemoteProxyConfig) */ public void setProxy(final httpRemoteProxyConfig proxyConfig) { - if (proxyConfig != null && proxyConfig.useProxy()) { - this.proxyConfig = proxyConfig; - } + this.useGlobalProxyConfig = false; + this.proxyConfig = proxyConfig; } /* @@ -381,8 +406,8 @@ public class JakartaCommonsHttpClient { } else { headers = new Header[requestHeader.size()]; int i = 0; - for (final String name : requestHeader.keySet()) { - headers[i] = new Header(name, requestHeader.get(name)); + for (final Entry header : requestHeader.entrySet()) { + headers[i] = new Header(header.getKey(), header.getValue()); i++; } } @@ -520,13 +545,17 @@ public class JakartaCommonsHttpClient { /** * * @param hostname - * @return + * @return null if no proxy should be used */ private httpRemoteProxyConfig getProxyConfig(final String hostname) { final httpRemoteProxyConfig hostProxyConfig; - if (proxyConfig != null) { + if (!useGlobalProxyConfig) { // client specific - hostProxyConfig = proxyConfig.useForHost(hostname) ? proxyConfig : null; + if(proxyConfig == null) { + hostProxyConfig = null; + } else { + hostProxyConfig = proxyConfig.useForHost(hostname) ? proxyConfig : null; + } } else { // default settings hostProxyConfig = httpRemoteProxyConfig.getProxyConfigForHost(hostname); @@ -603,7 +632,7 @@ public class JakartaCommonsHttpClient { files.add(new FilePart("anotherfile.raw", new ByteArrayPartSource("anotherfile.raw", "this is not a binary file ;)".getBytes()))); System.out.println("POST " + files.size() + " elements to " + url); - final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(1000, null, null); + final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(1000); resp = client.POST(url, files); System.out.println("----- Header: -----"); System.out.println(resp.getResponseHeader().toString()); diff --git a/source/de/anomic/http/httpRemoteProxyConfig.java b/source/de/anomic/http/httpRemoteProxyConfig.java index 5dcd58366..60ea54b98 100644 --- a/source/de/anomic/http/httpRemoteProxyConfig.java +++ b/source/de/anomic/http/httpRemoteProxyConfig.java @@ -70,7 +70,7 @@ public final class httpRemoteProxyConfig { /** * @return the remoteProxyConfig */ - public static httpRemoteProxyConfig getRemoteProxyConfig() { + public static synchronized httpRemoteProxyConfig getRemoteProxyConfig() { return remoteProxyConfig; } diff --git a/source/de/anomic/http/httpd.java b/source/de/anomic/http/httpd.java index 6e5a26258..35b04e091 100644 --- a/source/de/anomic/http/httpd.java +++ b/source/de/anomic/http/httpd.java @@ -45,6 +45,7 @@ import java.util.Iterator; import java.util.List; import java.util.Properties; import java.util.StringTokenizer; +import java.util.Map.Entry; import org.apache.commons.fileupload.FileItem; import org.apache.commons.fileupload.FileItemFactory; @@ -89,7 +90,7 @@ public final class httpd implements serverHandler, Cloneable { public static final int ERRORCASE_MESSAGE = 4; public static final int ERRORCASE_FILE = 5; - public static httpdAlternativeDomainNames alternativeResolver = null; + private static httpdAlternativeDomainNames alternativeResolver = null; /** * A hashset containing extensions that indicate content that should not be transported @@ -349,8 +350,8 @@ public final class httpd implements serverHandler, Cloneable { // user = addressed peer name // pw = addressed peer hash (b64-hash) final String auth = (String) header.get(httpHeader.PROXY_AUTHORIZATION,"xxxxxx"); - if (alternativeResolver != null) { - final String test = kelondroBase64Order.standardCoder.encodeString(alternativeResolver.myName() + ":" + alternativeResolver.myID()); + if (getAlternativeResolver() != null) { + final String test = kelondroBase64Order.standardCoder.encodeString(getAlternativeResolver().myName() + ":" + getAlternativeResolver().myID()); if (!test.equals(auth.trim().substring(6))) return false; } @@ -1311,15 +1312,15 @@ public final class httpd implements serverHandler, Cloneable { } // if peer has public address it will be used - if (alternativeResolver != null) { - tp.put("extAddress", alternativeResolver.myIP() + ":" + alternativeResolver.myPort()); + if (getAlternativeResolver() != null) { + tp.put("extAddress", getAlternativeResolver().myIP() + ":" + getAlternativeResolver().myPort()); } // otherwise the local ip address will be used else { tp.put("extAddress", tp.get("host", "127.0.0.1") + ":" + tp.get("port", "8080")); } - tp.put("peerName", (alternativeResolver == null) ? "" : alternativeResolver.myName()); + tp.put("peerName", (getAlternativeResolver() == null) ? "" : getAlternativeResolver().myName()); tp.put("errorMessageType", errorcase); tp.put("httpStatus", Integer.toString(httpStatusCode) + " " + httpStatusText); tp.put("requestMethod", conProp.getProperty(httpHeader.CONNECTION_PROP_METHOD)); @@ -1330,10 +1331,8 @@ public final class httpd implements serverHandler, Cloneable { tp.put("errorMessageType_file", (detailedErrorMsgFile == null) ? "" : detailedErrorMsgFile.toString()); if ((detailedErrorMsgValues != null) && (detailedErrorMsgValues.size() > 0)) { // rewriting the value-names and add the proper name prefix: - final Iterator nameIter = detailedErrorMsgValues.keySet().iterator(); - while (nameIter.hasNext()) { - final String name = nameIter.next(); - tp.put("errorMessageType_" + name, detailedErrorMsgValues.get(name)); + for(Entry entry: detailedErrorMsgValues.entrySet()) { + tp.put("errorMessageType_" + entry.getKey(), entry.getValue()); } } break; @@ -1385,11 +1384,9 @@ public final class httpd implements serverHandler, Cloneable { serverFileUtils.copy(result, respond); } respond.flush(); - } catch (final Exception e) { - throw new IOException(e.getMessage()); } finally { - if (fis != null) try { fis.close(); } catch (final Exception e) {} - if (o != null) try { o.close(); } catch (final Exception e) {} + if (fis != null) try { fis.close(); } catch (final Exception e) { e.printStackTrace(); } + if (o != null) try { o.close(); } catch (final Exception e) { e.printStackTrace(); } } } @@ -1622,10 +1619,10 @@ public final class httpd implements serverHandler, Cloneable { if ((hostName == null) || (hostName.length() == 0)) return false; // getting ip address and port of this seed - if (alternativeResolver == null) return false; + if (getAlternativeResolver() == null) return false; // resolve ip addresses - final InetAddress seedInetAddress = serverDomains.dnsResolve(alternativeResolver.myIP()); + final InetAddress seedInetAddress = serverDomains.dnsResolve(getAlternativeResolver().myIP()); final InetAddress hostInetAddress = serverDomains.dnsResolve(hostName); if (seedInetAddress == null || hostInetAddress == null) return false; @@ -1682,7 +1679,7 @@ public final class httpd implements serverHandler, Cloneable { final Integer dstPort = (idx != -1) ? Integer.valueOf(hostName.substring(idx+1).trim()) : Integer.valueOf(80); // if the hostname endswith thisPeerName.yacy ... - final String alternativeAddress = (alternativeResolver == null) ? null : alternativeResolver.myAlternativeAddress(); + final String alternativeAddress = (getAlternativeResolver() == null) ? null : getAlternativeResolver().myAlternativeAddress(); if ((alternativeAddress != null) && (dstHost.endsWith(alternativeAddress))) { return true; /* @@ -1703,5 +1700,19 @@ public final class httpd implements serverHandler, Cloneable { } } catch (final Exception e) {} return false; + } + + /** + * @param alternativeResolver the alternativeResolver to set + */ + public static void setAlternativeResolver(httpdAlternativeDomainNames alternativeResolver) { + httpd.alternativeResolver = alternativeResolver; + } + + /** + * @return the alternativeResolver + */ + static httpdAlternativeDomainNames getAlternativeResolver() { + return alternativeResolver; } } diff --git a/source/de/anomic/http/httpdProxyHandler.java b/source/de/anomic/http/httpdProxyHandler.java index c1d4787bc..b0f7893b2 100644 --- a/source/de/anomic/http/httpdProxyHandler.java +++ b/source/de/anomic/http/httpdProxyHandler.java @@ -1068,7 +1068,7 @@ public final class httpdProxyHandler { * @return */ private static String resolveYacyDomains(final String host) { - return (httpd.alternativeResolver == null) ? null : httpd.alternativeResolver.resolve(host); + return (httpd.getAlternativeResolver() == null) ? null : httpd.getAlternativeResolver().resolve(host); } /** @@ -1121,7 +1121,7 @@ public final class httpdProxyHandler { */ private static JakartaCommonsHttpClient setupHttpClient(final httpHeader requestHeader, final String connectHost) { // setup HTTP-client - final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader, null); + final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader); client.setFollowRedirects(false); // cookies are handled by the user's browser client.setIgnoreCookies(true); @@ -1244,7 +1244,7 @@ public final class httpdProxyHandler { private static void setViaHeader(final httpHeader header, final String httpVer) { if (!switchboard.getConfigBool("proxy.sendViaHeader", true)) return; - final String myAddress = (httpd.alternativeResolver == null) ? null : httpd.alternativeResolver.myAlternativeAddress(); + final String myAddress = (httpd.getAlternativeResolver() == null) ? null : httpd.getAlternativeResolver().myAlternativeAddress(); if (myAddress != null) { // getting header set by other proxies in the chain diff --git a/source/de/anomic/index/indexRepositoryReference.java b/source/de/anomic/index/indexRepositoryReference.java index 71bf9e526..435382eb9 100644 --- a/source/de/anomic/index/indexRepositoryReference.java +++ b/source/de/anomic/index/indexRepositoryReference.java @@ -248,7 +248,7 @@ public final class indexRepositoryReference { final yacyURL newUrl = new yacyURL(newUrlStr, null); // doing a http head request to test if the url is correct - final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(10000, null, null); + final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(10000); client.setProxy(proxyConfig); JakartaCommonsHttpResponse res = null; try { diff --git a/source/de/anomic/plasma/plasmaParser.java b/source/de/anomic/plasma/plasmaParser.java index 56ad37f46..1775967c5 100644 --- a/source/de/anomic/plasma/plasmaParser.java +++ b/source/de/anomic/plasma/plasmaParser.java @@ -888,7 +888,7 @@ public final class plasmaParser { contentURL = new yacyURL(args[1], null); // downloading the document content - final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(5000, null, null); + final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(5000); res = client.GET(args[1]); if (res.getStatusCode() != 200) { diff --git a/source/de/anomic/server/serverSystem.java b/source/de/anomic/server/serverSystem.java index bcf03bb51..e408fc717 100644 --- a/source/de/anomic/server/serverSystem.java +++ b/source/de/anomic/server/serverSystem.java @@ -46,11 +46,11 @@ public final class serverSystem { public static final String blankTypeString = "____"; // system-identification statics - public static int systemOS = systemUnknown; - public static boolean isMacArchitecture = false; - public static boolean isUnixFS = false; - public static boolean canExecUnix = false; - public static boolean isWindows = false; + public static final int systemOS; + public static final boolean isMacArchitecture; + public static final boolean isUnixFS; + public static final boolean canExecUnix; + public static final boolean isWindows; // calculated system constants public static int maxPathLength = 65535; @@ -317,6 +317,7 @@ public final class serverSystem { } } } catch (final Exception e) { + System.err.println("ERROR "+ e.getClass() +" in openBrowser(): "+ e.getMessage()); System.out.println("please start your browser and open the following location: " + url); } } diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index 2fd9424e7..10d2e9b3b 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -270,7 +270,7 @@ public final class yacyClient { final httpHeader header = new httpHeader(); header.put(httpHeader.USER_AGENT, HTTPLoader.yacyUserAgent); header.put(httpHeader.HOST, vhost); - final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, header, null); + final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, header); client.setProxy(proxyConfig()); JakartaCommonsHttpResponse res = null; diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index c028a93d3..cf36782b3 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -127,7 +127,7 @@ public final class yacySeedDB implements httpdAlternativeDomainNames { lastSeedUpload_seedDBSize = sizeConnected(); // tell the httpdProxy how to find this table as address resolver - httpd.alternativeResolver = this; + httpd.setAlternativeResolver(this); } private synchronized void initMySeed() { @@ -850,7 +850,7 @@ public final class yacySeedDB implements httpdAlternativeDomainNames { reqHeader.put(httpHeader.USER_AGENT, HTTPLoader.yacyUserAgent); // init http-client - final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(10000, reqHeader, null); + final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(10000, reqHeader); byte[] content = null; JakartaCommonsHttpResponse res = null; try { diff --git a/source/de/anomic/yacy/yacyVersion.java b/source/de/anomic/yacy/yacyVersion.java index 067d11d83..4a278e804 100644 --- a/source/de/anomic/yacy/yacyVersion.java +++ b/source/de/anomic/yacy/yacyVersion.java @@ -340,7 +340,7 @@ public final class yacyVersion implements Comparator, Comparable