From 64f32e8f009501a73e262922d17f0b018aac03c6 Mon Sep 17 00:00:00 2001 From: low012 Date: Fri, 11 Feb 2011 23:37:13 +0000 Subject: [PATCH] *) replaced all IPs in IP filters for proxy with the proper regular expression *) some cleanup git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7477 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- defaults/yacy.init | 12 +- defaults/yacy.network.allip.unit | 2 +- defaults/yacy.network.freeworld.unit | 2 +- defaults/yacy.network.intranet.unit | 2 +- defaults/yacy.network.webportal.unit | 2 +- source/de/anomic/http/server/HTTPDemon.java | 231 +++++++++----------- source/net/yacy/cora/protocol/Domains.java | 155 +++++++------ 7 files changed, 193 insertions(+), 213 deletions(-) diff --git a/defaults/yacy.init b/defaults/yacy.init index e90a41628..2c4d1a9b9 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -329,7 +329,7 @@ remoteProxyPort=4239 remoteProxyUser= remoteProxyPwd= -remoteProxyNoProxy=10\..*,127.*,172.(1[6-9]|2[0-9]|3[0-1])\..*,169.254.*,192.168.*,localhost,0:0:0:0:0:0:0:1 +remoteProxyNoProxy=10\..*,127\..*,172\.(1[6-9]|2[0-9]|3[0-1])\..*,169\.254\..*,192\.168\..*,localhost,0:0:0:0:0:0:0:1 # the proxy may filter the content of transferred web pages # the bluelist removes specific keywords from web pages @@ -346,9 +346,9 @@ proxyBlueList=yacy.blue # proxyClient: client-ip's that may connect the proxy for proxy service # if several ip's are allowed then they must be separated by a ',' -# any ip may contain the wildcard-sign '*' +# regular expressions may be used #proxyClient=192.168.0.4 -proxyClient=localhost,127.0.0.1,192.168.*,10\..*,0:0:0:0:0:0:0:1 +proxyClient=localhost,127\.0\.0\.1,192\.168\..*,10\..*,0:0:0:0:0:0:0:1 # YaCyHop: allow public usage of proxy for yacy-protocol # this enables usage of the internal http proxy for everyone, @@ -919,7 +919,7 @@ content.phpbb3.tableprefix = phpbb_ content.phpbb3.dbuser = notroot content.phpbb3.dbpw = joshua content.phpbb3.ppf = 1000 -content.phpbb3.dumpfile = +content.phpbb3.dumpfile = # segment assignment for index storage processes in YaCy: # each process can store its index result in it's own index segment @@ -930,7 +930,7 @@ segment.process.dhtout_tmp = default segment.process.proxy_tmp = default segment.process.localcrawling_tmp = default segment.process.remotecrawling_tmp = default -segment.process.default_tmp = default +segment.process.default_tmp = default # search engine teaser: an about box in search results # this is only shown, if the about.body is filled @@ -956,4 +956,4 @@ color_signgood = #009900 color_signother = #000099 color_searchheadline = #2200CC color_searchurl = #008000 -color_searchurlhover = #008000 +color_searchurlhover = #008000 diff --git a/defaults/yacy.network.allip.unit b/defaults/yacy.network.allip.unit index 521e78195..c2bd5008a 100644 --- a/defaults/yacy.network.allip.unit +++ b/defaults/yacy.network.allip.unit @@ -22,5 +22,5 @@ network.unit.update.location3 = https://latestyacy.f1ori.de/ network.unit.protocol.control = uncontrolled # white/blacklists -network.unit.access.whitelist = 10\..*,127.*,172.(1[6-9]|2[0-9]|3[0-1])\..*,169.254.*,192.168.*,localhost +network.unit.access.whitelist = 10\..*,127\..*,172\.(1[6-9]|2[0-9]|3[0-1])\..*,169\.254\..*,192\.168\..*,localhost network.unit.access.blacklist = \ No newline at end of file diff --git a/defaults/yacy.network.freeworld.unit b/defaults/yacy.network.freeworld.unit index f43219b91..120370c7d 100644 --- a/defaults/yacy.network.freeworld.unit +++ b/defaults/yacy.network.freeworld.unit @@ -34,5 +34,5 @@ network.unit.update.location3 = https://latestyacy.f1ori.de/ network.unit.protocol.control = uncontrolled # white/blacklists -network.unit.access.whitelist = 10\..*,127.*,172.(1[6-9]|2[0-9]|3[0-1])\..*,169.254.*,192.168.*,localhost +network.unit.access.whitelist = 10\..*,127\..*,172\.(1[6-9]|2[0-9]|3[0-1])\..*,169\.254\..*,192\.168\..*,localhost network.unit.access.blacklist = \ No newline at end of file diff --git a/defaults/yacy.network.intranet.unit b/defaults/yacy.network.intranet.unit index 5ec5a50aa..2af27b0a1 100644 --- a/defaults/yacy.network.intranet.unit +++ b/defaults/yacy.network.intranet.unit @@ -29,5 +29,5 @@ network.unit.update.location3 = https://latestyacy.f1ori.de/ network.unit.protocol.control = uncontrolled # white/blacklists -network.unit.access.whitelist = 10\..*,127.*,172.(1[6-9]|2[0-9]|3[0-1])\..*,169.254.*,192.168.*,localhost +network.unit.access.whitelist = 10\..*,127\..*,172\.(1[6-9]|2[0-9]|3[0-1])\..*,169\.254\..*,192\.168\..*,localhost network.unit.access.blacklist = \ No newline at end of file diff --git a/defaults/yacy.network.webportal.unit b/defaults/yacy.network.webportal.unit index 9762f3f26..859c5ed07 100644 --- a/defaults/yacy.network.webportal.unit +++ b/defaults/yacy.network.webportal.unit @@ -26,5 +26,5 @@ network.unit.update.location3 = https://latestyacy.f1ori.de/ network.unit.protocol.control = uncontrolled # white/blacklists -network.unit.access.whitelist = 10\..*,127.*,172.(1[6-9]|2[0-9]|3[0-1])\..*,169.254.*,192.168.*,localhost +network.unit.access.whitelist = 10\..*,127\..*,172\.(1[6-9]|2[0-9]|3[0-1])\..*,169\.254\..*,192\.168\..*,localhost network.unit.access.blacklist = \ No newline at end of file diff --git a/source/de/anomic/http/server/HTTPDemon.java b/source/de/anomic/http/server/HTTPDemon.java index 64433c2b5..c3231fc8f 100644 --- a/source/de/anomic/http/server/HTTPDemon.java +++ b/source/de/anomic/http/server/HTTPDemon.java @@ -102,8 +102,8 @@ public final class HTTPDemon implements serverHandler, Cloneable { public static final int ERRORCASE_MESSAGE = 4; public static final int ERRORCASE_FILE = 5; - private static File TMPDIR = new File(System.getProperty("java.io.tmpdir")); - private static FileItemFactory diskFileItemFactory = new DiskFileItemFactory(5 * 1024 * 1024, TMPDIR); + private static final File TMPDIR = new File(System.getProperty("java.io.tmpdir")); + private static final FileItemFactory DISK_FILE_ITEM_FACTORY = new DiskFileItemFactory(5 * 1024 * 1024, TMPDIR); private static AlternativeDomainNames alternativeResolver = null; @@ -124,8 +124,8 @@ public final class HTTPDemon implements serverHandler, Cloneable { public static final String hline = "-------------------------------------------------------------------------------"; public static final ConcurrentMap reverseMappingCache = new ConcurrentHashMap(); - private static volatile Switchboard switchboard = null; - private static String virtualHost = null; + private static volatile Switchboard switchboard; + private static String virtualHost; public static boolean keepAliveSupport = false; private static ConcurrentMap YaCyHopAccessRequester = new ConcurrentHashMap(); @@ -166,17 +166,17 @@ public final class HTTPDemon implements serverHandler, Cloneable { this.keepAliveRequestCount = 0; } - private static boolean allowProxy(Session session) { + private static boolean allowProxy(final Session session) { final String proxyClient = switchboard.getConfig("proxyClient", "*"); return (proxyClient.equals("*")) ? true : match(session.userAddress.getHostAddress(), proxyClient); } - private static boolean allowServer(Session session) { + private static boolean allowServer(final Session session) { final String serverClient = switchboard.getConfig("serverClient", "*"); return (serverClient.equals("*")) ? true : match(session.userAddress.getHostAddress(), serverClient); } - private static boolean allowYaCyHop(Session session) { + private static boolean allowYaCyHop(final Session session) { return switchboard.getConfigBool("YaCyHop", false); } @@ -221,9 +221,10 @@ public final class HTTPDemon implements serverHandler, Cloneable { /** * This function is used to determine if a persistent connection was requested by the client. * @param header the received http-headers + * @param prop * @return true if a persistent connection was requested or false otherwise */ - private boolean handlePersistentConnection(final RequestHeader header, Properties prop) { + private boolean handlePersistentConnection(final RequestHeader header, final Properties prop) { if (!keepAliveSupport) { prop.put(HeaderFramework.CONNECTION_PROP_PERSISTENT,"close"); @@ -315,45 +316,44 @@ public final class HTTPDemon implements serverHandler, Cloneable { private static long lastAccessDelta(final Map accessTable, final String domain) { final Long lastAccess = accessTable.get(domain); - if (lastAccess == null) return Long.MAX_VALUE; // never accessed - return System.currentTimeMillis() - lastAccess.longValue(); + return (lastAccess == null) ? Long.MAX_VALUE : System.currentTimeMillis() - lastAccess.longValue(); } - private boolean handleProxyAuthentication(final RequestHeader header, Properties prop, Session session) throws IOException { + private boolean handleProxyAuthentication(final RequestHeader header, final Properties prop, final Session session) throws IOException { // getting the http version that is used by the client final String httpVersion = prop.getProperty("HTTP", "HTTP/0.9"); // reading the authentication settings from switchboard if (!this.proxyAccounts_init) { this.use_proxyAccounts = switchboard.getConfigBool("use_proxyAccounts", false); - this.proxyAccounts_init = true; // is initialised - } + this.proxyAccounts_init = true; // is initialised + } if (this.use_proxyAccounts) { final String auth = header.get(RequestHeader.PROXY_AUTHORIZATION,"xxxxxx"); - UserDB.Entry entry=switchboard.userDB.ipAuth(session.userAddress.getHostAddress()); - if(entry == null){ - entry=switchboard.userDB.proxyAuth(auth, session.userAddress.getHostAddress()); - } - if(entry != null){ + UserDB.Entry entry = switchboard.userDB.ipAuth(session.userAddress.getHostAddress()); + if (entry == null) { + entry = switchboard.userDB.proxyAuth(auth, session.userAddress.getHostAddress()); + } + if (entry != null) { final int returncode=entry.surfRight(); - if(returncode==UserDB.Entry.PROXY_ALLOK){ - return true; - } - final serverObjects tp=new serverObjects(); - if(returncode==UserDB.Entry.PROXY_TIMELIMIT_REACHED){ + if (returncode == UserDB.Entry.PROXY_ALLOK) { + return true; + } + final serverObjects tp = new serverObjects(); + if (returncode == UserDB.Entry.PROXY_TIMELIMIT_REACHED) { tp.put("limit", "1");//time per day tp.put("limit_timelimit", entry.getTimeLimit()); sendRespondError(prop, session.out, 403, "Internet-Timelimit reached", new File("proxymsg/proxylimits.inc"), tp, null); - }else if(returncode==UserDB.Entry.PROXY_NORIGHT){ + } else if (returncode == UserDB.Entry.PROXY_NORIGHT){ tp.put("limit", "0"); sendRespondError(prop, session.out, 403, "Proxy use forbidden", new File("proxymsg/proxylimits.inc"), tp, null); } return false; - } + } // ask for authenticate session.out.write((httpVersion + " 407 Proxy Authentication Required" + serverCore.CRLF_STRING + - RequestHeader.PROXY_AUTHENTICATE + ": Basic realm=\"log-in\"" + serverCore.CRLF_STRING).getBytes()); + RequestHeader.PROXY_AUTHENTICATE + ": Basic realm=\"log-in\"" + serverCore.CRLF_STRING).getBytes()); session.out.write((HeaderFramework.CONTENT_LENGTH + ": 0\r\n").getBytes()); session.out.write("\r\n".getBytes()); session.out.flush(); @@ -363,12 +363,11 @@ public final class HTTPDemon implements serverHandler, Cloneable { return true; } - public Boolean EMPTY(final String arg, Session session) throws IOException { - if (++this.emptyRequestCount > 10) return serverCore.TERMINATE_CONNECTION; - return serverCore.RESUME_CONNECTION; + public Boolean EMPTY(final String arg, final Session session) throws IOException { + return (++this.emptyRequestCount > 10) ? serverCore.TERMINATE_CONNECTION : serverCore.RESUME_CONNECTION; } - public Boolean UNKNOWN(final String requestLine, Session session) throws IOException { + public Boolean UNKNOWN(final String requestLine, final Session session) throws IOException { Properties prop = parseRequestLine(HeaderFramework.METHOD_GET, requestLine, session); int pos; @@ -387,16 +386,16 @@ public final class HTTPDemon implements serverHandler, Cloneable { return serverCore.TERMINATE_CONNECTION; } - public Boolean GET(final String arg, Session session) { + public Boolean GET(final String arg, final Session session) { try { // parsing the http request line - Properties prop = parseRequestLine(HeaderFramework.METHOD_GET, arg, session); + final Properties prop = parseRequestLine(HeaderFramework.METHOD_GET, arg, session); // we now know the HTTP version. depending on that, we read the header final String httpVersion = prop.getProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_0_9); - final RequestHeader header = (httpVersion.equals(HeaderFramework.HTTP_VERSION_0_9)) - ? new RequestHeader(reverseMappingCache) - : readHeader(prop, session); + final RequestHeader header = (httpVersion.equals(HeaderFramework.HTTP_VERSION_0_9)) + ? new RequestHeader(reverseMappingCache) + : readHeader(prop, session); // handling transparent proxy support handleTransparentProxySupport(header, prop, virtualHost, HTTPDProxyHandler.isTransparentProxy); @@ -432,7 +431,7 @@ public final class HTTPDemon implements serverHandler, Cloneable { } } - private void logUnexpectedError(final Exception e, String address) { + private void logUnexpectedError(final Exception e, final String address) { if (e instanceof InterruptedException) { log.logInfo("Interruption detected"); } else { @@ -454,15 +453,15 @@ public final class HTTPDemon implements serverHandler, Cloneable { } } - public Boolean HEAD(final String arg, Session session) { + public Boolean HEAD(final String arg, final Session session) { try { - Properties prop = parseRequestLine(HeaderFramework.METHOD_HEAD, arg, session); + final Properties prop = parseRequestLine(HeaderFramework.METHOD_HEAD, arg, session); // we now know the HTTP version. depending on that, we read the header - RequestHeader header; final String httpVersion = prop.getProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_0_9); - if (httpVersion.equals(HeaderFramework.HTTP_VERSION_0_9)) header = new RequestHeader(reverseMappingCache); - else header = readHeader(prop,session); + final RequestHeader header = (httpVersion.equals(HeaderFramework.HTTP_VERSION_0_9)) + ? new RequestHeader(reverseMappingCache) + : readHeader(prop,session); // handle transparent proxy support handleTransparentProxySupport(header, prop, virtualHost, HTTPDProxyHandler.isTransparentProxy); @@ -500,15 +499,15 @@ public final class HTTPDemon implements serverHandler, Cloneable { } } - public Boolean POST(final String arg, Session session) { + public Boolean POST(final String arg, final Session session) { try { - Properties prop = parseRequestLine(HeaderFramework.METHOD_POST, arg, session); + final Properties prop = parseRequestLine(HeaderFramework.METHOD_POST, arg, session); // we now know the HTTP version. depending on that, we read the header - RequestHeader header; final String httpVersion = prop.getProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_0_9); - if (httpVersion.equals(HeaderFramework.HTTP_VERSION_0_9)) header = new RequestHeader(reverseMappingCache); - else header = readHeader(prop, session); + final RequestHeader header = (httpVersion.equals(HeaderFramework.HTTP_VERSION_0_9)) + ? new RequestHeader(reverseMappingCache) + : readHeader(prop, session); // handle transfer-coding final InputStream sessionIn; @@ -582,16 +581,18 @@ public final class HTTPDemon implements serverHandler, Cloneable { prop.setProperty(HeaderFramework.CONNECTION_PROP_HOST,(HTTPDemon.isThisHostName(dstHostSocket)?virtualHost:dstHostSocket)); } - public Boolean CONNECT(String arg, Session session) throws IOException { + public Boolean CONNECT(String arg, final Session session) throws IOException { // establish a ssh-tunneled http connection // this is to support https // parse HTTP version int pos = arg.indexOf(' '); - String httpVersion = "HTTP/1.0"; + final String httpVersion; if (pos >= 0) { httpVersion = arg.substring(pos + 1); arg = arg.substring(0, pos); + } else { + httpVersion = "HTTP/1.0"; } Properties prop = new Properties(); prop.setProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, httpVersion); @@ -641,10 +642,10 @@ public final class HTTPDemon implements serverHandler, Cloneable { return serverCore.TERMINATE_CONNECTION; } - private final Properties parseRequestLine(final String cmd, final String s, Session session) { + private final Properties parseRequestLine(final String cmd, final String s, final Session session) { // parsing the header - Properties p = parseRequestLine(cmd, s, virtualHost); + final Properties p = parseRequestLine(cmd, s, virtualHost); // track the request final String path = p.getProperty(HeaderFramework.CONNECTION_PROP_URL); @@ -788,7 +789,7 @@ public final class HTTPDemon implements serverHandler, Cloneable { // replace all other final CharArrayWriter b = new CharArrayWriter(s.length()); int end; - for (int i=0; i i) { if (s.charAt(i + 1) == '#') { // Ӓ symbols b.write(Integer.parseInt(s.substring(i + 2, end))); @@ -818,7 +819,7 @@ public final class HTTPDemon implements serverHandler, Cloneable { */ @SuppressWarnings("unchecked") public static Map parseMultipart(final RequestHeader header, final serverObjects args, final InputStream in) throws IOException { - //ByteArrayInputStream in = new ByteArrayInputStream(FileUtils.read(inx)); + final InputStream body = prepareBody(header, in); RequestContext request = new yacyContextRequest(header, body); @@ -834,20 +835,18 @@ public final class HTTPDemon implements serverHandler, Cloneable { } // parse data in memory - FileUpload upload = new FileUpload(diskFileItemFactory); - List items; - //long time = System.currentTimeMillis(); + final FileUpload upload = new FileUpload(DISK_FILE_ITEM_FACTORY); + final List items; + try { items = upload.parseRequest(request); } catch (FileUploadException e) { - //Log.logException(e); throw new IOException("FileUploadException " + e.getMessage()); } - //System.out.println("**** FileUploadBase.parseRequest time = " + (System.currentTimeMillis() - time)); - + // format information for further usage - final HashMap files = new HashMap(); - for (FileItem item : items) { + final Map files = new HashMap(); + for (final FileItem item : items) { if (item.isFormField()) { // simple text if (item.getContentType() == null || !item.getContentType().contains("charset")) { @@ -915,7 +914,7 @@ public final class HTTPDemon implements serverHandler, Cloneable { * @param header * @param in */ - public yacyContextRequest(Map requestHeader, InputStream in) { + public yacyContextRequest(final Map requestHeader, final InputStream in) { super(null, requestHeader); this.inStream = in; } @@ -932,38 +931,25 @@ public final class HTTPDemon implements serverHandler, Cloneable { } - /* - static int nextPos = -1; - private static byte[] readLine(final int start, final byte[] array) { - // read a string from an array; line ending is always CRLF - // but we are also fuzzy with that: may also be only CR or LF - // if no remaining CR, CRLF or LF can be found, return null - if (start > array.length) return null; - int pos = indexOf(start, array, serverCore.CRLF); nextPos = pos + 2; - if (pos < 0) {pos = indexOf(start, array, new byte[] {serverCore.CR}); nextPos = pos + 1;} - if (pos < 0) {pos = indexOf(start, array, new byte[] {serverCore.LF}); nextPos = pos + 1;} - if (pos < 0) {nextPos = start; return null;} - final byte[] result = new byte[pos - start]; - java.lang.System.arraycopy(array, start, result, 0, pos - start); - return result; - } - */ - public static int indexOf(final int start, final byte[] array, final byte[] pattern) { // return a position of a pattern in an array if (start > array.length - pattern.length) return -1; if (pattern.length == 0) return start; - for (int pos = start; pos <= array.length - pattern.length; pos++) - if ((array[pos] == pattern[0]) && (equals(array, pos, pattern, 0, pattern.length))) + for (int pos = start, lens = array.length - pattern.length; pos <= lens; pos++) { + if ((array[pos] == pattern[0]) && (equals(array, pos, pattern, 0, pattern.length))) { return pos; + } + } return -1; } public static boolean equals(final byte[] a, final int aoff, final byte[] b, final int boff, final int len) { - //System.out.println("equals: a = " + new String(a) + ", aoff = " + aoff + ", b = " + new String(b) + ", boff = " + boff + ", length = " + len); if ((aoff + len > a.length) || (boff + len > b.length)) return false; - for (int i = 0; i < len; i++) if (a[aoff + i] != b[boff + i]) return false; - //System.out.println("TRUE!"); + for (int i = 0; i < len; i++) { + if (a[aoff + i] != b[boff + i]) { + return false; + } + } return true; } @@ -980,7 +966,6 @@ public final class HTTPDemon implements serverHandler, Cloneable { respond.flush(); } - public static final void sendRespondError( final Properties conProp, final OutputStream respond, @@ -1059,11 +1044,13 @@ public final class HTTPDemon implements serverHandler, Cloneable { final String args = conProp.getProperty(HeaderFramework.CONNECTION_PROP_ARGS); final String method = conProp.getProperty(HeaderFramework.CONNECTION_PROP_METHOD); - int port = 80; + final int port; final int pos = host.indexOf(':'); if (pos != -1) { port = Integer.parseInt(host.substring(pos + 1)); host = host.substring(0, pos); + } else { + port = 80; } String urlString; @@ -1102,7 +1089,7 @@ public final class HTTPDemon implements serverHandler, Cloneable { tp.put("errorMessageType_file", (detailedErrorMsgFile == null) ? "" : detailedErrorMsgFile.toString()); if ((detailedErrorMsgValues != null) && !detailedErrorMsgValues.isEmpty()) { // rewriting the value-names and add the proper name prefix: - for (Entry entry: detailedErrorMsgValues.entrySet()) { + for (final Entry entry: detailedErrorMsgValues.entrySet()) { tp.put("errorMessageType_" + entry.getKey(), entry.getValue()); } } @@ -1141,8 +1128,9 @@ public final class HTTPDemon implements serverHandler, Cloneable { final byte[] result = o.toByteArray(); o.close(); o = null; - if(header == null) + if(header == null) { header = new ResponseHeader(); + } header.put(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_STATUS, Integer.toString(httpStatusCode)); header.put(HeaderFramework.DATE, systemDate); header.put(HeaderFramework.CONTENT_TYPE, "text/html"); @@ -1184,23 +1172,23 @@ public final class HTTPDemon implements serverHandler, Cloneable { } if (!reqMethod.equals(HeaderFramework.METHOD_HEAD)){ - if (!conProp.getProperty(HeaderFramework.CONNECTION_PROP_PERSISTENT,"close").equals("close")) { - if (transferEnc == null && contentLength < 0) { - throw new IllegalArgumentException("Message MUST contain a Content-Length or a non-identity transfer-coding header field."); - } + if (!conProp.getProperty(HeaderFramework.CONNECTION_PROP_PERSISTENT,"close").equals("close") && + transferEnc == null && contentLength < 0) { + throw new IllegalArgumentException("Message MUST contain a Content-Length or a non-identity transfer-coding header field."); } if (transferEnc != null && contentLength >= 0) { throw new IllegalArgumentException("Messages MUST NOT include both a Content-Length header field and a non-identity transfer-coding."); } } - if(headers==null) headers = new ResponseHeader(); + if (headers == null) { + headers = new ResponseHeader(); + } final Date now = new Date(System.currentTimeMillis()); headers.put(HeaderFramework.SERVER, "AnomicHTTPD (www.anomic.de)"); headers.put(HeaderFramework.DATE, HeaderFramework.formatRFC1123(now)); if (moddate.after(now)) { - //System.out.println("*** DEBUG: correcting moddate = " + moddate.toString() + " to now = " + now.toString()); moddate = now; } headers.put(HeaderFramework.LAST_MODIFIED, HeaderFramework.formatRFC1123(moddate)); @@ -1286,30 +1274,19 @@ public final class HTTPDemon implements serverHandler, Cloneable { //responseHeader.put(HeaderFramework.X_YACY_PREVIOUS_REQUEST_LINE,conProp.getProperty(HeaderFramework.CONNECTION_PROP_PREV_REQUESTLINE)); //read custom headers - /* - if (requestProperties != null) - { - httpHeader outgoingHeader=requestProperties.getOutgoingHeader(); - if (outgoingHeader!=null) - {*/ - final Iterator it = responseHeader.getAdditionalHeaderProperties().iterator(); - while(it.hasNext()) { - //Append user properties to the main String - //TODO: Should we check for user properites. What if they intersect properties that are already in header? - final ResponseHeader.Entry e = it.next(); - header.append(e.getKey()).append(": ").append(e.getValue()).append("\r\n"); - } - - /* - } - }*/ - + final Iterator it = responseHeader.getAdditionalHeaderProperties().iterator(); + while(it.hasNext()) { + //Append user properties to the main String + //TODO: Should we check for user properites. What if they intersect properties that are already in header? + final ResponseHeader.Entry e = it.next(); + header.append(e.getKey()).append(": ").append(e.getValue()).append("\r\n"); + } + // write header final Iterator i = responseHeader.keySet().iterator(); String key; char tag; int count; - //System.out.println("vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv"); while (i.hasNext()) { key = i.next(); tag = key.charAt(0); @@ -1318,7 +1295,6 @@ public final class HTTPDemon implements serverHandler, Cloneable { for (int j = 0; j < count; j++) { header.append(key).append(": ").append(responseHeader.getSingle(key, j)).append("\r\n"); } - //System.out.println("#" + key + ": " + value); } } @@ -1386,10 +1362,10 @@ public final class HTTPDemon implements serverHandler, Cloneable { // check if the destination port is equal to the port yacy is listening to dstPort.equals(Integer.valueOf(serverCore.getPortNr(switchboard.getConfig("port", "8090")))) && ( - // check if the destination host is our local IP address - Domains.isThisHostIP(dstHost) || - // check if the destination host is our seed ip address - isThisSeedIP(dstHost) + // check if the destination host is our local IP address + Domains.isThisHostIP(dstHost) || + // check if the destination host is our seed ip address + isThisSeedIP(dstHost) ) ) { return true; @@ -1401,7 +1377,7 @@ public final class HTTPDemon implements serverHandler, Cloneable { /** * @param alternativeResolver the alternativeResolver to set */ - public static void setAlternativeResolver(AlternativeDomainNames alternativeResolver) { + public static void setAlternativeResolver(final AlternativeDomainNames alternativeResolver) { HTTPDemon.alternativeResolver = alternativeResolver; } @@ -1462,7 +1438,7 @@ public final class HTTPDemon implements serverHandler, Cloneable { prop.setProperty(HeaderFramework.CONNECTION_PROP_REQUESTLINE, cmd + " " + args); // this parses a whole URL - if (args.length() == 0) { + if (args.isEmpty()) { prop.setProperty(HeaderFramework.CONNECTION_PROP_HOST, virtualHost); prop.setProperty(HeaderFramework.CONNECTION_PROP_PATH, "/"); prop.setProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_0_9); @@ -1472,7 +1448,7 @@ public final class HTTPDemon implements serverHandler, Cloneable { // store the version propery "HTTP" and cut the query at both ends int sep = args.lastIndexOf(' '); - if ((sep >= 0)&&(args.substring(sep + 1).toLowerCase().startsWith("http/"))) { + if ((sep >= 0) && (args.substring(sep + 1).toLowerCase().startsWith("http/"))) { // HTTP version is given prop.setProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, args.substring(sep + 1).trim()); args = args.substring(0, sep).trim(); // cut off HTTP version mark @@ -1496,19 +1472,22 @@ public final class HTTPDemon implements serverHandler, Cloneable { // properties of the query are stored with the prefix "&" // additionally, the values URL and ARGC are computed - String argsString = ""; + final String argsString; sep = args.indexOf('?'); if (sep >= 0) { // there are values attached to the query string argsString = args.substring(sep + 1); // cut head from tail of query args = args.substring(0, sep); + } else { + argsString = ""; } prop.setProperty(HeaderFramework.CONNECTION_PROP_URL, args); // store URL - //System.out.println("HTTPD: ARGS=" + argsString); - if (argsString.length() != 0) prop.setProperty(HeaderFramework.CONNECTION_PROP_ARGS, argsString); // store arguments in original form + if (!argsString.isEmpty()) { + prop.setProperty(HeaderFramework.CONNECTION_PROP_ARGS, argsString); + } // store arguments in original form // finally find host string - String path; + final String path; if (args.toUpperCase().startsWith("HTTP://")) { // a host was given. extract it and set path args = args.substring(7); @@ -1547,7 +1526,7 @@ public final class HTTPDemon implements serverHandler, Cloneable { prop.setProperty(HeaderFramework.CONNECTION_PROP_PATH, path); // find out file extension (we already stripped ?-parameters from args) - String ext = ""; // default when no file extension + final String ext; sep = path.lastIndexOf('.'); if (sep >= 0) { final int ancpos = path.indexOf("#", sep + 1); @@ -1558,6 +1537,8 @@ public final class HTTPDemon implements serverHandler, Cloneable { // ex: /foo/bar.php => php ext = path.substring(sep + 1).toLowerCase(); } + } else { + ext = ""; // default when no file extension } prop.setProperty(HeaderFramework.CONNECTION_PROP_EXT, ext); diff --git a/source/net/yacy/cora/protocol/Domains.java b/source/net/yacy/cora/protocol/Domains.java index 664ccf428..cc14b9258 100644 --- a/source/net/yacy/cora/protocol/Domains.java +++ b/source/net/yacy/cora/protocol/Domains.java @@ -25,6 +25,7 @@ import java.net.NetworkInterface; import java.net.SocketException; import java.net.UnknownHostException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Enumeration; @@ -42,18 +43,18 @@ import net.yacy.cora.storage.ConcurrentARC; public class Domains { private static final String PRESENT = ""; - private static final String localPatterns = "10\\..*,127.*,172.(1[6-9]|2[0-9]|3[0-1])\\..*,169.254.*,192.168.*,localhost"; - private static final int maxNameCacheHitSize = 20000; - private static final int maxNameCacheMissSize = 20000; - private static final int maxNameNoCachingListSize = 20000; - private static final int concurrencyLevel = Runtime.getRuntime().availableProcessors() + 1; + private static final String LOCAL_PATTERNS = "10\\..*,127\\..*,172\\.(1[6-9]|2[0-9]|3[0-1])\\..*,169\\.254\\..*,192\\.168\\..*,localhost"; + private static final int MAX_NAME_CACHE_HIT_SIZE = 20000; + private static final int MAX_NAME_CACHE_MISS_SIZE = 20000; + private static final int MAX_NAME_NO_CACHING_LIST_SIZE = 20000; + private static final int CONCURRENCY_LEVEL = Runtime.getRuntime().availableProcessors() + 1; // a dns cache - private static final ARC nameCacheHit = new ConcurrentARC(maxNameCacheHitSize, concurrencyLevel); - private static final ARC nameCacheMiss = new ConcurrentARC(maxNameCacheMissSize, concurrencyLevel); - private static final ARC nameCacheNoCachingList = new ConcurrentARC(maxNameNoCachingListSize, concurrencyLevel); + private static final ARC NAME_CACHE_HIT = new ConcurrentARC(MAX_NAME_CACHE_HIT_SIZE, CONCURRENCY_LEVEL); + private static final ARC NAME_CACHE_MISS = new ConcurrentARC(MAX_NAME_CACHE_MISS_SIZE, CONCURRENCY_LEVEL); + private static final ARC NAME_CACHE_NO_CACHING_LIST = new ConcurrentARC(MAX_NAME_NO_CACHING_LIST_SIZE, CONCURRENCY_LEVEL); public static List nameCacheNoCachingPatterns = Collections.synchronizedList(new LinkedList()); - public static final List localhostPatterns = makePatterns(localPatterns); + public static final List LOCALHOST_PATTERNS = makePatterns(LOCAL_PATTERNS); /** * ! ! ! A T T E N T I O N A T T E N T I O N A T T E N T I O N ! ! ! @@ -61,7 +62,7 @@ public class Domains { * Do not move a TLD to another group (if you do not exactly know what you * are doing)! Because it will change the hash of the url! */ - private static final String[] TLD_NorthAmericaOceania={ + private static final String[] TLD_NorthAmericaOceania = { // primary english-speaking countries // english-speaking countries from central america are also included // includes also dutch and french colonies in the caribbean sea @@ -383,10 +384,10 @@ public class Domains { String tld; //String name; final Integer ID = Integer.valueOf(id); - for (int i = 0; i < TLDList.length; i++) { - p = TLDList[i].indexOf('='); + for (final String TLDelement : TLDList) { + p = TLDelement.indexOf('='); if (p > 0) { - tld = TLDList[i].substring(0, p).toLowerCase(); + tld = TLDelement.substring(0, p).toLowerCase(); //name = TLDList[i].substring(p + 1); TLDID.put(tld, ID); //TLDName.put(tld, name); @@ -425,46 +426,46 @@ public class Domains { * @return String with the ip. null, if the host could not be resolved. */ public static InetAddress dnsResolveFromCache(String host) throws UnknownHostException { - if ((host == null) || (host.length() == 0)) return null; - host = host.toLowerCase().trim(); + if ((host == null) || host.isEmpty()) return null; + host = host.toLowerCase().trim(); // try to simply parse the address InetAddress ip = parseInetAddress(host); if (ip != null) return ip; // trying to resolve host by doing a name cache lookup - ip = nameCacheHit.get(host); + ip = NAME_CACHE_HIT.get(host); if (ip != null) return ip; - if (nameCacheMiss.containsKey(host)) return null; + if (NAME_CACHE_MISS.containsKey(host)) return null; throw new UnknownHostException("host not in cache"); } - public static void setNoCachingPatterns(String patternList) { + public static void setNoCachingPatterns(final String patternList) { nameCacheNoCachingPatterns = makePatterns(patternList); } - public static List makePatterns(String patternList) { - final String[] entries = patternList.split(","); + public static List makePatterns(final String patternList) { + final String[] entries = (patternList != null) ? patternList.split(",") : new String[0]; final List patterns = new ArrayList(entries.length); - for (int i = 0; i < entries.length; i++) { - patterns.add(Pattern.compile(entries[i].trim())); + for (final String entry : entries) { + patterns.add(Pattern.compile(entry.trim())); } return patterns; } - public static boolean matchesList(String obj, List patterns) { - for (Pattern nextPattern: patterns) { + public static boolean matchesList(final String obj, final List patterns) { + for (final Pattern nextPattern: patterns) { if (nextPattern.matcher(obj).matches()) return true; } return false; } public static String getHostName(final InetAddress i) { - Collection hosts = nameCacheHit.getKeys(i); - if (hosts.size() > 0) return hosts.iterator().next(); - String host = i.getHostName(); - nameCacheHit.put(host, i); + final Collection hosts = NAME_CACHE_HIT.getKeys(i); + if (!hosts.isEmpty()) return hosts.iterator().next(); + final String host = i.getHostName(); + NAME_CACHE_HIT.put(host, i); return host; /* // call i.getHostName() using concurrency to interrupt execution in case of a time-out @@ -478,16 +479,16 @@ public class Domains { public static InetAddress dnsResolve(String host) { if ((host == null) || (host.length() == 0)) return null; - host = host.toLowerCase().trim(); + host = host.toLowerCase().trim(); // try to simply parse the address InetAddress ip = parseInetAddress(host); if (ip != null) return ip; // try to resolve host by doing a name cache lookup - ip = nameCacheHit.get(host); + ip = NAME_CACHE_HIT.get(host); if (ip != null) return ip; - if (nameCacheMiss.containsKey(host)) return null; + if (NAME_CACHE_MISS.containsKey(host)) return null; // call dnsResolveNetBased(host) using concurrency to interrupt execution in case of a time-out try { @@ -495,12 +496,12 @@ public class Domains { ip = InetAddress.getByName(host); //TimeoutRequest.getByName(host, 1000); // this makes the DNS request to backbone if ((ip == null) || (ip.isLoopbackAddress()) || - (nameCacheNoCachingList.containsKey(host)) + (NAME_CACHE_NO_CACHING_LIST.containsKey(host)) ) { doCaching = false; } else { if (matchesList(host, nameCacheNoCachingPatterns)) { - nameCacheNoCachingList.put(host, PRESENT); + NAME_CACHE_NO_CACHING_LIST.put(host, PRESENT); doCaching = false; } } @@ -508,7 +509,7 @@ public class Domains { if (doCaching && ip != null) { // add new entries - nameCacheHit.put(host, ip); + NAME_CACHE_HIT.put(host, ip); } return ip; } catch (final UnknownHostException e) { @@ -516,14 +517,13 @@ public class Domains { flushMissNameCache(); // add new entries - nameCacheMiss.put(host, PRESENT); + NAME_CACHE_MISS.put(host, PRESENT); } return null; } private static final InetAddress parseInetAddress(final String ip) { - if (ip == null) return null; - if (ip.length() < 8) return null; + if (ip == null || ip.length() < 8) return null; final String[] ips = ip.split("\\."); if (ips.length != 4) return null; final byte[] ipb = new byte[4]; @@ -548,11 +548,11 @@ public class Domains { * @return int The number of entries in the nameCacheHit map */ public static int nameCacheHitSize() { - return nameCacheHit.size(); + return NAME_CACHE_HIT.size(); } public static int nameCacheMissSize() { - return nameCacheMiss.size(); + return NAME_CACHE_MISS.size(); } /** @@ -561,14 +561,14 @@ public class Domains { * @return int The number of entries in the nameCacheNoCachingList list */ public static int nameCacheNoCachingListSize() { - return nameCacheNoCachingList.size(); + return NAME_CACHE_NO_CACHING_LIST.size(); } /** * Removes old entries from the dns miss cache */ public static void flushMissNameCache() { - if (nameCacheMiss.size() > maxNameCacheMissSize) nameCacheMiss.clear(); + if (NAME_CACHE_MISS.size() > MAX_NAME_CACHE_MISS_SIZE) NAME_CACHE_MISS.clear(); } private static String localHostName = "127.0.0.1"; @@ -580,8 +580,8 @@ public class Domains { if (localHostAddress != null) localHostAddresses.add(localHostAddress); } catch (UnknownHostException e) {} try { - InetAddress[] moreAddresses = InetAddress.getAllByName(localHostName); - if (moreAddresses != null) for (InetAddress a: moreAddresses) localHostAddresses.add(a); + final InetAddress[] moreAddresses = InetAddress.getAllByName(localHostName); + if (moreAddresses != null) localHostAddresses.addAll(Arrays.asList(moreAddresses)); } catch (UnknownHostException e) {} // to get the local host name, a dns lookup is necessary. @@ -589,15 +589,16 @@ public class Domains { // therefore we start the host name lookup as concurrent thread // meanwhile the host name is "127.0.0.1" which is not completely wrong new Thread() { + @Override public void run() { // try to get local addresses from interfaces try { - Enumeration nis = NetworkInterface.getNetworkInterfaces(); + final Enumeration nis = NetworkInterface.getNetworkInterfaces(); while (nis.hasMoreElements()) { - NetworkInterface ni = nis.nextElement(); - Enumeration addrs = ni.getInetAddresses(); + final NetworkInterface ni = nis.nextElement(); + final Enumeration addrs = ni.getInetAddresses(); while (addrs.hasMoreElements()) { - InetAddress addr = addrs.nextElement(); + final InetAddress addr = addrs.nextElement(); if (addr != null) localHostAddresses.add(addr); } } @@ -612,14 +613,14 @@ public class Domains { // after the host name was resolved, we try to look up more local addresses // using the host name: try { - InetAddress[] moreAddresses = InetAddress.getAllByName(localHostName); - if (moreAddresses != null) for (InetAddress a: moreAddresses) localHostAddresses.add(a); + final InetAddress[] moreAddresses = InetAddress.getAllByName(localHostName); + if (moreAddresses != null) localHostAddresses.addAll(Arrays.asList(moreAddresses)); } catch (UnknownHostException e) { } // fill a cache of local host names - for (InetAddress a: localHostAddresses) { - String hostname = getHostName(a); + for (final InetAddress a: localHostAddresses) { + final String hostname = getHostName(a); if (hostname != null) { localHostNames.add(hostname); localHostNames.add(a.getHostAddress()); @@ -632,7 +633,7 @@ public class Domains { public static InetAddress myPublicLocalIP() { // list all addresses // for (int i = 0; i < localHostAddresses.length; i++) System.out.println("IP: " + localHostAddresses[i].getHostAddress()); // DEBUG - if (localHostAddresses.size() == 0) { + if (localHostAddresses.isEmpty()) { return null; } if (localHostAddresses.size() == 1) { @@ -641,7 +642,7 @@ public class Domains { } // we have more addresses, find an address that is not local int b0, b1; - for (InetAddress a: localHostAddresses) { + for (final InetAddress a: localHostAddresses) { b0 = 0Xff & a.getAddress()[0]; b1 = 0Xff & a.getAddress()[1]; if (b0 != 10 && // class A reserved @@ -655,26 +656,26 @@ public class Domains { // return that one that is returned with InetAddress.getLocalHost() // if appropriate try { - InetAddress localHostAddress = InetAddress.getLocalHost(); + final InetAddress localHostAddress = InetAddress.getLocalHost(); if (localHostAddress != null && (0Xff & localHostAddress.getAddress()[0]) != 127 && localHostAddress.getHostAddress().indexOf(":") < 0) return localHostAddress; } catch (UnknownHostException e) { } // we filter out the loopback address 127.0.0.1 and all addresses without a name - for (InetAddress a: localHostAddresses) { + for (final InetAddress a: localHostAddresses) { if ((0Xff & a.getAddress()[0]) != 127 && a.getHostAddress().indexOf(":") < 0 && a.getHostName() != null && - a.getHostName().length() > 0) return a; + !a.getHostName().isEmpty()) return a; } // if no address has a name, then take any other than the loopback - for (InetAddress a: localHostAddresses) { + for (final InetAddress a: localHostAddresses) { if ((0Xff & a.getAddress()[0]) != 127 && a.getHostAddress().indexOf(":") < 0) return a; } // if all fails, give back whatever we have - for (InetAddress a: localHostAddresses) { + for (final InetAddress a: localHostAddresses) { if (a.getHostAddress().indexOf(":") < 0) return a; } // finally, just get any @@ -688,11 +689,11 @@ public class Domains { public static Set myIntranetIPs() { // list all local addresses if (localHostAddresses.size() < 1) try {Thread.sleep(1000);} catch (InterruptedException e) {} - Set list = new HashSet(); - if (localHostAddresses.size() == 0) return list; // give up - for (InetAddress a: localHostAddresses) { - if ((0Xff & a.getAddress()[0]) == 127) continue; - if (!matchesList(a.getHostAddress(), localhostPatterns)) continue; + final Set list = new HashSet(); + if (localHostAddresses.isEmpty()) return list; // give up + for (final InetAddress a: localHostAddresses) { + if (((0Xff & a.getAddress()[0]) == 127) || + (!matchesList(a.getHostAddress(), LOCALHOST_PATTERNS))) continue; list.add(a); } return list; @@ -706,7 +707,7 @@ public class Domains { final InetAddress clientAddress = Domains.dnsResolve(hostName); if (clientAddress == null) return false; if (clientAddress.isAnyLocalAddress() || clientAddress.isLoopbackAddress()) return true; - for (InetAddress a: localHostAddresses) { + for (final InetAddress a: localHostAddresses) { if (a.equals(clientAddress)) { isThisHostIP = true; break; @@ -723,7 +724,7 @@ public class Domains { try { if (clientAddress.isAnyLocalAddress() || clientAddress.isLoopbackAddress()) return true; - for (InetAddress a: localHostAddresses) { + for (final InetAddress a: localHostAddresses) { if (a.equals(clientAddress)) { isThisHostIP = true; break; @@ -734,20 +735,18 @@ public class Domains { } public static int getDomainID(final String host) { - if (host == null || host.length() == 0) return TLD_Local_ID; - if (isLocal(host)) return TLD_Local_ID; + if (host == null || host.isEmpty() || isLocal(host)) return TLD_Local_ID; final int p = host.lastIndexOf('.'); - String tld = (p > 0) ? host.substring(p + 1) : ""; + final String tld = (p > 0) ? host.substring(p + 1) : ""; final Integer i = TLDID.get(tld); - if (i == null) return TLD_Generic_ID; - return i.intValue(); + return (i == null) ? TLD_Generic_ID : i.intValue(); } public static boolean isLocalhost(final String host) { - if (host.equals("127.0.0.1")) return true; - if (host.equals("localhost")) return true; - if (host.startsWith("0:0:0:0:0:0:0:1")) return true; - return false; + return ("127.0.0.1".equals(host) || + "localhost".equals(host) || + host.startsWith("0:0:0:0:0:0:0:1") + ); } public static boolean isLocal(final String host) { @@ -759,7 +758,7 @@ public class Domains { // FIXME IPv4 only // check local ip addresses - if (matchesList(host, localhostPatterns)) return true; + if (matchesList(host, LOCALHOST_PATTERNS)) return true; if (host.startsWith("0:0:0:0:0:0:0:1")) return true; // check if there are other local IP addresses that are not in @@ -775,7 +774,7 @@ public class Domains { // check dns lookup: may be a local address even if the domain name looks global if (!recursive) return false; - InetAddress a = dnsResolve(host); + final InetAddress a = dnsResolve(host); /* if (a == null) { // unknown if this is a local address. Could also be a timeout. @@ -786,7 +785,7 @@ public class Domains { return a == null || a.isAnyLocalAddress() || a.isLinkLocalAddress() || a.isLoopbackAddress() || a.isSiteLocalAddress() || isLocal(a.getHostAddress(), false); } - public static void main(String[] args) { + public static void main(final String[] args) { /* try { Enumeration nis = NetworkInterface.getNetworkInterfaces(); @@ -804,7 +803,7 @@ public class Domains { */ try { Thread.sleep(1000);} catch (InterruptedException e) {} // get time for class init System.out.println("myPublicLocalIP: " + myPublicLocalIP()); - for (InetAddress a : myIntranetIPs()) { + for (final InetAddress a : myIntranetIPs()) { System.out.println("Intranet IP: " + a); } }