boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

*) replaced all IPs in IP filters for proxy with the proper regular expression

Browse Source 
*) some cleanup

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7477 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
low012 14 years ago
parent 9f6e3f9793
commit 64f32e8f00
7 changed files with 193 additions and 213 deletions
Show Stats Download Patch File Download Diff File

12
defaults/yacy.init
Unescape View File

@ -329,7 +329,7 @@ remoteProxyPort=4239
remoteProxyUser= remoteProxyUser=
remoteProxyPwd= remoteProxyPwd=
remoteProxyNoProxy=10\..*,127.*,172.(1[6-9]|2[0-9]|3[0-1])\..*,169.254.*,192.168.*,localhost,0:0:0:0:0:0:0:1 remoteProxyNoProxy=10\..*,127\..*,172\.(1[6-9]|2[0-9]|3[0-1])\..*,169\.254\..*,192\.168\..*,localhost,0:0:0:0:0:0:0:1
# the proxy may filter the content of transferred web pages # the proxy may filter the content of transferred web pages
# the bluelist removes specific keywords from web pages # the bluelist removes specific keywords from web pages
@ -346,9 +346,9 @@ proxyBlueList=yacy.blue
# proxyClient: client-ip's that may connect the proxy for proxy service # proxyClient: client-ip's that may connect the proxy for proxy service
# if several ip's are allowed then they must be separated by a ',' # if several ip's are allowed then they must be separated by a ','
# any ip may contain the wildcard-sign '*' # regular expressions may be used
#proxyClient=192.168.0.4 #proxyClient=192.168.0.4
proxyClient=localhost,127.0.0.1,192.168.*,10\..*,0:0:0:0:0:0:0:1 proxyClient=localhost,127\.0\.0\.1,192\.168\..*,10\..*,0:0:0:0:0:0:0:1
# YaCyHop: allow public usage of proxy for yacy-protocol # YaCyHop: allow public usage of proxy for yacy-protocol
# this enables usage of the internal http proxy for everyone, # this enables usage of the internal http proxy for everyone,
@ -919,7 +919,7 @@ content.phpbb3.tableprefix = phpbb_
content.phpbb3.dbuser = notroot content.phpbb3.dbuser = notroot
content.phpbb3.dbpw = joshua content.phpbb3.dbpw = joshua
content.phpbb3.ppf = 1000 content.phpbb3.ppf = 1000
content.phpbb3.dumpfile = content.phpbb3.dumpfile =
# segment assignment for index storage processes in YaCy: # segment assignment for index storage processes in YaCy:
# each process can store its index result in it's own index segment # each process can store its index result in it's own index segment
@ -930,7 +930,7 @@ segment.process.dhtout_tmp = default
segment.process.proxy_tmp = default segment.process.proxy_tmp = default
segment.process.localcrawling_tmp = default segment.process.localcrawling_tmp = default
segment.process.remotecrawling_tmp = default segment.process.remotecrawling_tmp = default
segment.process.default_tmp = default segment.process.default_tmp = default
# search engine teaser: an about box in search results # search engine teaser: an about box in search results
# this is only shown, if the about.body is filled # this is only shown, if the about.body is filled
@ -956,4 +956,4 @@ color_signgood = #009900
color_signother = #000099 color_signother = #000099
color_searchheadline = #2200CC color_searchheadline = #2200CC
color_searchurl = #008000 color_searchurl = #008000
color_searchurlhover = #008000 color_searchurlhover = #008000

2
defaults/yacy.network.allip.unit
Unescape View File

@ -22,5 +22,5 @@ network.unit.update.location3 = https://latestyacy.f1ori.de/
network.unit.protocol.control = uncontrolled network.unit.protocol.control = uncontrolled
# white/blacklists # white/blacklists
network.unit.access.whitelist = 10\..*,127.*,172.(1[6-9]|2[0-9]|3[0-1])\..*,169.254.*,192.168.*,localhost network.unit.access.whitelist = 10\..*,127\..*,172\.(1[6-9]|2[0-9]|3[0-1])\..*,169\.254\..*,192\.168\..*,localhost
network.unit.access.blacklist = network.unit.access.blacklist =

2
defaults/yacy.network.freeworld.unit
Unescape View File

@ -34,5 +34,5 @@ network.unit.update.location3 = https://latestyacy.f1ori.de/
network.unit.protocol.control = uncontrolled network.unit.protocol.control = uncontrolled
# white/blacklists # white/blacklists
network.unit.access.whitelist = 10\..*,127.*,172.(1[6-9]|2[0-9]|3[0-1])\..*,169.254.*,192.168.*,localhost network.unit.access.whitelist = 10\..*,127\..*,172\.(1[6-9]|2[0-9]|3[0-1])\..*,169\.254\..*,192\.168\..*,localhost
network.unit.access.blacklist = network.unit.access.blacklist =

2
defaults/yacy.network.intranet.unit
Unescape View File

@ -29,5 +29,5 @@ network.unit.update.location3 = https://latestyacy.f1ori.de/
network.unit.protocol.control = uncontrolled network.unit.protocol.control = uncontrolled
# white/blacklists # white/blacklists
network.unit.access.whitelist = 10\..*,127.*,172.(1[6-9]|2[0-9]|3[0-1])\..*,169.254.*,192.168.*,localhost network.unit.access.whitelist = 10\..*,127\..*,172\.(1[6-9]|2[0-9]|3[0-1])\..*,169\.254\..*,192\.168\..*,localhost
network.unit.access.blacklist = network.unit.access.blacklist =

2
defaults/yacy.network.webportal.unit
Unescape View File

@ -26,5 +26,5 @@ network.unit.update.location3 = https://latestyacy.f1ori.de/
network.unit.protocol.control = uncontrolled network.unit.protocol.control = uncontrolled
# white/blacklists # white/blacklists
network.unit.access.whitelist = 10\..*,127.*,172.(1[6-9]|2[0-9]|3[0-1])\..*,169.254.*,192.168.*,localhost network.unit.access.whitelist = 10\..*,127\..*,172\.(1[6-9]|2[0-9]|3[0-1])\..*,169\.254\..*,192\.168\..*,localhost
network.unit.access.blacklist = network.unit.access.blacklist =

231
source/de/anomic/http/server/HTTPDemon.java
Unescape View File

@ -102,8 +102,8 @@ public final class HTTPDemon implements serverHandler, Cloneable {
public static final int ERRORCASE_MESSAGE = 4; public static final int ERRORCASE_MESSAGE = 4;
public static final int ERRORCASE_FILE = 5; public static final int ERRORCASE_FILE = 5;
private static File TMPDIR = new File(System.getProperty("java.io.tmpdir")); private static final File TMPDIR = new File(System.getProperty("java.io.tmpdir"));
private static FileItemFactory diskFileItemFactory = new DiskFileItemFactory(5 * 1024 * 1024, TMPDIR); private static final FileItemFactory DISK_FILE_ITEM_FACTORY = new DiskFileItemFactory(5 * 1024 * 1024, TMPDIR);
private static AlternativeDomainNames alternativeResolver = null; private static AlternativeDomainNames alternativeResolver = null;
@ -124,8 +124,8 @@ public final class HTTPDemon implements serverHandler, Cloneable {
public static final String hline = "-------------------------------------------------------------------------------"; public static final String hline = "-------------------------------------------------------------------------------";
public static final ConcurrentMap<String, String> reverseMappingCache = new ConcurrentHashMap<String, String>(); public static final ConcurrentMap<String, String> reverseMappingCache = new ConcurrentHashMap<String, String>();
private static volatile Switchboard switchboard = null; private static volatile Switchboard switchboard;
private static String virtualHost = null; private static String virtualHost;
public static boolean keepAliveSupport = false; public static boolean keepAliveSupport = false;
private static ConcurrentMap<String, Long> YaCyHopAccessRequester = new ConcurrentHashMap<String, Long>(); private static ConcurrentMap<String, Long> YaCyHopAccessRequester = new ConcurrentHashMap<String, Long>();
@ -166,17 +166,17 @@ public final class HTTPDemon implements serverHandler, Cloneable {
this.keepAliveRequestCount = 0; this.keepAliveRequestCount = 0;
} }
private static boolean allowProxy(Session session) { private static boolean allowProxy(final Session session) {
final String proxyClient = switchboard.getConfig("proxyClient", "*"); final String proxyClient = switchboard.getConfig("proxyClient", "*");
return (proxyClient.equals("*")) ? true : match(session.userAddress.getHostAddress(), proxyClient); return (proxyClient.equals("*")) ? true : match(session.userAddress.getHostAddress(), proxyClient);
} }
private static boolean allowServer(Session session) { private static boolean allowServer(final Session session) {
final String serverClient = switchboard.getConfig("serverClient", "*"); final String serverClient = switchboard.getConfig("serverClient", "*");
return (serverClient.equals("*")) ? true : match(session.userAddress.getHostAddress(), serverClient); return (serverClient.equals("*")) ? true : match(session.userAddress.getHostAddress(), serverClient);
} }
private static boolean allowYaCyHop(Session session) { private static boolean allowYaCyHop(final Session session) {
return switchboard.getConfigBool("YaCyHop", false); return switchboard.getConfigBool("YaCyHop", false);
} }
@ -221,9 +221,10 @@ public final class HTTPDemon implements serverHandler, Cloneable {
/** /**
* This function is used to determine if a persistent connection was requested by the client. * This function is used to determine if a persistent connection was requested by the client.
* @param header the received http-headers * @param header the received http-headers
* @param prop
* @return <code>true</code> if a persistent connection was requested or <code>false</code> otherwise * @return <code>true</code> if a persistent connection was requested or <code>false</code> otherwise
*/ */
private boolean handlePersistentConnection(final RequestHeader header, Properties prop) { private boolean handlePersistentConnection(final RequestHeader header, final Properties prop) {
if (!keepAliveSupport) { if (!keepAliveSupport) {
prop.put(HeaderFramework.CONNECTION_PROP_PERSISTENT,"close"); prop.put(HeaderFramework.CONNECTION_PROP_PERSISTENT,"close");
@ -315,45 +316,44 @@ public final class HTTPDemon implements serverHandler, Cloneable {
private static long lastAccessDelta(final Map<String, Long> accessTable, final String domain) { private static long lastAccessDelta(final Map<String, Long> accessTable, final String domain) {
final Long lastAccess = accessTable.get(domain); final Long lastAccess = accessTable.get(domain);
if (lastAccess == null) return Long.MAX_VALUE; // never accessed return (lastAccess == null) ? Long.MAX_VALUE : System.currentTimeMillis() - lastAccess.longValue();
return System.currentTimeMillis() - lastAccess.longValue();
} }
private boolean handleProxyAuthentication(final RequestHeader header, Properties prop, Session session) throws IOException { private boolean handleProxyAuthentication(final RequestHeader header, final Properties prop, final Session session) throws IOException {
// getting the http version that is used by the client // getting the http version that is used by the client
final String httpVersion = prop.getProperty("HTTP", "HTTP/0.9"); final String httpVersion = prop.getProperty("HTTP", "HTTP/0.9");
// reading the authentication settings from switchboard // reading the authentication settings from switchboard
if (!this.proxyAccounts_init) { if (!this.proxyAccounts_init) {
this.use_proxyAccounts = switchboard.getConfigBool("use_proxyAccounts", false); this.use_proxyAccounts = switchboard.getConfigBool("use_proxyAccounts", false);
this.proxyAccounts_init = true; // is initialised this.proxyAccounts_init = true; // is initialised
} }
if (this.use_proxyAccounts) { if (this.use_proxyAccounts) {
final String auth = header.get(RequestHeader.PROXY_AUTHORIZATION,"xxxxxx"); final String auth = header.get(RequestHeader.PROXY_AUTHORIZATION,"xxxxxx");
UserDB.Entry entry=switchboard.userDB.ipAuth(session.userAddress.getHostAddress()); UserDB.Entry entry = switchboard.userDB.ipAuth(session.userAddress.getHostAddress());
if(entry == null){ if (entry == null) {
entry=switchboard.userDB.proxyAuth(auth, session.userAddress.getHostAddress()); entry = switchboard.userDB.proxyAuth(auth, session.userAddress.getHostAddress());
} }
if(entry != null){ if (entry != null) {
final int returncode=entry.surfRight(); final int returncode=entry.surfRight();
if(returncode==UserDB.Entry.PROXY_ALLOK){ if (returncode == UserDB.Entry.PROXY_ALLOK) {
return true; return true;
} }
final serverObjects tp=new serverObjects(); final serverObjects tp = new serverObjects();
if(returncode==UserDB.Entry.PROXY_TIMELIMIT_REACHED){ if (returncode == UserDB.Entry.PROXY_TIMELIMIT_REACHED) {
tp.put("limit", "1");//time per day tp.put("limit", "1");//time per day
tp.put("limit_timelimit", entry.getTimeLimit()); tp.put("limit_timelimit", entry.getTimeLimit());
sendRespondError(prop, session.out, 403, "Internet-Timelimit reached", new File("proxymsg/proxylimits.inc"), tp, null); sendRespondError(prop, session.out, 403, "Internet-Timelimit reached", new File("proxymsg/proxylimits.inc"), tp, null);
}else if(returncode==UserDB.Entry.PROXY_NORIGHT){ } else if (returncode == UserDB.Entry.PROXY_NORIGHT){
tp.put("limit", "0"); tp.put("limit", "0");
sendRespondError(prop, session.out, 403, "Proxy use forbidden", new File("proxymsg/proxylimits.inc"), tp, null); sendRespondError(prop, session.out, 403, "Proxy use forbidden", new File("proxymsg/proxylimits.inc"), tp, null);
} }
return false; return false;
} }
// ask for authenticate // ask for authenticate
session.out.write((httpVersion + " 407 Proxy Authentication Required" + serverCore.CRLF_STRING + session.out.write((httpVersion + " 407 Proxy Authentication Required" + serverCore.CRLF_STRING +
RequestHeader.PROXY_AUTHENTICATE + ": Basic realm=\"log-in\"" + serverCore.CRLF_STRING).getBytes()); RequestHeader.PROXY_AUTHENTICATE + ": Basic realm=\"log-in\"" + serverCore.CRLF_STRING).getBytes());
session.out.write((HeaderFramework.CONTENT_LENGTH + ": 0\r\n").getBytes()); session.out.write((HeaderFramework.CONTENT_LENGTH + ": 0\r\n").getBytes());
session.out.write("\r\n".getBytes()); session.out.write("\r\n".getBytes());
session.out.flush(); session.out.flush();
@ -363,12 +363,11 @@ public final class HTTPDemon implements serverHandler, Cloneable {
return true; return true;
} }
public Boolean EMPTY(final String arg, Session session) throws IOException { public Boolean EMPTY(final String arg, final Session session) throws IOException {
if (++this.emptyRequestCount > 10) return serverCore.TERMINATE_CONNECTION; return (++this.emptyRequestCount > 10) ? serverCore.TERMINATE_CONNECTION : serverCore.RESUME_CONNECTION;
return serverCore.RESUME_CONNECTION;
} }
public Boolean UNKNOWN(final String requestLine, Session session) throws IOException { public Boolean UNKNOWN(final String requestLine, final Session session) throws IOException {
Properties prop = parseRequestLine(HeaderFramework.METHOD_GET, requestLine, session); Properties prop = parseRequestLine(HeaderFramework.METHOD_GET, requestLine, session);
int pos; int pos;
@ -387,16 +386,16 @@ public final class HTTPDemon implements serverHandler, Cloneable {
return serverCore.TERMINATE_CONNECTION; return serverCore.TERMINATE_CONNECTION;
} }
public Boolean GET(final String arg, Session session) { public Boolean GET(final String arg, final Session session) {
try { try {
// parsing the http request line // parsing the http request line
Properties prop = parseRequestLine(HeaderFramework.METHOD_GET, arg, session); final Properties prop = parseRequestLine(HeaderFramework.METHOD_GET, arg, session);
// we now know the HTTP version. depending on that, we read the header // we now know the HTTP version. depending on that, we read the header
final String httpVersion = prop.getProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_0_9); final String httpVersion = prop.getProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_0_9);
final RequestHeader header = (httpVersion.equals(HeaderFramework.HTTP_VERSION_0_9)) final RequestHeader header = (httpVersion.equals(HeaderFramework.HTTP_VERSION_0_9))
? new RequestHeader(reverseMappingCache) ? new RequestHeader(reverseMappingCache)
: readHeader(prop, session); : readHeader(prop, session);
// handling transparent proxy support // handling transparent proxy support
handleTransparentProxySupport(header, prop, virtualHost, HTTPDProxyHandler.isTransparentProxy); handleTransparentProxySupport(header, prop, virtualHost, HTTPDProxyHandler.isTransparentProxy);
@ -432,7 +431,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
} }
} }
private void logUnexpectedError(final Exception e, String address) { private void logUnexpectedError(final Exception e, final String address) {
if (e instanceof InterruptedException) { if (e instanceof InterruptedException) {
log.logInfo("Interruption detected"); log.logInfo("Interruption detected");
} else { } else {
@ -454,15 +453,15 @@ public final class HTTPDemon implements serverHandler, Cloneable {
} }
} }
public Boolean HEAD(final String arg, Session session) { public Boolean HEAD(final String arg, final Session session) {
try { try {
Properties prop = parseRequestLine(HeaderFramework.METHOD_HEAD, arg, session); final Properties prop = parseRequestLine(HeaderFramework.METHOD_HEAD, arg, session);
// we now know the HTTP version. depending on that, we read the header // we now know the HTTP version. depending on that, we read the header
RequestHeader header;
final String httpVersion = prop.getProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_0_9); final String httpVersion = prop.getProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_0_9);
if (httpVersion.equals(HeaderFramework.HTTP_VERSION_0_9)) header = new RequestHeader(reverseMappingCache); final RequestHeader header = (httpVersion.equals(HeaderFramework.HTTP_VERSION_0_9))
else header = readHeader(prop,session); ? new RequestHeader(reverseMappingCache)
: readHeader(prop,session);
// handle transparent proxy support // handle transparent proxy support
handleTransparentProxySupport(header, prop, virtualHost, HTTPDProxyHandler.isTransparentProxy); handleTransparentProxySupport(header, prop, virtualHost, HTTPDProxyHandler.isTransparentProxy);
@ -500,15 +499,15 @@ public final class HTTPDemon implements serverHandler, Cloneable {
} }
} }
public Boolean POST(final String arg, Session session) { public Boolean POST(final String arg, final Session session) {
try { try {
Properties prop = parseRequestLine(HeaderFramework.METHOD_POST, arg, session); final Properties prop = parseRequestLine(HeaderFramework.METHOD_POST, arg, session);
// we now know the HTTP version. depending on that, we read the header // we now know the HTTP version. depending on that, we read the header
RequestHeader header;
final String httpVersion = prop.getProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_0_9); final String httpVersion = prop.getProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_0_9);
if (httpVersion.equals(HeaderFramework.HTTP_VERSION_0_9)) header = new RequestHeader(reverseMappingCache); final RequestHeader header = (httpVersion.equals(HeaderFramework.HTTP_VERSION_0_9))
else header = readHeader(prop, session); ? new RequestHeader(reverseMappingCache)
: readHeader(prop, session);
// handle transfer-coding // handle transfer-coding
final InputStream sessionIn; final InputStream sessionIn;
@ -582,16 +581,18 @@ public final class HTTPDemon implements serverHandler, Cloneable {
prop.setProperty(HeaderFramework.CONNECTION_PROP_HOST,(HTTPDemon.isThisHostName(dstHostSocket)?virtualHost:dstHostSocket)); prop.setProperty(HeaderFramework.CONNECTION_PROP_HOST,(HTTPDemon.isThisHostName(dstHostSocket)?virtualHost:dstHostSocket));
} }
public Boolean CONNECT(String arg, Session session) throws IOException { public Boolean CONNECT(String arg, final Session session) throws IOException {
// establish a ssh-tunneled http connection // establish a ssh-tunneled http connection
// this is to support https // this is to support https
// parse HTTP version // parse HTTP version
int pos = arg.indexOf(' '); int pos = arg.indexOf(' ');
String httpVersion = "HTTP/1.0"; final String httpVersion;
if (pos >= 0) { if (pos >= 0) {
httpVersion = arg.substring(pos + 1); httpVersion = arg.substring(pos + 1);
arg = arg.substring(0, pos); arg = arg.substring(0, pos);
} else {
httpVersion = "HTTP/1.0";
} }
Properties prop = new Properties(); Properties prop = new Properties();
prop.setProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, httpVersion); prop.setProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, httpVersion);
@ -641,10 +642,10 @@ public final class HTTPDemon implements serverHandler, Cloneable {
return serverCore.TERMINATE_CONNECTION; return serverCore.TERMINATE_CONNECTION;
} }
private final Properties parseRequestLine(final String cmd, final String s, Session session) { private final Properties parseRequestLine(final String cmd, final String s, final Session session) {
// parsing the header // parsing the header
Properties p = parseRequestLine(cmd, s, virtualHost); final Properties p = parseRequestLine(cmd, s, virtualHost);
// track the request // track the request
final String path = p.getProperty(HeaderFramework.CONNECTION_PROP_URL); final String path = p.getProperty(HeaderFramework.CONNECTION_PROP_URL);
@ -788,7 +789,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
// replace all other // replace all other
final CharArrayWriter b = new CharArrayWriter(s.length()); final CharArrayWriter b = new CharArrayWriter(s.length());
int end; int end;
for (int i=0; i<s.length(); i++) { for (int i = 0, len = s.length(); i < len; i++) {
if (s.charAt(i) == '&' && (end = s.indexOf(';', i + 1)) > i) { if (s.charAt(i) == '&' && (end = s.indexOf(';', i + 1)) > i) {
if (s.charAt(i + 1) == '#') { // &#1234; symbols if (s.charAt(i + 1) == '#') { // &#1234; symbols
b.write(Integer.parseInt(s.substring(i + 2, end))); b.write(Integer.parseInt(s.substring(i + 2, end)));
@ -818,7 +819,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
*/ */
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public static Map<String, byte[]> parseMultipart(final RequestHeader header, final serverObjects args, final InputStream in) throws IOException { public static Map<String, byte[]> parseMultipart(final RequestHeader header, final serverObjects args, final InputStream in) throws IOException {
//ByteArrayInputStream in = new ByteArrayInputStream(FileUtils.read(inx));
final InputStream body = prepareBody(header, in); final InputStream body = prepareBody(header, in);
RequestContext request = new yacyContextRequest(header, body); RequestContext request = new yacyContextRequest(header, body);
@ -834,20 +835,18 @@ public final class HTTPDemon implements serverHandler, Cloneable {
} }
// parse data in memory // parse data in memory
FileUpload upload = new FileUpload(diskFileItemFactory); final FileUpload upload = new FileUpload(DISK_FILE_ITEM_FACTORY);
List<FileItem> items; final List<FileItem> items;
//long time = System.currentTimeMillis();
try { try {
items = upload.parseRequest(request); items = upload.parseRequest(request);
} catch (FileUploadException e) { } catch (FileUploadException e) {
//Log.logException(e);
throw new IOException("FileUploadException " + e.getMessage()); throw new IOException("FileUploadException " + e.getMessage());
} }
//System.out.println("**** FileUploadBase.parseRequest time = " + (System.currentTimeMillis() - time));
// format information for further usage // format information for further usage
final HashMap<String, byte[]> files = new HashMap<String, byte[]>(); final Map<String, byte[]> files = new HashMap<String, byte[]>();
for (FileItem item : items) { for (final FileItem item : items) {
if (item.isFormField()) { if (item.isFormField()) {
// simple text // simple text
if (item.getContentType() == null || !item.getContentType().contains("charset")) { if (item.getContentType() == null || !item.getContentType().contains("charset")) {
@ -915,7 +914,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
* @param header * @param header
* @param in * @param in
*/ */
public yacyContextRequest(Map<String, String> requestHeader, InputStream in) { public yacyContextRequest(final Map<String, String> requestHeader, final InputStream in) {
super(null, requestHeader); super(null, requestHeader);
this.inStream = in; this.inStream = in;
} }
@ -932,38 +931,25 @@ public final class HTTPDemon implements serverHandler, Cloneable {
} }
/*
static int nextPos = -1;
private static byte[] readLine(final int start, final byte[] array) {
// read a string from an array; line ending is always CRLF
// but we are also fuzzy with that: may also be only CR or LF
// if no remaining CR, CRLF or LF can be found, return null
if (start > array.length) return null;
int pos = indexOf(start, array, serverCore.CRLF); nextPos = pos + 2;
if (pos < 0) {pos = indexOf(start, array, new byte[] {serverCore.CR}); nextPos = pos + 1;}
if (pos < 0) {pos = indexOf(start, array, new byte[] {serverCore.LF}); nextPos = pos + 1;}
if (pos < 0) {nextPos = start; return null;}
final byte[] result = new byte[pos - start];
java.lang.System.arraycopy(array, start, result, 0, pos - start);
return result;
}
*/
public static int indexOf(final int start, final byte[] array, final byte[] pattern) { public static int indexOf(final int start, final byte[] array, final byte[] pattern) {
// return a position of a pattern in an array // return a position of a pattern in an array
if (start > array.length - pattern.length) return -1; if (start > array.length - pattern.length) return -1;
if (pattern.length == 0) return start; if (pattern.length == 0) return start;
for (int pos = start; pos <= array.length - pattern.length; pos++) for (int pos = start, lens = array.length - pattern.length; pos <= lens; pos++) {
if ((array[pos] == pattern[0]) && (equals(array, pos, pattern, 0, pattern.length))) if ((array[pos] == pattern[0]) && (equals(array, pos, pattern, 0, pattern.length))) {
return pos; return pos;
}
}
return -1; return -1;
} }
public static boolean equals(final byte[] a, final int aoff, final byte[] b, final int boff, final int len) { public static boolean equals(final byte[] a, final int aoff, final byte[] b, final int boff, final int len) {
//System.out.println("equals: a = " + new String(a) + ", aoff = " + aoff + ", b = " + new String(b) + ", boff = " + boff + ", length = " + len);
if ((aoff + len > a.length) || (boff + len > b.length)) return false; if ((aoff + len > a.length) || (boff + len > b.length)) return false;
for (int i = 0; i < len; i++) if (a[aoff + i] != b[boff + i]) return false; for (int i = 0; i < len; i++) {
//System.out.println("TRUE!"); if (a[aoff + i] != b[boff + i]) {
return false;
}
}
return true; return true;
} }
@ -980,7 +966,6 @@ public final class HTTPDemon implements serverHandler, Cloneable {
respond.flush(); respond.flush();
} }
public static final void sendRespondError( public static final void sendRespondError(
final Properties conProp, final Properties conProp,
final OutputStream respond, final OutputStream respond,
@ -1059,11 +1044,13 @@ public final class HTTPDemon implements serverHandler, Cloneable {
final String args = conProp.getProperty(HeaderFramework.CONNECTION_PROP_ARGS); final String args = conProp.getProperty(HeaderFramework.CONNECTION_PROP_ARGS);
final String method = conProp.getProperty(HeaderFramework.CONNECTION_PROP_METHOD); final String method = conProp.getProperty(HeaderFramework.CONNECTION_PROP_METHOD);
int port = 80; final int port;
final int pos = host.indexOf(':'); final int pos = host.indexOf(':');
if (pos != -1) { if (pos != -1) {
port = Integer.parseInt(host.substring(pos + 1)); port = Integer.parseInt(host.substring(pos + 1));
host = host.substring(0, pos); host = host.substring(0, pos);
} else {
port = 80;
} }
String urlString; String urlString;
@ -1102,7 +1089,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
tp.put("errorMessageType_file", (detailedErrorMsgFile == null) ? "" : detailedErrorMsgFile.toString()); tp.put("errorMessageType_file", (detailedErrorMsgFile == null) ? "" : detailedErrorMsgFile.toString());
if ((detailedErrorMsgValues != null) && !detailedErrorMsgValues.isEmpty()) { if ((detailedErrorMsgValues != null) && !detailedErrorMsgValues.isEmpty()) {
// rewriting the value-names and add the proper name prefix: // rewriting the value-names and add the proper name prefix:
for (Entry<String, String> entry: detailedErrorMsgValues.entrySet()) { for (final Entry<String, String> entry: detailedErrorMsgValues.entrySet()) {
tp.put("errorMessageType_" + entry.getKey(), entry.getValue()); tp.put("errorMessageType_" + entry.getKey(), entry.getValue());
} }
} }
@ -1141,8 +1128,9 @@ public final class HTTPDemon implements serverHandler, Cloneable {
final byte[] result = o.toByteArray(); final byte[] result = o.toByteArray();
o.close(); o = null; o.close(); o = null;
if(header == null) if(header == null) {
header = new ResponseHeader(); header = new ResponseHeader();
}
header.put(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_STATUS, Integer.toString(httpStatusCode)); header.put(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_STATUS, Integer.toString(httpStatusCode));
header.put(HeaderFramework.DATE, systemDate); header.put(HeaderFramework.DATE, systemDate);
header.put(HeaderFramework.CONTENT_TYPE, "text/html"); header.put(HeaderFramework.CONTENT_TYPE, "text/html");
@ -1184,23 +1172,23 @@ public final class HTTPDemon implements serverHandler, Cloneable {
} }
if (!reqMethod.equals(HeaderFramework.METHOD_HEAD)){ if (!reqMethod.equals(HeaderFramework.METHOD_HEAD)){
if (!conProp.getProperty(HeaderFramework.CONNECTION_PROP_PERSISTENT,"close").equals("close")) { if (!conProp.getProperty(HeaderFramework.CONNECTION_PROP_PERSISTENT,"close").equals("close") &&
if (transferEnc == null && contentLength < 0) { transferEnc == null && contentLength < 0) {
throw new IllegalArgumentException("Message MUST contain a Content-Length or a non-identity transfer-coding header field."); throw new IllegalArgumentException("Message MUST contain a Content-Length or a non-identity transfer-coding header field.");
}
} }
if (transferEnc != null && contentLength >= 0) { if (transferEnc != null && contentLength >= 0) {
throw new IllegalArgumentException("Messages MUST NOT include both a Content-Length header field and a non-identity transfer-coding."); throw new IllegalArgumentException("Messages MUST NOT include both a Content-Length header field and a non-identity transfer-coding.");
} }
} }
if(headers==null) headers = new ResponseHeader(); if (headers == null) {
headers = new ResponseHeader();
}
final Date now = new Date(System.currentTimeMillis()); final Date now = new Date(System.currentTimeMillis());
headers.put(HeaderFramework.SERVER, "AnomicHTTPD (www.anomic.de)"); headers.put(HeaderFramework.SERVER, "AnomicHTTPD (www.anomic.de)");
headers.put(HeaderFramework.DATE, HeaderFramework.formatRFC1123(now)); headers.put(HeaderFramework.DATE, HeaderFramework.formatRFC1123(now));
if (moddate.after(now)) { if (moddate.after(now)) {
//System.out.println("*** DEBUG: correcting moddate = " + moddate.toString() + " to now = " + now.toString());
moddate = now; moddate = now;
} }
headers.put(HeaderFramework.LAST_MODIFIED, HeaderFramework.formatRFC1123(moddate)); headers.put(HeaderFramework.LAST_MODIFIED, HeaderFramework.formatRFC1123(moddate));
@ -1286,30 +1274,19 @@ public final class HTTPDemon implements serverHandler, Cloneable {
//responseHeader.put(HeaderFramework.X_YACY_PREVIOUS_REQUEST_LINE,conProp.getProperty(HeaderFramework.CONNECTION_PROP_PREV_REQUESTLINE)); //responseHeader.put(HeaderFramework.X_YACY_PREVIOUS_REQUEST_LINE,conProp.getProperty(HeaderFramework.CONNECTION_PROP_PREV_REQUESTLINE));
//read custom headers //read custom headers
/* final Iterator<ResponseHeader.Entry> it = responseHeader.getAdditionalHeaderProperties().iterator();
if (requestProperties != null) while(it.hasNext()) {
{ //Append user properties to the main String
httpHeader outgoingHeader=requestProperties.getOutgoingHeader(); //TODO: Should we check for user properites. What if they intersect properties that are already in header?
if (outgoingHeader!=null) final ResponseHeader.Entry e = it.next();
{*/ header.append(e.getKey()).append(": ").append(e.getValue()).append("\r\n");
final Iterator<ResponseHeader.Entry> it = responseHeader.getAdditionalHeaderProperties().iterator(); }
while(it.hasNext()) {
//Append user properties to the main String
//TODO: Should we check for user properites. What if they intersect properties that are already in header?
final ResponseHeader.Entry e = it.next();
header.append(e.getKey()).append(": ").append(e.getValue()).append("\r\n");
}
/*
}
}*/
// write header // write header
final Iterator<String> i = responseHeader.keySet().iterator(); final Iterator<String> i = responseHeader.keySet().iterator();
String key; String key;
char tag; char tag;
int count; int count;
//System.out.println("vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv");
while (i.hasNext()) { while (i.hasNext()) {
key = i.next(); key = i.next();
tag = key.charAt(0); tag = key.charAt(0);
@ -1318,7 +1295,6 @@ public final class HTTPDemon implements serverHandler, Cloneable {
for (int j = 0; j < count; j++) { for (int j = 0; j < count; j++) {
header.append(key).append(": ").append(responseHeader.getSingle(key, j)).append("\r\n"); header.append(key).append(": ").append(responseHeader.getSingle(key, j)).append("\r\n");
} }
//System.out.println("#" + key + ": " + value);
} }
} }
@ -1386,10 +1362,10 @@ public final class HTTPDemon implements serverHandler, Cloneable {
// check if the destination port is equal to the port yacy is listening to // check if the destination port is equal to the port yacy is listening to
dstPort.equals(Integer.valueOf(serverCore.getPortNr(switchboard.getConfig("port", "8090")))) && dstPort.equals(Integer.valueOf(serverCore.getPortNr(switchboard.getConfig("port", "8090")))) &&
( (
// check if the destination host is our local IP address // check if the destination host is our local IP address
Domains.isThisHostIP(dstHost) || Domains.isThisHostIP(dstHost) ||
// check if the destination host is our seed ip address // check if the destination host is our seed ip address
isThisSeedIP(dstHost) isThisSeedIP(dstHost)
) )
) { ) {
return true; return true;
@ -1401,7 +1377,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
/** /**
* @param alternativeResolver the alternativeResolver to set * @param alternativeResolver the alternativeResolver to set
*/ */
public static void setAlternativeResolver(AlternativeDomainNames alternativeResolver) { public static void setAlternativeResolver(final AlternativeDomainNames alternativeResolver) {
HTTPDemon.alternativeResolver = alternativeResolver; HTTPDemon.alternativeResolver = alternativeResolver;
} }
@ -1462,7 +1438,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
prop.setProperty(HeaderFramework.CONNECTION_PROP_REQUESTLINE, cmd + " " + args); prop.setProperty(HeaderFramework.CONNECTION_PROP_REQUESTLINE, cmd + " " + args);
// this parses a whole URL // this parses a whole URL
if (args.length() == 0) { if (args.isEmpty()) {
prop.setProperty(HeaderFramework.CONNECTION_PROP_HOST, virtualHost); prop.setProperty(HeaderFramework.CONNECTION_PROP_HOST, virtualHost);
prop.setProperty(HeaderFramework.CONNECTION_PROP_PATH, "/"); prop.setProperty(HeaderFramework.CONNECTION_PROP_PATH, "/");
prop.setProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_0_9); prop.setProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_0_9);
@ -1472,7 +1448,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
// store the version propery "HTTP" and cut the query at both ends // store the version propery "HTTP" and cut the query at both ends
int sep = args.lastIndexOf(' '); int sep = args.lastIndexOf(' ');
if ((sep >= 0)&&(args.substring(sep + 1).toLowerCase().startsWith("http/"))) { if ((sep >= 0) && (args.substring(sep + 1).toLowerCase().startsWith("http/"))) {
// HTTP version is given // HTTP version is given
prop.setProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, args.substring(sep + 1).trim()); prop.setProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, args.substring(sep + 1).trim());
args = args.substring(0, sep).trim(); // cut off HTTP version mark args = args.substring(0, sep).trim(); // cut off HTTP version mark
@ -1496,19 +1472,22 @@ public final class HTTPDemon implements serverHandler, Cloneable {
// properties of the query are stored with the prefix "&" // properties of the query are stored with the prefix "&"
// additionally, the values URL and ARGC are computed // additionally, the values URL and ARGC are computed
String argsString = ""; final String argsString;
sep = args.indexOf('?'); sep = args.indexOf('?');
if (sep >= 0) { if (sep >= 0) {
// there are values attached to the query string // there are values attached to the query string
argsString = args.substring(sep + 1); // cut head from tail of query argsString = args.substring(sep + 1); // cut head from tail of query
args = args.substring(0, sep); args = args.substring(0, sep);
} else {
argsString = "";
} }
prop.setProperty(HeaderFramework.CONNECTION_PROP_URL, args); // store URL prop.setProperty(HeaderFramework.CONNECTION_PROP_URL, args); // store URL
//System.out.println("HTTPD: ARGS=" + argsString); if (!argsString.isEmpty()) {
if (argsString.length() != 0) prop.setProperty(HeaderFramework.CONNECTION_PROP_ARGS, argsString); // store arguments in original form prop.setProperty(HeaderFramework.CONNECTION_PROP_ARGS, argsString);
} // store arguments in original form
// finally find host string // finally find host string
String path; final String path;
if (args.toUpperCase().startsWith("HTTP://")) { if (args.toUpperCase().startsWith("HTTP://")) {
// a host was given. extract it and set path // a host was given. extract it and set path
args = args.substring(7); args = args.substring(7);
@ -1547,7 +1526,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
prop.setProperty(HeaderFramework.CONNECTION_PROP_PATH, path); prop.setProperty(HeaderFramework.CONNECTION_PROP_PATH, path);
// find out file extension (we already stripped ?-parameters from args) // find out file extension (we already stripped ?-parameters from args)
String ext = ""; // default when no file extension final String ext;
sep = path.lastIndexOf('.'); sep = path.lastIndexOf('.');
if (sep >= 0) { if (sep >= 0) {
final int ancpos = path.indexOf("#", sep + 1); final int ancpos = path.indexOf("#", sep + 1);
@ -1558,6 +1537,8 @@ public final class HTTPDemon implements serverHandler, Cloneable {
// ex: /foo/bar.php => php // ex: /foo/bar.php => php
ext = path.substring(sep + 1).toLowerCase(); ext = path.substring(sep + 1).toLowerCase();
} }
} else {
ext = ""; // default when no file extension
} }
prop.setProperty(HeaderFramework.CONNECTION_PROP_EXT, ext); prop.setProperty(HeaderFramework.CONNECTION_PROP_EXT, ext);

155
source/net/yacy/cora/protocol/Domains.java
Unescape View File

@ -25,6 +25,7 @@ import java.net.NetworkInterface;
import java.net.SocketException; import java.net.SocketException;
import java.net.UnknownHostException; import java.net.UnknownHostException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.Enumeration; import java.util.Enumeration;
@ -42,18 +43,18 @@ import net.yacy.cora.storage.ConcurrentARC;
public class Domains { public class Domains {
private static final String PRESENT = ""; private static final String PRESENT = "";
private static final String localPatterns = "10\\..*,127.*,172.(1[6-9]|2[0-9]|3[0-1])\\..*,169.254.*,192.168.*,localhost"; private static final String LOCAL_PATTERNS = "10\\..*,127\\..*,172\\.(1[6-9]|2[0-9]|3[0-1])\\..*,169\\.254\\..*,192\\.168\\..*,localhost";
private static final int maxNameCacheHitSize = 20000; private static final int MAX_NAME_CACHE_HIT_SIZE = 20000;
private static final int maxNameCacheMissSize = 20000; private static final int MAX_NAME_CACHE_MISS_SIZE = 20000;
private static final int maxNameNoCachingListSize = 20000; private static final int MAX_NAME_NO_CACHING_LIST_SIZE = 20000;
private static final int concurrencyLevel = Runtime.getRuntime().availableProcessors() + 1; private static final int CONCURRENCY_LEVEL = Runtime.getRuntime().availableProcessors() + 1;
// a dns cache // a dns cache
private static final ARC<String, InetAddress> nameCacheHit = new ConcurrentARC<String, InetAddress>(maxNameCacheHitSize, concurrencyLevel); private static final ARC<String, InetAddress> NAME_CACHE_HIT = new ConcurrentARC<String, InetAddress>(MAX_NAME_CACHE_HIT_SIZE, CONCURRENCY_LEVEL);
private static final ARC<String, String> nameCacheMiss = new ConcurrentARC<String, String>(maxNameCacheMissSize, concurrencyLevel); private static final ARC<String, String> NAME_CACHE_MISS = new ConcurrentARC<String, String>(MAX_NAME_CACHE_MISS_SIZE, CONCURRENCY_LEVEL);
private static final ARC<String, String> nameCacheNoCachingList = new ConcurrentARC<String, String>(maxNameNoCachingListSize, concurrencyLevel); private static final ARC<String, String> NAME_CACHE_NO_CACHING_LIST = new ConcurrentARC<String, String>(MAX_NAME_NO_CACHING_LIST_SIZE, CONCURRENCY_LEVEL);
public static List<Pattern> nameCacheNoCachingPatterns = Collections.synchronizedList(new LinkedList<Pattern>()); public static List<Pattern> nameCacheNoCachingPatterns = Collections.synchronizedList(new LinkedList<Pattern>());
public static final List<Pattern> localhostPatterns = makePatterns(localPatterns); public static final List<Pattern> LOCALHOST_PATTERNS = makePatterns(LOCAL_PATTERNS);
/** /**
* ! ! ! A T T E N T I O N A T T E N T I O N A T T E N T I O N ! ! ! * ! ! ! A T T E N T I O N A T T E N T I O N A T T E N T I O N ! ! !
@ -61,7 +62,7 @@ public class Domains {
* Do not move a TLD to another group (if you do not exactly know what you * Do not move a TLD to another group (if you do not exactly know what you
* are doing)! Because it will change the hash of the url! * are doing)! Because it will change the hash of the url!
*/ */
private static final String[] TLD_NorthAmericaOceania={ private static final String[] TLD_NorthAmericaOceania = {
// primary english-speaking countries // primary english-speaking countries
// english-speaking countries from central america are also included // english-speaking countries from central america are also included
// includes also dutch and french colonies in the caribbean sea // includes also dutch and french colonies in the caribbean sea
@ -383,10 +384,10 @@ public class Domains {
String tld; String tld;
//String name; //String name;
final Integer ID = Integer.valueOf(id); final Integer ID = Integer.valueOf(id);
for (int i = 0; i < TLDList.length; i++) { for (final String TLDelement : TLDList) {
p = TLDList[i].indexOf('='); p = TLDelement.indexOf('=');
if (p > 0) { if (p > 0) {
tld = TLDList[i].substring(0, p).toLowerCase(); tld = TLDelement.substring(0, p).toLowerCase();
//name = TLDList[i].substring(p + 1); //name = TLDList[i].substring(p + 1);
TLDID.put(tld, ID); TLDID.put(tld, ID);
//TLDName.put(tld, name); //TLDName.put(tld, name);
@ -425,46 +426,46 @@ public class Domains {
* @return String with the ip. null, if the host could not be resolved. * @return String with the ip. null, if the host could not be resolved.
*/ */
public static InetAddress dnsResolveFromCache(String host) throws UnknownHostException { public static InetAddress dnsResolveFromCache(String host) throws UnknownHostException {
if ((host == null) || (host.length() == 0)) return null; if ((host == null) || host.isEmpty()) return null;
host = host.toLowerCase().trim(); host = host.toLowerCase().trim();
// try to simply parse the address // try to simply parse the address
InetAddress ip = parseInetAddress(host); InetAddress ip = parseInetAddress(host);
if (ip != null) return ip; if (ip != null) return ip;
// trying to resolve host by doing a name cache lookup // trying to resolve host by doing a name cache lookup
ip = nameCacheHit.get(host); ip = NAME_CACHE_HIT.get(host);
if (ip != null) return ip; if (ip != null) return ip;
if (nameCacheMiss.containsKey(host)) return null; if (NAME_CACHE_MISS.containsKey(host)) return null;
throw new UnknownHostException("host not in cache"); throw new UnknownHostException("host not in cache");
} }
public static void setNoCachingPatterns(String patternList) { public static void setNoCachingPatterns(final String patternList) {
nameCacheNoCachingPatterns = makePatterns(patternList); nameCacheNoCachingPatterns = makePatterns(patternList);
} }
public static List<Pattern> makePatterns(String patternList) { public static List<Pattern> makePatterns(final String patternList) {
final String[] entries = patternList.split(","); final String[] entries = (patternList != null) ? patternList.split(",") : new String[0];
final List<Pattern> patterns = new ArrayList<Pattern>(entries.length); final List<Pattern> patterns = new ArrayList<Pattern>(entries.length);
for (int i = 0; i < entries.length; i++) { for (final String entry : entries) {
patterns.add(Pattern.compile(entries[i].trim())); patterns.add(Pattern.compile(entry.trim()));
} }
return patterns; return patterns;
} }
public static boolean matchesList(String obj, List<Pattern> patterns) { public static boolean matchesList(final String obj, final List<Pattern> patterns) {
for (Pattern nextPattern: patterns) { for (final Pattern nextPattern: patterns) {
if (nextPattern.matcher(obj).matches()) return true; if (nextPattern.matcher(obj).matches()) return true;
} }
return false; return false;
} }
public static String getHostName(final InetAddress i) { public static String getHostName(final InetAddress i) {
Collection<String> hosts = nameCacheHit.getKeys(i); final Collection<String> hosts = NAME_CACHE_HIT.getKeys(i);
if (hosts.size() > 0) return hosts.iterator().next(); if (!hosts.isEmpty()) return hosts.iterator().next();
String host = i.getHostName(); final String host = i.getHostName();
nameCacheHit.put(host, i); NAME_CACHE_HIT.put(host, i);
return host; return host;
/* /*
// call i.getHostName() using concurrency to interrupt execution in case of a time-out // call i.getHostName() using concurrency to interrupt execution in case of a time-out
@ -478,16 +479,16 @@ public class Domains {
public static InetAddress dnsResolve(String host) { public static InetAddress dnsResolve(String host) {
if ((host == null) || (host.length() == 0)) return null; if ((host == null) || (host.length() == 0)) return null;
host = host.toLowerCase().trim(); host = host.toLowerCase().trim();
// try to simply parse the address // try to simply parse the address
InetAddress ip = parseInetAddress(host); InetAddress ip = parseInetAddress(host);
if (ip != null) return ip; if (ip != null) return ip;
// try to resolve host by doing a name cache lookup // try to resolve host by doing a name cache lookup
ip = nameCacheHit.get(host); ip = NAME_CACHE_HIT.get(host);
if (ip != null) return ip; if (ip != null) return ip;
if (nameCacheMiss.containsKey(host)) return null; if (NAME_CACHE_MISS.containsKey(host)) return null;
// call dnsResolveNetBased(host) using concurrency to interrupt execution in case of a time-out // call dnsResolveNetBased(host) using concurrency to interrupt execution in case of a time-out
try { try {
@ -495,12 +496,12 @@ public class Domains {
ip = InetAddress.getByName(host); //TimeoutRequest.getByName(host, 1000); // this makes the DNS request to backbone ip = InetAddress.getByName(host); //TimeoutRequest.getByName(host, 1000); // this makes the DNS request to backbone
if ((ip == null) || if ((ip == null) ||
(ip.isLoopbackAddress()) || (ip.isLoopbackAddress()) ||
(nameCacheNoCachingList.containsKey(host)) (NAME_CACHE_NO_CACHING_LIST.containsKey(host))
) { ) {
doCaching = false; doCaching = false;
} else { } else {
if (matchesList(host, nameCacheNoCachingPatterns)) { if (matchesList(host, nameCacheNoCachingPatterns)) {
nameCacheNoCachingList.put(host, PRESENT); NAME_CACHE_NO_CACHING_LIST.put(host, PRESENT);
doCaching = false; doCaching = false;
} }
} }
@ -508,7 +509,7 @@ public class Domains {
if (doCaching && ip != null) { if (doCaching && ip != null) {
// add new entries // add new entries
nameCacheHit.put(host, ip); NAME_CACHE_HIT.put(host, ip);
} }
return ip; return ip;
} catch (final UnknownHostException e) { } catch (final UnknownHostException e) {
@ -516,14 +517,13 @@ public class Domains {
flushMissNameCache(); flushMissNameCache();
// add new entries // add new entries
nameCacheMiss.put(host, PRESENT); NAME_CACHE_MISS.put(host, PRESENT);
} }
return null; return null;
} }
private static final InetAddress parseInetAddress(final String ip) { private static final InetAddress parseInetAddress(final String ip) {
if (ip == null) return null; if (ip == null || ip.length() < 8) return null;
if (ip.length() < 8) return null;
final String[] ips = ip.split("\\."); final String[] ips = ip.split("\\.");
if (ips.length != 4) return null; if (ips.length != 4) return null;
final byte[] ipb = new byte[4]; final byte[] ipb = new byte[4];
@ -548,11 +548,11 @@ public class Domains {
* @return int The number of entries in the nameCacheHit map * @return int The number of entries in the nameCacheHit map
*/ */
public static int nameCacheHitSize() { public static int nameCacheHitSize() {
return nameCacheHit.size(); return NAME_CACHE_HIT.size();
} }
public static int nameCacheMissSize() { public static int nameCacheMissSize() {
return nameCacheMiss.size(); return NAME_CACHE_MISS.size();
} }
/** /**
@ -561,14 +561,14 @@ public class Domains {
* @return int The number of entries in the nameCacheNoCachingList list * @return int The number of entries in the nameCacheNoCachingList list
*/ */
public static int nameCacheNoCachingListSize() { public static int nameCacheNoCachingListSize() {
return nameCacheNoCachingList.size(); return NAME_CACHE_NO_CACHING_LIST.size();
} }
/** /**
* Removes old entries from the dns miss cache * Removes old entries from the dns miss cache
*/ */
public static void flushMissNameCache() { public static void flushMissNameCache() {
if (nameCacheMiss.size() > maxNameCacheMissSize) nameCacheMiss.clear(); if (NAME_CACHE_MISS.size() > MAX_NAME_CACHE_MISS_SIZE) NAME_CACHE_MISS.clear();
} }
private static String localHostName = "127.0.0.1"; private static String localHostName = "127.0.0.1";
@ -580,8 +580,8 @@ public class Domains {
if (localHostAddress != null) localHostAddresses.add(localHostAddress); if (localHostAddress != null) localHostAddresses.add(localHostAddress);
} catch (UnknownHostException e) {} } catch (UnknownHostException e) {}
try { try {
InetAddress[] moreAddresses = InetAddress.getAllByName(localHostName); final InetAddress[] moreAddresses = InetAddress.getAllByName(localHostName);
if (moreAddresses != null) for (InetAddress a: moreAddresses) localHostAddresses.add(a); if (moreAddresses != null) localHostAddresses.addAll(Arrays.asList(moreAddresses));
} catch (UnknownHostException e) {} } catch (UnknownHostException e) {}
// to get the local host name, a dns lookup is necessary. // to get the local host name, a dns lookup is necessary.
@ -589,15 +589,16 @@ public class Domains {
// therefore we start the host name lookup as concurrent thread // therefore we start the host name lookup as concurrent thread
// meanwhile the host name is "127.0.0.1" which is not completely wrong // meanwhile the host name is "127.0.0.1" which is not completely wrong
new Thread() { new Thread() {
@Override
public void run() { public void run() {
// try to get local addresses from interfaces // try to get local addresses from interfaces
try { try {
Enumeration<NetworkInterface> nis = NetworkInterface.getNetworkInterfaces(); final Enumeration<NetworkInterface> nis = NetworkInterface.getNetworkInterfaces();
while (nis.hasMoreElements()) { while (nis.hasMoreElements()) {
NetworkInterface ni = nis.nextElement(); final NetworkInterface ni = nis.nextElement();
Enumeration<InetAddress> addrs = ni.getInetAddresses(); final Enumeration<InetAddress> addrs = ni.getInetAddresses();
while (addrs.hasMoreElements()) { while (addrs.hasMoreElements()) {
InetAddress addr = addrs.nextElement(); final InetAddress addr = addrs.nextElement();
if (addr != null) localHostAddresses.add(addr); if (addr != null) localHostAddresses.add(addr);
} }
} }
@ -612,14 +613,14 @@ public class Domains {
// after the host name was resolved, we try to look up more local addresses // after the host name was resolved, we try to look up more local addresses
// using the host name: // using the host name:
try { try {
InetAddress[] moreAddresses = InetAddress.getAllByName(localHostName); final InetAddress[] moreAddresses = InetAddress.getAllByName(localHostName);
if (moreAddresses != null) for (InetAddress a: moreAddresses) localHostAddresses.add(a); if (moreAddresses != null) localHostAddresses.addAll(Arrays.asList(moreAddresses));
} catch (UnknownHostException e) { } catch (UnknownHostException e) {
} }
// fill a cache of local host names // fill a cache of local host names
for (InetAddress a: localHostAddresses) { for (final InetAddress a: localHostAddresses) {
String hostname = getHostName(a); final String hostname = getHostName(a);
if (hostname != null) { if (hostname != null) {
localHostNames.add(hostname); localHostNames.add(hostname);
localHostNames.add(a.getHostAddress()); localHostNames.add(a.getHostAddress());
@ -632,7 +633,7 @@ public class Domains {
public static InetAddress myPublicLocalIP() { public static InetAddress myPublicLocalIP() {
// list all addresses // list all addresses
// for (int i = 0; i < localHostAddresses.length; i++) System.out.println("IP: " + localHostAddresses[i].getHostAddress()); // DEBUG // for (int i = 0; i < localHostAddresses.length; i++) System.out.println("IP: " + localHostAddresses[i].getHostAddress()); // DEBUG
if (localHostAddresses.size() == 0) { if (localHostAddresses.isEmpty()) {
return null; return null;
} }
if (localHostAddresses.size() == 1) { if (localHostAddresses.size() == 1) {
@ -641,7 +642,7 @@ public class Domains {
} }
// we have more addresses, find an address that is not local // we have more addresses, find an address that is not local
int b0, b1; int b0, b1;
for (InetAddress a: localHostAddresses) { for (final InetAddress a: localHostAddresses) {
b0 = 0Xff & a.getAddress()[0]; b0 = 0Xff & a.getAddress()[0];
b1 = 0Xff & a.getAddress()[1]; b1 = 0Xff & a.getAddress()[1];
if (b0 != 10 && // class A reserved if (b0 != 10 && // class A reserved
@ -655,26 +656,26 @@ public class Domains {
// return that one that is returned with InetAddress.getLocalHost() // return that one that is returned with InetAddress.getLocalHost()
// if appropriate // if appropriate
try { try {
InetAddress localHostAddress = InetAddress.getLocalHost(); final InetAddress localHostAddress = InetAddress.getLocalHost();
if (localHostAddress != null && if (localHostAddress != null &&
(0Xff & localHostAddress.getAddress()[0]) != 127 && (0Xff & localHostAddress.getAddress()[0]) != 127 &&
localHostAddress.getHostAddress().indexOf(":") < 0) return localHostAddress; localHostAddress.getHostAddress().indexOf(":") < 0) return localHostAddress;
} catch (UnknownHostException e) { } catch (UnknownHostException e) {
} }
// we filter out the loopback address 127.0.0.1 and all addresses without a name // we filter out the loopback address 127.0.0.1 and all addresses without a name
for (InetAddress a: localHostAddresses) { for (final InetAddress a: localHostAddresses) {
if ((0Xff & a.getAddress()[0]) != 127 && if ((0Xff & a.getAddress()[0]) != 127 &&
a.getHostAddress().indexOf(":") < 0 && a.getHostAddress().indexOf(":") < 0 &&
a.getHostName() != null && a.getHostName() != null &&
a.getHostName().length() > 0) return a; !a.getHostName().isEmpty()) return a;
} }
// if no address has a name, then take any other than the loopback // if no address has a name, then take any other than the loopback
for (InetAddress a: localHostAddresses) { for (final InetAddress a: localHostAddresses) {
if ((0Xff & a.getAddress()[0]) != 127 && if ((0Xff & a.getAddress()[0]) != 127 &&
a.getHostAddress().indexOf(":") < 0) return a; a.getHostAddress().indexOf(":") < 0) return a;
} }
// if all fails, give back whatever we have // if all fails, give back whatever we have
for (InetAddress a: localHostAddresses) { for (final InetAddress a: localHostAddresses) {
if (a.getHostAddress().indexOf(":") < 0) return a; if (a.getHostAddress().indexOf(":") < 0) return a;
} }
// finally, just get any // finally, just get any
@ -688,11 +689,11 @@ public class Domains {
public static Set<InetAddress> myIntranetIPs() { public static Set<InetAddress> myIntranetIPs() {
// list all local addresses // list all local addresses
if (localHostAddresses.size() < 1) try {Thread.sleep(1000);} catch (InterruptedException e) {} if (localHostAddresses.size() < 1) try {Thread.sleep(1000);} catch (InterruptedException e) {}
Set<InetAddress> list = new HashSet<InetAddress>(); final Set<InetAddress> list = new HashSet<InetAddress>();
if (localHostAddresses.size() == 0) return list; // give up if (localHostAddresses.isEmpty()) return list; // give up
for (InetAddress a: localHostAddresses) { for (final InetAddress a: localHostAddresses) {
if ((0Xff & a.getAddress()[0]) == 127) continue; if (((0Xff & a.getAddress()[0]) == 127) ||
if (!matchesList(a.getHostAddress(), localhostPatterns)) continue; (!matchesList(a.getHostAddress(), LOCALHOST_PATTERNS))) continue;
list.add(a); list.add(a);
} }
return list; return list;
@ -706,7 +707,7 @@ public class Domains {
final InetAddress clientAddress = Domains.dnsResolve(hostName); final InetAddress clientAddress = Domains.dnsResolve(hostName);
if (clientAddress == null) return false; if (clientAddress == null) return false;
if (clientAddress.isAnyLocalAddress() || clientAddress.isLoopbackAddress()) return true; if (clientAddress.isAnyLocalAddress() || clientAddress.isLoopbackAddress()) return true;
for (InetAddress a: localHostAddresses) { for (final InetAddress a: localHostAddresses) {
if (a.equals(clientAddress)) { if (a.equals(clientAddress)) {
isThisHostIP = true; isThisHostIP = true;
break; break;
@ -723,7 +724,7 @@ public class Domains {
try { try {
if (clientAddress.isAnyLocalAddress() || clientAddress.isLoopbackAddress()) return true; if (clientAddress.isAnyLocalAddress() || clientAddress.isLoopbackAddress()) return true;
for (InetAddress a: localHostAddresses) { for (final InetAddress a: localHostAddresses) {
if (a.equals(clientAddress)) { if (a.equals(clientAddress)) {
isThisHostIP = true; isThisHostIP = true;
break; break;
@ -734,20 +735,18 @@ public class Domains {
} }
public static int getDomainID(final String host) { public static int getDomainID(final String host) {
if (host == null || host.length() == 0) return TLD_Local_ID; if (host == null || host.isEmpty() || isLocal(host)) return TLD_Local_ID;
if (isLocal(host)) return TLD_Local_ID;
final int p = host.lastIndexOf('.'); final int p = host.lastIndexOf('.');
String tld = (p > 0) ? host.substring(p + 1) : ""; final String tld = (p > 0) ? host.substring(p + 1) : "";
final Integer i = TLDID.get(tld); final Integer i = TLDID.get(tld);
if (i == null) return TLD_Generic_ID; return (i == null) ? TLD_Generic_ID : i.intValue();
return i.intValue();
} }
public static boolean isLocalhost(final String host) { public static boolean isLocalhost(final String host) {
if (host.equals("127.0.0.1")) return true; return ("127.0.0.1".equals(host) ||
if (host.equals("localhost")) return true; "localhost".equals(host) ||
if (host.startsWith("0:0:0:0:0:0:0:1")) return true; host.startsWith("0:0:0:0:0:0:0:1")
return false; );
} }
public static boolean isLocal(final String host) { public static boolean isLocal(final String host) {
@ -759,7 +758,7 @@ public class Domains {
// FIXME IPv4 only // FIXME IPv4 only
// check local ip addresses // check local ip addresses
if (matchesList(host, localhostPatterns)) return true; if (matchesList(host, LOCALHOST_PATTERNS)) return true;
if (host.startsWith("0:0:0:0:0:0:0:1")) return true; if (host.startsWith("0:0:0:0:0:0:0:1")) return true;
// check if there are other local IP addresses that are not in // check if there are other local IP addresses that are not in
@ -775,7 +774,7 @@ public class Domains {
// check dns lookup: may be a local address even if the domain name looks global // check dns lookup: may be a local address even if the domain name looks global
if (!recursive) return false; if (!recursive) return false;
InetAddress a = dnsResolve(host); final InetAddress a = dnsResolve(host);
/* /*
if (a == null) { if (a == null) {
// unknown if this is a local address. Could also be a timeout. // unknown if this is a local address. Could also be a timeout.
@ -786,7 +785,7 @@ public class Domains {
return a == null || a.isAnyLocalAddress() || a.isLinkLocalAddress() || a.isLoopbackAddress() || a.isSiteLocalAddress() || isLocal(a.getHostAddress(), false); return a == null || a.isAnyLocalAddress() || a.isLinkLocalAddress() || a.isLoopbackAddress() || a.isSiteLocalAddress() || isLocal(a.getHostAddress(), false);
} }
public static void main(String[] args) { public static void main(final String[] args) {
/* /*
try { try {
Enumeration<NetworkInterface> nis = NetworkInterface.getNetworkInterfaces(); Enumeration<NetworkInterface> nis = NetworkInterface.getNetworkInterfaces();
@ -804,7 +803,7 @@ public class Domains {
*/ */
try { Thread.sleep(1000);} catch (InterruptedException e) {} // get time for class init try { Thread.sleep(1000);} catch (InterruptedException e) {} // get time for class init
System.out.println("myPublicLocalIP: " + myPublicLocalIP()); System.out.println("myPublicLocalIP: " + myPublicLocalIP());
for (InetAddress a : myIntranetIPs()) { for (final InetAddress a : myIntranetIPs()) {
System.out.println("Intranet IP: " + a); System.out.println("Intranet IP: " + a);
} }
} }

Write Preview
Loading…
Cancel
Save