boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

*) replaced all IPs in IP filters for proxy with the proper regular expression

Browse Source 
*) some cleanup

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7477 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
low012 14 years ago
parent 9f6e3f9793
commit 64f32e8f00
7 changed files with 193 additions and 213 deletions
Show Stats Download Patch File Download Diff File

12
defaults/yacy.init
Unescape View File

@ -329,7 +329,7 @@ remoteProxyPort=4239
remoteProxyUser=
remoteProxyPwd=
remoteProxyNoProxy=10\..*,127.*,172.(1[6-9]|2[0-9]|3[0-1])\..*,169.254.*,192.168.*,localhost,0:0:0:0:0:0:0:1
remoteProxyNoProxy=10\..*,127\..*,172\.(1[6-9]|2[0-9]|3[0-1])\..*,169\.254\..*,192\.168\..*,localhost,0:0:0:0:0:0:0:1
# the proxy may filter the content of transferred web pages
# the bluelist removes specific keywords from web pages
@ -346,9 +346,9 @@ proxyBlueList=yacy.blue
# proxyClient: client-ip's that may connect the proxy for proxy service
# if several ip's are allowed then they must be separated by a ','
# any ip may contain the wildcard-sign '*'
# regular expressions may be used
#proxyClient=192.168.0.4
proxyClient=localhost,127.0.0.1,192.168.*,10\..*,0:0:0:0:0:0:0:1
proxyClient=localhost,127\.0\.0\.1,192\.168\..*,10\..*,0:0:0:0:0:0:0:1
# YaCyHop: allow public usage of proxy for yacy-protocol
# this enables usage of the internal http proxy for everyone,
@ -919,7 +919,7 @@ content.phpbb3.tableprefix = phpbb_
content.phpbb3.dbuser = notroot
content.phpbb3.dbpw = joshua
content.phpbb3.ppf = 1000
content.phpbb3.dumpfile =
content.phpbb3.dumpfile =
# segment assignment for index storage processes in YaCy:
# each process can store its index result in it's own index segment
@ -930,7 +930,7 @@ segment.process.dhtout_tmp = default
segment.process.proxy_tmp = default
segment.process.localcrawling_tmp = default
segment.process.remotecrawling_tmp = default
segment.process.default_tmp = default
segment.process.default_tmp = default
# search engine teaser: an about box in search results
# this is only shown, if the about.body is filled
@ -956,4 +956,4 @@ color_signgood = #009900
color_signother = #000099
color_searchheadline = #2200CC
color_searchurl = #008000
color_searchurlhover = #008000
color_searchurlhover = #008000

2
defaults/yacy.network.allip.unit
Unescape View File

@ -22,5 +22,5 @@ network.unit.update.location3 = https://latestyacy.f1ori.de/
network.unit.protocol.control = uncontrolled
# white/blacklists
network.unit.access.whitelist = 10\..*,127.*,172.(1[6-9]|2[0-9]|3[0-1])\..*,169.254.*,192.168.*,localhost
network.unit.access.whitelist = 10\..*,127\..*,172\.(1[6-9]|2[0-9]|3[0-1])\..*,169\.254\..*,192\.168\..*,localhost
network.unit.access.blacklist =

2
defaults/yacy.network.freeworld.unit
Unescape View File

@ -34,5 +34,5 @@ network.unit.update.location3 = https://latestyacy.f1ori.de/
network.unit.protocol.control = uncontrolled
# white/blacklists
network.unit.access.whitelist = 10\..*,127.*,172.(1[6-9]|2[0-9]|3[0-1])\..*,169.254.*,192.168.*,localhost
network.unit.access.whitelist = 10\..*,127\..*,172\.(1[6-9]|2[0-9]|3[0-1])\..*,169\.254\..*,192\.168\..*,localhost
network.unit.access.blacklist =

2
defaults/yacy.network.intranet.unit
Unescape View File

@ -29,5 +29,5 @@ network.unit.update.location3 = https://latestyacy.f1ori.de/
network.unit.protocol.control = uncontrolled
# white/blacklists
network.unit.access.whitelist = 10\..*,127.*,172.(1[6-9]|2[0-9]|3[0-1])\..*,169.254.*,192.168.*,localhost
network.unit.access.whitelist = 10\..*,127\..*,172\.(1[6-9]|2[0-9]|3[0-1])\..*,169\.254\..*,192\.168\..*,localhost
network.unit.access.blacklist =

2
defaults/yacy.network.webportal.unit
Unescape View File

@ -26,5 +26,5 @@ network.unit.update.location3 = https://latestyacy.f1ori.de/
network.unit.protocol.control = uncontrolled
# white/blacklists
network.unit.access.whitelist = 10\..*,127.*,172.(1[6-9]|2[0-9]|3[0-1])\..*,169.254.*,192.168.*,localhost
network.unit.access.whitelist = 10\..*,127\..*,172\.(1[6-9]|2[0-9]|3[0-1])\..*,169\.254\..*,192\.168\..*,localhost
network.unit.access.blacklist =

231
source/de/anomic/http/server/HTTPDemon.java
Unescape View File

@ -102,8 +102,8 @@ public final class HTTPDemon implements serverHandler, Cloneable {
public static final int ERRORCASE_MESSAGE = 4;
public static final int ERRORCASE_FILE = 5;
private static File TMPDIR = new File(System.getProperty("java.io.tmpdir"));
private static FileItemFactory diskFileItemFactory = new DiskFileItemFactory(5 * 1024 * 1024, TMPDIR);
private static final File TMPDIR = new File(System.getProperty("java.io.tmpdir"));
private static final FileItemFactory DISK_FILE_ITEM_FACTORY = new DiskFileItemFactory(5 * 1024 * 1024, TMPDIR);
private static AlternativeDomainNames alternativeResolver = null;
@ -124,8 +124,8 @@ public final class HTTPDemon implements serverHandler, Cloneable {
public static final String hline = "-------------------------------------------------------------------------------";
public static final ConcurrentMap<String, String> reverseMappingCache = new ConcurrentHashMap<String, String>();
private static volatile Switchboard switchboard = null;
private static String virtualHost = null;
private static volatile Switchboard switchboard;
private static String virtualHost;
public static boolean keepAliveSupport = false;
private static ConcurrentMap<String, Long> YaCyHopAccessRequester = new ConcurrentHashMap<String, Long>();
@ -166,17 +166,17 @@ public final class HTTPDemon implements serverHandler, Cloneable {
this.keepAliveRequestCount = 0;
}
private static boolean allowProxy(Session session) {
private static boolean allowProxy(final Session session) {
final String proxyClient = switchboard.getConfig("proxyClient", "*");
return (proxyClient.equals("*")) ? true : match(session.userAddress.getHostAddress(), proxyClient);
}
private static boolean allowServer(Session session) {
private static boolean allowServer(final Session session) {
final String serverClient = switchboard.getConfig("serverClient", "*");
return (serverClient.equals("*")) ? true : match(session.userAddress.getHostAddress(), serverClient);
}
private static boolean allowYaCyHop(Session session) {
private static boolean allowYaCyHop(final Session session) {
return switchboard.getConfigBool("YaCyHop", false);
}
@ -221,9 +221,10 @@ public final class HTTPDemon implements serverHandler, Cloneable {
/**
* This function is used to determine if a persistent connection was requested by the client.
* @param header the received http-headers
* @param prop
* @return <code>true</code> if a persistent connection was requested or <code>false</code> otherwise
*/
private boolean handlePersistentConnection(final RequestHeader header, Properties prop) {
private boolean handlePersistentConnection(final RequestHeader header, final Properties prop) {
if (!keepAliveSupport) {
prop.put(HeaderFramework.CONNECTION_PROP_PERSISTENT,"close");
@ -315,45 +316,44 @@ public final class HTTPDemon implements serverHandler, Cloneable {
private static long lastAccessDelta(final Map<String, Long> accessTable, final String domain) {
final Long lastAccess = accessTable.get(domain);
if (lastAccess == null) return Long.MAX_VALUE; // never accessed
return System.currentTimeMillis() - lastAccess.longValue();
return (lastAccess == null) ? Long.MAX_VALUE : System.currentTimeMillis() - lastAccess.longValue();
}
private boolean handleProxyAuthentication(final RequestHeader header, Properties prop, Session session) throws IOException {
private boolean handleProxyAuthentication(final RequestHeader header, final Properties prop, final Session session) throws IOException {
// getting the http version that is used by the client
final String httpVersion = prop.getProperty("HTTP", "HTTP/0.9");
// reading the authentication settings from switchboard
if (!this.proxyAccounts_init) {
this.use_proxyAccounts = switchboard.getConfigBool("use_proxyAccounts", false);
this.proxyAccounts_init = true; // is initialised
}
this.proxyAccounts_init = true; // is initialised
}
if (this.use_proxyAccounts) {
final String auth = header.get(RequestHeader.PROXY_AUTHORIZATION,"xxxxxx");
UserDB.Entry entry=switchboard.userDB.ipAuth(session.userAddress.getHostAddress());
if(entry == null){
entry=switchboard.userDB.proxyAuth(auth, session.userAddress.getHostAddress());
}
if(entry != null){
UserDB.Entry entry = switchboard.userDB.ipAuth(session.userAddress.getHostAddress());
if (entry == null) {
entry = switchboard.userDB.proxyAuth(auth, session.userAddress.getHostAddress());
}
if (entry != null) {
final int returncode=entry.surfRight();
if(returncode==UserDB.Entry.PROXY_ALLOK){
return true;
}
final serverObjects tp=new serverObjects();
if(returncode==UserDB.Entry.PROXY_TIMELIMIT_REACHED){
if (returncode == UserDB.Entry.PROXY_ALLOK) {
return true;
}
final serverObjects tp = new serverObjects();
if (returncode == UserDB.Entry.PROXY_TIMELIMIT_REACHED) {
tp.put("limit", "1");//time per day
tp.put("limit_timelimit", entry.getTimeLimit());
sendRespondError(prop, session.out, 403, "Internet-Timelimit reached", new File("proxymsg/proxylimits.inc"), tp, null);
}else if(returncode==UserDB.Entry.PROXY_NORIGHT){
} else if (returncode == UserDB.Entry.PROXY_NORIGHT){
tp.put("limit", "0");
sendRespondError(prop, session.out, 403, "Proxy use forbidden", new File("proxymsg/proxylimits.inc"), tp, null);
}
return false;
}
}
// ask for authenticate
session.out.write((httpVersion + " 407 Proxy Authentication Required" + serverCore.CRLF_STRING +
RequestHeader.PROXY_AUTHENTICATE + ": Basic realm=\"log-in\"" + serverCore.CRLF_STRING).getBytes());
RequestHeader.PROXY_AUTHENTICATE + ": Basic realm=\"log-in\"" + serverCore.CRLF_STRING).getBytes());
session.out.write((HeaderFramework.CONTENT_LENGTH + ": 0\r\n").getBytes());
session.out.write("\r\n".getBytes());
session.out.flush();
@ -363,12 +363,11 @@ public final class HTTPDemon implements serverHandler, Cloneable {
return true;
}
public Boolean EMPTY(final String arg, Session session) throws IOException {
if (++this.emptyRequestCount > 10) return serverCore.TERMINATE_CONNECTION;
return serverCore.RESUME_CONNECTION;
public Boolean EMPTY(final String arg, final Session session) throws IOException {
return (++this.emptyRequestCount > 10) ? serverCore.TERMINATE_CONNECTION : serverCore.RESUME_CONNECTION;
}
public Boolean UNKNOWN(final String requestLine, Session session) throws IOException {
public Boolean UNKNOWN(final String requestLine, final Session session) throws IOException {
Properties prop = parseRequestLine(HeaderFramework.METHOD_GET, requestLine, session);
int pos;
@ -387,16 +386,16 @@ public final class HTTPDemon implements serverHandler, Cloneable {
return serverCore.TERMINATE_CONNECTION;
}
public Boolean GET(final String arg, Session session) {
public Boolean GET(final String arg, final Session session) {
try {
// parsing the http request line
Properties prop = parseRequestLine(HeaderFramework.METHOD_GET, arg, session);
final Properties prop = parseRequestLine(HeaderFramework.METHOD_GET, arg, session);
// we now know the HTTP version. depending on that, we read the header
final String httpVersion = prop.getProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_0_9);
final RequestHeader header = (httpVersion.equals(HeaderFramework.HTTP_VERSION_0_9))
? new RequestHeader(reverseMappingCache)
: readHeader(prop, session);
final RequestHeader header = (httpVersion.equals(HeaderFramework.HTTP_VERSION_0_9))
? new RequestHeader(reverseMappingCache)
: readHeader(prop, session);
// handling transparent proxy support
handleTransparentProxySupport(header, prop, virtualHost, HTTPDProxyHandler.isTransparentProxy);
@ -432,7 +431,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
}
}
private void logUnexpectedError(final Exception e, String address) {
private void logUnexpectedError(final Exception e, final String address) {
if (e instanceof InterruptedException) {
log.logInfo("Interruption detected");
} else {
@ -454,15 +453,15 @@ public final class HTTPDemon implements serverHandler, Cloneable {
}
}
public Boolean HEAD(final String arg, Session session) {
public Boolean HEAD(final String arg, final Session session) {
try {
Properties prop = parseRequestLine(HeaderFramework.METHOD_HEAD, arg, session);
final Properties prop = parseRequestLine(HeaderFramework.METHOD_HEAD, arg, session);
// we now know the HTTP version. depending on that, we read the header
RequestHeader header;
final String httpVersion = prop.getProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_0_9);
if (httpVersion.equals(HeaderFramework.HTTP_VERSION_0_9)) header = new RequestHeader(reverseMappingCache);
else header = readHeader(prop,session);
final RequestHeader header = (httpVersion.equals(HeaderFramework.HTTP_VERSION_0_9))
? new RequestHeader(reverseMappingCache)
: readHeader(prop,session);
// handle transparent proxy support
handleTransparentProxySupport(header, prop, virtualHost, HTTPDProxyHandler.isTransparentProxy);
@ -500,15 +499,15 @@ public final class HTTPDemon implements serverHandler, Cloneable {
}
}
public Boolean POST(final String arg, Session session) {
public Boolean POST(final String arg, final Session session) {
try {
Properties prop = parseRequestLine(HeaderFramework.METHOD_POST, arg, session);
final Properties prop = parseRequestLine(HeaderFramework.METHOD_POST, arg, session);
// we now know the HTTP version. depending on that, we read the header
RequestHeader header;
final String httpVersion = prop.getProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_0_9);
if (httpVersion.equals(HeaderFramework.HTTP_VERSION_0_9)) header = new RequestHeader(reverseMappingCache);
else header = readHeader(prop, session);
final RequestHeader header = (httpVersion.equals(HeaderFramework.HTTP_VERSION_0_9))
? new RequestHeader(reverseMappingCache)
: readHeader(prop, session);
// handle transfer-coding
final InputStream sessionIn;
@ -582,16 +581,18 @@ public final class HTTPDemon implements serverHandler, Cloneable {
prop.setProperty(HeaderFramework.CONNECTION_PROP_HOST,(HTTPDemon.isThisHostName(dstHostSocket)?virtualHost:dstHostSocket));
}
public Boolean CONNECT(String arg, Session session) throws IOException {
public Boolean CONNECT(String arg, final Session session) throws IOException {
// establish a ssh-tunneled http connection
// this is to support https
// parse HTTP version
int pos = arg.indexOf(' ');
String httpVersion = "HTTP/1.0";
final String httpVersion;
if (pos >= 0) {
httpVersion = arg.substring(pos + 1);
arg = arg.substring(0, pos);
} else {
httpVersion = "HTTP/1.0";
}
Properties prop = new Properties();
prop.setProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, httpVersion);
@ -641,10 +642,10 @@ public final class HTTPDemon implements serverHandler, Cloneable {
return serverCore.TERMINATE_CONNECTION;
}
private final Properties parseRequestLine(final String cmd, final String s, Session session) {
private final Properties parseRequestLine(final String cmd, final String s, final Session session) {
// parsing the header
Properties p = parseRequestLine(cmd, s, virtualHost);
final Properties p = parseRequestLine(cmd, s, virtualHost);
// track the request
final String path = p.getProperty(HeaderFramework.CONNECTION_PROP_URL);
@ -788,7 +789,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
// replace all other
final CharArrayWriter b = new CharArrayWriter(s.length());
int end;
for (int i=0; i<s.length(); i++) {
for (int i = 0, len = s.length(); i < len; i++) {
if (s.charAt(i) == '&' && (end = s.indexOf(';', i + 1)) > i) {
if (s.charAt(i + 1) == '#') { // &#1234; symbols
b.write(Integer.parseInt(s.substring(i + 2, end)));
@ -818,7 +819,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
*/
@SuppressWarnings("unchecked")
public static Map<String, byte[]> parseMultipart(final RequestHeader header, final serverObjects args, final InputStream in) throws IOException {
//ByteArrayInputStream in = new ByteArrayInputStream(FileUtils.read(inx));
final InputStream body = prepareBody(header, in);
RequestContext request = new yacyContextRequest(header, body);
@ -834,20 +835,18 @@ public final class HTTPDemon implements serverHandler, Cloneable {
}
// parse data in memory
FileUpload upload = new FileUpload(diskFileItemFactory);
List<FileItem> items;
//long time = System.currentTimeMillis();
final FileUpload upload = new FileUpload(DISK_FILE_ITEM_FACTORY);
final List<FileItem> items;
try {
items = upload.parseRequest(request);
} catch (FileUploadException e) {
//Log.logException(e);
throw new IOException("FileUploadException " + e.getMessage());
}
//System.out.println("**** FileUploadBase.parseRequest time = " + (System.currentTimeMillis() - time));
// format information for further usage
final HashMap<String, byte[]> files = new HashMap<String, byte[]>();
for (FileItem item : items) {
final Map<String, byte[]> files = new HashMap<String, byte[]>();
for (final FileItem item : items) {
if (item.isFormField()) {
// simple text
if (item.getContentType() == null || !item.getContentType().contains("charset")) {
@ -915,7 +914,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
* @param header
* @param in
*/
public yacyContextRequest(Map<String, String> requestHeader, InputStream in) {
public yacyContextRequest(final Map<String, String> requestHeader, final InputStream in) {
super(null, requestHeader);
this.inStream = in;
}
@ -932,38 +931,25 @@ public final class HTTPDemon implements serverHandler, Cloneable {
}
/*
static int nextPos = -1;
private static byte[] readLine(final int start, final byte[] array) {
// read a string from an array; line ending is always CRLF
// but we are also fuzzy with that: may also be only CR or LF
// if no remaining CR, CRLF or LF can be found, return null
if (start > array.length) return null;
int pos = indexOf(start, array, serverCore.CRLF); nextPos = pos + 2;
if (pos < 0) {pos = indexOf(start, array, new byte[] {serverCore.CR}); nextPos = pos + 1;}
if (pos < 0) {pos = indexOf(start, array, new byte[] {serverCore.LF}); nextPos = pos + 1;}
if (pos < 0) {nextPos = start; return null;}
final byte[] result = new byte[pos - start];
java.lang.System.arraycopy(array, start, result, 0, pos - start);
return result;
}
*/
public static int indexOf(final int start, final byte[] array, final byte[] pattern) {
// return a position of a pattern in an array
if (start > array.length - pattern.length) return -1;
if (pattern.length == 0) return start;
for (int pos = start; pos <= array.length - pattern.length; pos++)
if ((array[pos] == pattern[0]) && (equals(array, pos, pattern, 0, pattern.length)))
for (int pos = start, lens = array.length - pattern.length; pos <= lens; pos++) {
if ((array[pos] == pattern[0]) && (equals(array, pos, pattern, 0, pattern.length))) {
return pos;
}
}
return -1;
}
public static boolean equals(final byte[] a, final int aoff, final byte[] b, final int boff, final int len) {
//System.out.println("equals: a = " + new String(a) + ", aoff = " + aoff + ", b = " + new String(b) + ", boff = " + boff + ", length = " + len);
if ((aoff + len > a.length) || (boff + len > b.length)) return false;
for (int i = 0; i < len; i++) if (a[aoff + i] != b[boff + i]) return false;
//System.out.println("TRUE!");
for (int i = 0; i < len; i++) {
if (a[aoff + i] != b[boff + i]) {
return false;
}
}
return true;
}
@ -980,7 +966,6 @@ public final class HTTPDemon implements serverHandler, Cloneable {
respond.flush();
}
public static final void sendRespondError(
final Properties conProp,
final OutputStream respond,
@ -1059,11 +1044,13 @@ public final class HTTPDemon implements serverHandler, Cloneable {
final String args = conProp.getProperty(HeaderFramework.CONNECTION_PROP_ARGS);
final String method = conProp.getProperty(HeaderFramework.CONNECTION_PROP_METHOD);
int port = 80;
final int port;
final int pos = host.indexOf(':');
if (pos != -1) {
port = Integer.parseInt(host.substring(pos + 1));
host = host.substring(0, pos);
} else {
port = 80;
}
String urlString;
@ -1102,7 +1089,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
tp.put("errorMessageType_file", (detailedErrorMsgFile == null) ? "" : detailedErrorMsgFile.toString());
if ((detailedErrorMsgValues != null) && !detailedErrorMsgValues.isEmpty()) {
// rewriting the value-names and add the proper name prefix:
for (Entry<String, String> entry: detailedErrorMsgValues.entrySet()) {
for (final Entry<String, String> entry: detailedErrorMsgValues.entrySet()) {
tp.put("errorMessageType_" + entry.getKey(), entry.getValue());
}
}
@ -1141,8 +1128,9 @@ public final class HTTPDemon implements serverHandler, Cloneable {
final byte[] result = o.toByteArray();
o.close(); o = null;
if(header == null)
if(header == null) {
header = new ResponseHeader();
}
header.put(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_STATUS, Integer.toString(httpStatusCode));
header.put(HeaderFramework.DATE, systemDate);
header.put(HeaderFramework.CONTENT_TYPE, "text/html");
@ -1184,23 +1172,23 @@ public final class HTTPDemon implements serverHandler, Cloneable {
}
if (!reqMethod.equals(HeaderFramework.METHOD_HEAD)){
if (!conProp.getProperty(HeaderFramework.CONNECTION_PROP_PERSISTENT,"close").equals("close")) {
if (transferEnc == null && contentLength < 0) {
throw new IllegalArgumentException("Message MUST contain a Content-Length or a non-identity transfer-coding header field.");
}
if (!conProp.getProperty(HeaderFramework.CONNECTION_PROP_PERSISTENT,"close").equals("close") &&
transferEnc == null && contentLength < 0) {
throw new IllegalArgumentException("Message MUST contain a Content-Length or a non-identity transfer-coding header field.");
}
if (transferEnc != null && contentLength >= 0) {
throw new IllegalArgumentException("Messages MUST NOT include both a Content-Length header field and a non-identity transfer-coding.");
}
}
if(headers==null) headers = new ResponseHeader();
if (headers == null) {
headers = new ResponseHeader();
}
final Date now = new Date(System.currentTimeMillis());
headers.put(HeaderFramework.SERVER, "AnomicHTTPD (www.anomic.de)");
headers.put(HeaderFramework.DATE, HeaderFramework.formatRFC1123(now));
if (moddate.after(now)) {
//System.out.println("*** DEBUG: correcting moddate = " + moddate.toString() + " to now = " + now.toString());
moddate = now;
}
headers.put(HeaderFramework.LAST_MODIFIED, HeaderFramework.formatRFC1123(moddate));
@ -1286,30 +1274,19 @@ public final class HTTPDemon implements serverHandler, Cloneable {
//responseHeader.put(HeaderFramework.X_YACY_PREVIOUS_REQUEST_LINE,conProp.getProperty(HeaderFramework.CONNECTION_PROP_PREV_REQUESTLINE));
//read custom headers
/*
if (requestProperties != null)
{
httpHeader outgoingHeader=requestProperties.getOutgoingHeader();
if (outgoingHeader!=null)
{*/
final Iterator<ResponseHeader.Entry> it = responseHeader.getAdditionalHeaderProperties().iterator();
while(it.hasNext()) {
//Append user properties to the main String
//TODO: Should we check for user properites. What if they intersect properties that are already in header?
final ResponseHeader.Entry e = it.next();
header.append(e.getKey()).append(": ").append(e.getValue()).append("\r\n");
}
/*
}
}*/
final Iterator<ResponseHeader.Entry> it = responseHeader.getAdditionalHeaderProperties().iterator();
while(it.hasNext()) {
//Append user properties to the main String
//TODO: Should we check for user properites. What if they intersect properties that are already in header?
final ResponseHeader.Entry e = it.next();
header.append(e.getKey()).append(": ").append(e.getValue()).append("\r\n");
}
// write header
final Iterator<String> i = responseHeader.keySet().iterator();
String key;
char tag;
int count;
//System.out.println("vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv");
while (i.hasNext()) {
key = i.next();
tag = key.charAt(0);
@ -1318,7 +1295,6 @@ public final class HTTPDemon implements serverHandler, Cloneable {
for (int j = 0; j < count; j++) {
header.append(key).append(": ").append(responseHeader.getSingle(key, j)).append("\r\n");
}
//System.out.println("#" + key + ": " + value);
}
}
@ -1386,10 +1362,10 @@ public final class HTTPDemon implements serverHandler, Cloneable {
// check if the destination port is equal to the port yacy is listening to
dstPort.equals(Integer.valueOf(serverCore.getPortNr(switchboard.getConfig("port", "8090")))) &&
(
// check if the destination host is our local IP address
Domains.isThisHostIP(dstHost) ||
// check if the destination host is our seed ip address
isThisSeedIP(dstHost)
// check if the destination host is our local IP address
Domains.isThisHostIP(dstHost) ||
// check if the destination host is our seed ip address
isThisSeedIP(dstHost)
)
) {
return true;
@ -1401,7 +1377,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
/**
* @param alternativeResolver the alternativeResolver to set
*/
public static void setAlternativeResolver(AlternativeDomainNames alternativeResolver) {
public static void setAlternativeResolver(final AlternativeDomainNames alternativeResolver) {
HTTPDemon.alternativeResolver = alternativeResolver;
}
@ -1462,7 +1438,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
prop.setProperty(HeaderFramework.CONNECTION_PROP_REQUESTLINE, cmd + " " + args);
// this parses a whole URL
if (args.length() == 0) {
if (args.isEmpty()) {
prop.setProperty(HeaderFramework.CONNECTION_PROP_HOST, virtualHost);
prop.setProperty(HeaderFramework.CONNECTION_PROP_PATH, "/");
prop.setProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_0_9);
@ -1472,7 +1448,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
// store the version propery "HTTP" and cut the query at both ends
int sep = args.lastIndexOf(' ');
if ((sep >= 0)&&(args.substring(sep + 1).toLowerCase().startsWith("http/"))) {
if ((sep >= 0) && (args.substring(sep + 1).toLowerCase().startsWith("http/"))) {
// HTTP version is given
prop.setProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, args.substring(sep + 1).trim());
args = args.substring(0, sep).trim(); // cut off HTTP version mark
@ -1496,19 +1472,22 @@ public final class HTTPDemon implements serverHandler, Cloneable {
// properties of the query are stored with the prefix "&"
// additionally, the values URL and ARGC are computed
String argsString = "";
final String argsString;
sep = args.indexOf('?');
if (sep >= 0) {
// there are values attached to the query string
argsString = args.substring(sep + 1); // cut head from tail of query
args = args.substring(0, sep);
} else {
argsString = "";
}
prop.setProperty(HeaderFramework.CONNECTION_PROP_URL, args); // store URL
//System.out.println("HTTPD: ARGS=" + argsString);
if (argsString.length() != 0) prop.setProperty(HeaderFramework.CONNECTION_PROP_ARGS, argsString); // store arguments in original form
if (!argsString.isEmpty()) {
prop.setProperty(HeaderFramework.CONNECTION_PROP_ARGS, argsString);
} // store arguments in original form
// finally find host string
String path;
final String path;
if (args.toUpperCase().startsWith("HTTP://")) {
// a host was given. extract it and set path
args = args.substring(7);
@ -1547,7 +1526,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
prop.setProperty(HeaderFramework.CONNECTION_PROP_PATH, path);
// find out file extension (we already stripped ?-parameters from args)
String ext = ""; // default when no file extension
final String ext;
sep = path.lastIndexOf('.');
if (sep >= 0) {
final int ancpos = path.indexOf("#", sep + 1);
@ -1558,6 +1537,8 @@ public final class HTTPDemon implements serverHandler, Cloneable {
// ex: /foo/bar.php => php
ext = path.substring(sep + 1).toLowerCase();
}
} else {
ext = ""; // default when no file extension
}
prop.setProperty(HeaderFramework.CONNECTION_PROP_EXT, ext);

155
source/net/yacy/cora/protocol/Domains.java
Unescape View File

@ -25,6 +25,7 @@ import java.net.NetworkInterface;
import java.net.SocketException;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Enumeration;
@ -42,18 +43,18 @@ import net.yacy.cora.storage.ConcurrentARC;
public class Domains {
private static final String PRESENT = "";
private static final String localPatterns = "10\\..*,127.*,172.(1[6-9]|2[0-9]|3[0-1])\\..*,169.254.*,192.168.*,localhost";
private static final int maxNameCacheHitSize = 20000;
private static final int maxNameCacheMissSize = 20000;
private static final int maxNameNoCachingListSize = 20000;
private static final int concurrencyLevel = Runtime.getRuntime().availableProcessors() + 1;
private static final String LOCAL_PATTERNS = "10\\..*,127\\..*,172\\.(1[6-9]|2[0-9]|3[0-1])\\..*,169\\.254\\..*,192\\.168\\..*,localhost";
private static final int MAX_NAME_CACHE_HIT_SIZE = 20000;
private static final int MAX_NAME_CACHE_MISS_SIZE = 20000;
private static final int MAX_NAME_NO_CACHING_LIST_SIZE = 20000;
private static final int CONCURRENCY_LEVEL = Runtime.getRuntime().availableProcessors() + 1;
// a dns cache
private static final ARC<String, InetAddress> nameCacheHit = new ConcurrentARC<String, InetAddress>(maxNameCacheHitSize, concurrencyLevel);
private static final ARC<String, String> nameCacheMiss = new ConcurrentARC<String, String>(maxNameCacheMissSize, concurrencyLevel);
private static final ARC<String, String> nameCacheNoCachingList = new ConcurrentARC<String, String>(maxNameNoCachingListSize, concurrencyLevel);
private static final ARC<String, InetAddress> NAME_CACHE_HIT = new ConcurrentARC<String, InetAddress>(MAX_NAME_CACHE_HIT_SIZE, CONCURRENCY_LEVEL);
private static final ARC<String, String> NAME_CACHE_MISS = new ConcurrentARC<String, String>(MAX_NAME_CACHE_MISS_SIZE, CONCURRENCY_LEVEL);
private static final ARC<String, String> NAME_CACHE_NO_CACHING_LIST = new ConcurrentARC<String, String>(MAX_NAME_NO_CACHING_LIST_SIZE, CONCURRENCY_LEVEL);
public static List<Pattern> nameCacheNoCachingPatterns = Collections.synchronizedList(new LinkedList<Pattern>());
public static final List<Pattern> localhostPatterns = makePatterns(localPatterns);
public static final List<Pattern> LOCALHOST_PATTERNS = makePatterns(LOCAL_PATTERNS);
/**
* ! ! ! A T T E N T I O N A T T E N T I O N A T T E N T I O N ! ! !
@ -61,7 +62,7 @@ public class Domains {
* Do not move a TLD to another group (if you do not exactly know what you
* are doing)! Because it will change the hash of the url!
*/
private static final String[] TLD_NorthAmericaOceania={
private static final String[] TLD_NorthAmericaOceania = {
// primary english-speaking countries
// english-speaking countries from central america are also included
// includes also dutch and french colonies in the caribbean sea
@ -383,10 +384,10 @@ public class Domains {
String tld;
//String name;
final Integer ID = Integer.valueOf(id);
for (int i = 0; i < TLDList.length; i++) {
p = TLDList[i].indexOf('=');
for (final String TLDelement : TLDList) {
p = TLDelement.indexOf('=');
if (p > 0) {
tld = TLDList[i].substring(0, p).toLowerCase();
tld = TLDelement.substring(0, p).toLowerCase();
//name = TLDList[i].substring(p + 1);
TLDID.put(tld, ID);
//TLDName.put(tld, name);
@ -425,46 +426,46 @@ public class Domains {
* @return String with the ip. null, if the host could not be resolved.
*/
public static InetAddress dnsResolveFromCache(String host) throws UnknownHostException {
if ((host == null) || (host.length() == 0)) return null;
host = host.toLowerCase().trim();
if ((host == null) || host.isEmpty()) return null;
host = host.toLowerCase().trim();
// try to simply parse the address
InetAddress ip = parseInetAddress(host);
if (ip != null) return ip;
// trying to resolve host by doing a name cache lookup
ip = nameCacheHit.get(host);
ip = NAME_CACHE_HIT.get(host);
if (ip != null) return ip;
if (nameCacheMiss.containsKey(host)) return null;
if (NAME_CACHE_MISS.containsKey(host)) return null;
throw new UnknownHostException("host not in cache");
}
public static void setNoCachingPatterns(String patternList) {
public static void setNoCachingPatterns(final String patternList) {
nameCacheNoCachingPatterns = makePatterns(patternList);
}
public static List<Pattern> makePatterns(String patternList) {
final String[] entries = patternList.split(",");
public static List<Pattern> makePatterns(final String patternList) {
final String[] entries = (patternList != null) ? patternList.split(",") : new String[0];
final List<Pattern> patterns = new ArrayList<Pattern>(entries.length);
for (int i = 0; i < entries.length; i++) {
patterns.add(Pattern.compile(entries[i].trim()));
for (final String entry : entries) {
patterns.add(Pattern.compile(entry.trim()));
}
return patterns;
}
public static boolean matchesList(String obj, List<Pattern> patterns) {
for (Pattern nextPattern: patterns) {
public static boolean matchesList(final String obj, final List<Pattern> patterns) {
for (final Pattern nextPattern: patterns) {
if (nextPattern.matcher(obj).matches()) return true;
}
return false;
}
public static String getHostName(final InetAddress i) {
Collection<String> hosts = nameCacheHit.getKeys(i);
if (hosts.size() > 0) return hosts.iterator().next();
String host = i.getHostName();
nameCacheHit.put(host, i);
final Collection<String> hosts = NAME_CACHE_HIT.getKeys(i);
if (!hosts.isEmpty()) return hosts.iterator().next();
final String host = i.getHostName();
NAME_CACHE_HIT.put(host, i);
return host;
/*
// call i.getHostName() using concurrency to interrupt execution in case of a time-out
@ -478,16 +479,16 @@ public class Domains {
public static InetAddress dnsResolve(String host) {
if ((host == null) || (host.length() == 0)) return null;
host = host.toLowerCase().trim();
host = host.toLowerCase().trim();
// try to simply parse the address
InetAddress ip = parseInetAddress(host);
if (ip != null) return ip;
// try to resolve host by doing a name cache lookup
ip = nameCacheHit.get(host);
ip = NAME_CACHE_HIT.get(host);
if (ip != null) return ip;
if (nameCacheMiss.containsKey(host)) return null;
if (NAME_CACHE_MISS.containsKey(host)) return null;
// call dnsResolveNetBased(host) using concurrency to interrupt execution in case of a time-out
try {
@ -495,12 +496,12 @@ public class Domains {
ip = InetAddress.getByName(host); //TimeoutRequest.getByName(host, 1000); // this makes the DNS request to backbone
if ((ip == null) ||
(ip.isLoopbackAddress()) ||
(nameCacheNoCachingList.containsKey(host))
(NAME_CACHE_NO_CACHING_LIST.containsKey(host))
) {
doCaching = false;
} else {
if (matchesList(host, nameCacheNoCachingPatterns)) {
nameCacheNoCachingList.put(host, PRESENT);
NAME_CACHE_NO_CACHING_LIST.put(host, PRESENT);
doCaching = false;
}
}
@ -508,7 +509,7 @@ public class Domains {
if (doCaching && ip != null) {
// add new entries
nameCacheHit.put(host, ip);
NAME_CACHE_HIT.put(host, ip);
}
return ip;
} catch (final UnknownHostException e) {
@ -516,14 +517,13 @@ public class Domains {
flushMissNameCache();
// add new entries
nameCacheMiss.put(host, PRESENT);
NAME_CACHE_MISS.put(host, PRESENT);
}
return null;
}
private static final InetAddress parseInetAddress(final String ip) {
if (ip == null) return null;
if (ip.length() < 8) return null;
if (ip == null || ip.length() < 8) return null;
final String[] ips = ip.split("\\.");
if (ips.length != 4) return null;
final byte[] ipb = new byte[4];
@ -548,11 +548,11 @@ public class Domains {
* @return int The number of entries in the nameCacheHit map
*/
public static int nameCacheHitSize() {
return nameCacheHit.size();
return NAME_CACHE_HIT.size();
}
public static int nameCacheMissSize() {
return nameCacheMiss.size();
return NAME_CACHE_MISS.size();
}
/**
@ -561,14 +561,14 @@ public class Domains {
* @return int The number of entries in the nameCacheNoCachingList list
*/
public static int nameCacheNoCachingListSize() {
return nameCacheNoCachingList.size();
return NAME_CACHE_NO_CACHING_LIST.size();
}
/**
* Removes old entries from the dns miss cache
*/
public static void flushMissNameCache() {
if (nameCacheMiss.size() > maxNameCacheMissSize) nameCacheMiss.clear();
if (NAME_CACHE_MISS.size() > MAX_NAME_CACHE_MISS_SIZE) NAME_CACHE_MISS.clear();
}
private static String localHostName = "127.0.0.1";
@ -580,8 +580,8 @@ public class Domains {
if (localHostAddress != null) localHostAddresses.add(localHostAddress);
} catch (UnknownHostException e) {}
try {
InetAddress[] moreAddresses = InetAddress.getAllByName(localHostName);
if (moreAddresses != null) for (InetAddress a: moreAddresses) localHostAddresses.add(a);
final InetAddress[] moreAddresses = InetAddress.getAllByName(localHostName);
if (moreAddresses != null) localHostAddresses.addAll(Arrays.asList(moreAddresses));
} catch (UnknownHostException e) {}
// to get the local host name, a dns lookup is necessary.
@ -589,15 +589,16 @@ public class Domains {
// therefore we start the host name lookup as concurrent thread
// meanwhile the host name is "127.0.0.1" which is not completely wrong
new Thread() {
@Override
public void run() {
// try to get local addresses from interfaces
try {
Enumeration<NetworkInterface> nis = NetworkInterface.getNetworkInterfaces();
final Enumeration<NetworkInterface> nis = NetworkInterface.getNetworkInterfaces();
while (nis.hasMoreElements()) {
NetworkInterface ni = nis.nextElement();
Enumeration<InetAddress> addrs = ni.getInetAddresses();
final NetworkInterface ni = nis.nextElement();
final Enumeration<InetAddress> addrs = ni.getInetAddresses();
while (addrs.hasMoreElements()) {
InetAddress addr = addrs.nextElement();
final InetAddress addr = addrs.nextElement();
if (addr != null) localHostAddresses.add(addr);
}
}
@ -612,14 +613,14 @@ public class Domains {
// after the host name was resolved, we try to look up more local addresses
// using the host name:
try {
InetAddress[] moreAddresses = InetAddress.getAllByName(localHostName);
if (moreAddresses != null) for (InetAddress a: moreAddresses) localHostAddresses.add(a);
final InetAddress[] moreAddresses = InetAddress.getAllByName(localHostName);
if (moreAddresses != null) localHostAddresses.addAll(Arrays.asList(moreAddresses));
} catch (UnknownHostException e) {
}
// fill a cache of local host names
for (InetAddress a: localHostAddresses) {
String hostname = getHostName(a);
for (final InetAddress a: localHostAddresses) {
final String hostname = getHostName(a);
if (hostname != null) {
localHostNames.add(hostname);
localHostNames.add(a.getHostAddress());
@ -632,7 +633,7 @@ public class Domains {
public static InetAddress myPublicLocalIP() {
// list all addresses
// for (int i = 0; i < localHostAddresses.length; i++) System.out.println("IP: " + localHostAddresses[i].getHostAddress()); // DEBUG
if (localHostAddresses.size() == 0) {
if (localHostAddresses.isEmpty()) {
return null;
}
if (localHostAddresses.size() == 1) {
@ -641,7 +642,7 @@ public class Domains {
}
// we have more addresses, find an address that is not local
int b0, b1;
for (InetAddress a: localHostAddresses) {
for (final InetAddress a: localHostAddresses) {
b0 = 0Xff & a.getAddress()[0];
b1 = 0Xff & a.getAddress()[1];
if (b0 != 10 && // class A reserved
@ -655,26 +656,26 @@ public class Domains {
// return that one that is returned with InetAddress.getLocalHost()
// if appropriate
try {
InetAddress localHostAddress = InetAddress.getLocalHost();
final InetAddress localHostAddress = InetAddress.getLocalHost();
if (localHostAddress != null &&
(0Xff & localHostAddress.getAddress()[0]) != 127 &&
localHostAddress.getHostAddress().indexOf(":") < 0) return localHostAddress;
} catch (UnknownHostException e) {
}
// we filter out the loopback address 127.0.0.1 and all addresses without a name
for (InetAddress a: localHostAddresses) {
for (final InetAddress a: localHostAddresses) {
if ((0Xff & a.getAddress()[0]) != 127 &&
a.getHostAddress().indexOf(":") < 0 &&
a.getHostName() != null &&
a.getHostName().length() > 0) return a;
!a.getHostName().isEmpty()) return a;
}
// if no address has a name, then take any other than the loopback
for (InetAddress a: localHostAddresses) {
for (final InetAddress a: localHostAddresses) {
if ((0Xff & a.getAddress()[0]) != 127 &&
a.getHostAddress().indexOf(":") < 0) return a;
}
// if all fails, give back whatever we have
for (InetAddress a: localHostAddresses) {
for (final InetAddress a: localHostAddresses) {
if (a.getHostAddress().indexOf(":") < 0) return a;
}
// finally, just get any
@ -688,11 +689,11 @@ public class Domains {
public static Set<InetAddress> myIntranetIPs() {
// list all local addresses
if (localHostAddresses.size() < 1) try {Thread.sleep(1000);} catch (InterruptedException e) {}
Set<InetAddress> list = new HashSet<InetAddress>();
if (localHostAddresses.size() == 0) return list; // give up
for (InetAddress a: localHostAddresses) {
if ((0Xff & a.getAddress()[0]) == 127) continue;
if (!matchesList(a.getHostAddress(), localhostPatterns)) continue;
final Set<InetAddress> list = new HashSet<InetAddress>();
if (localHostAddresses.isEmpty()) return list; // give up
for (final InetAddress a: localHostAddresses) {
if (((0Xff & a.getAddress()[0]) == 127) ||
(!matchesList(a.getHostAddress(), LOCALHOST_PATTERNS))) continue;
list.add(a);
}
return list;
@ -706,7 +707,7 @@ public class Domains {
final InetAddress clientAddress = Domains.dnsResolve(hostName);
if (clientAddress == null) return false;
if (clientAddress.isAnyLocalAddress() || clientAddress.isLoopbackAddress()) return true;
for (InetAddress a: localHostAddresses) {
for (final InetAddress a: localHostAddresses) {
if (a.equals(clientAddress)) {
isThisHostIP = true;
break;
@ -723,7 +724,7 @@ public class Domains {
try {
if (clientAddress.isAnyLocalAddress() || clientAddress.isLoopbackAddress()) return true;
for (InetAddress a: localHostAddresses) {
for (final InetAddress a: localHostAddresses) {
if (a.equals(clientAddress)) {
isThisHostIP = true;
break;
@ -734,20 +735,18 @@ public class Domains {
}
public static int getDomainID(final String host) {
if (host == null || host.length() == 0) return TLD_Local_ID;
if (isLocal(host)) return TLD_Local_ID;
if (host == null || host.isEmpty() || isLocal(host)) return TLD_Local_ID;
final int p = host.lastIndexOf('.');
String tld = (p > 0) ? host.substring(p + 1) : "";
final String tld = (p > 0) ? host.substring(p + 1) : "";
final Integer i = TLDID.get(tld);
if (i == null) return TLD_Generic_ID;
return i.intValue();
return (i == null) ? TLD_Generic_ID : i.intValue();
}
public static boolean isLocalhost(final String host) {
if (host.equals("127.0.0.1")) return true;
if (host.equals("localhost")) return true;
if (host.startsWith("0:0:0:0:0:0:0:1")) return true;
return false;
return ("127.0.0.1".equals(host) ||
"localhost".equals(host) ||
host.startsWith("0:0:0:0:0:0:0:1")
);
}
public static boolean isLocal(final String host) {
@ -759,7 +758,7 @@ public class Domains {
// FIXME IPv4 only
// check local ip addresses
if (matchesList(host, localhostPatterns)) return true;
if (matchesList(host, LOCALHOST_PATTERNS)) return true;
if (host.startsWith("0:0:0:0:0:0:0:1")) return true;
// check if there are other local IP addresses that are not in
@ -775,7 +774,7 @@ public class Domains {
// check dns lookup: may be a local address even if the domain name looks global
if (!recursive) return false;
InetAddress a = dnsResolve(host);
final InetAddress a = dnsResolve(host);
/*
if (a == null) {
// unknown if this is a local address. Could also be a timeout.
@ -786,7 +785,7 @@ public class Domains {
return a == null || a.isAnyLocalAddress() || a.isLinkLocalAddress() || a.isLoopbackAddress() || a.isSiteLocalAddress() || isLocal(a.getHostAddress(), false);
}
public static void main(String[] args) {
public static void main(final String[] args) {
/*
try {
Enumeration<NetworkInterface> nis = NetworkInterface.getNetworkInterfaces();
@ -804,7 +803,7 @@ public class Domains {
*/
try { Thread.sleep(1000);} catch (InterruptedException e) {} // get time for class init
System.out.println("myPublicLocalIP: " + myPublicLocalIP());
for (InetAddress a : myIntranetIPs()) {
for (final InetAddress a : myIntranetIPs()) {
System.out.println("Intranet IP: " + a);
}
}

Write Preview
Loading…
Cancel
Save