diff --git a/addon/YaCy.app/Contents/Info.plist b/addon/YaCy.app/Contents/Info.plist index 54931fed0..a7fee64bc 100644 --- a/addon/YaCy.app/Contents/Info.plist +++ b/addon/YaCy.app/Contents/Info.plist @@ -25,7 +25,7 @@ Java VMOptions - -Xmx600m -Xms600m -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8 -Dsolr.directoryFactory=solr.MMapDirectoryFactory + -Xmx600m -Xms600m -Dfile.encoding=UTF-8 -Dsolr.directoryFactory=solr.MMapDirectoryFactory WorkingDirectory $APP_PACKAGE/Contents/Resources/Java MainClass diff --git a/addon/yacyInit.m4 b/addon/yacyInit.m4 index 333028cd2..b165f36ea 100644 --- a/addon/yacyInit.m4 +++ b/addon/yacyInit.m4 @@ -57,7 +57,7 @@ SHUTDOWN_TIMEOUT=50 # Default niceness if not set in config file NICE_VAL=0 -JAVA_ARGS="-server -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Dsolr.directoryFactory=solr.MMapDirectoryFactory" +JAVA_ARGS="-server -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Dsolr.directoryFactory=solr.MMapDirectoryFactory" ifdef(`openSUSE', `dnl . /etc/rc.status diff --git a/htroot/Blog.java b/htroot/Blog.java index 1a2ab6eea..7a5a59a08 100644 --- a/htroot/Blog.java +++ b/htroot/Blog.java @@ -192,7 +192,7 @@ public class Blog { prop.putHTML("mode_author", UTF8.String(author)); prop.putHTML("mode_subject", post.get("subject","")); prop.put("mode_date", dateString(new Date())); - prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_page", post.get("content", "")); + prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_page", post.get("content", "")); prop.putHTML("mode_page-code", post.get("content", "")); } else { @@ -327,7 +327,7 @@ public class Blog { prop.put("mode_entries_" + number + "_page", entry.getPage()); prop.put("mode_entries_" + number + "_timestamp", entry.getTimestamp()); } else { - prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_entries_" + number + "_page", entry.getPage()); + prop.putWiki(sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()), "mode_entries_" + number + "_page", entry.getPage()); } if (hasRights) { diff --git a/htroot/BlogComments.java b/htroot/BlogComments.java index 51849b73f..06a2d3508 100644 --- a/htroot/BlogComments.java +++ b/htroot/BlogComments.java @@ -175,7 +175,7 @@ public class BlogComments { prop.putHTML("mode_allow_author", UTF8.String(author)); prop.putHTML("mode_subject", post.get("subject","")); prop.put("mode_date", dateString(new Date())); - prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_page", post.get("content", "")); + prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_page", post.get("content", "")); prop.put("mode_page-code", post.get("content", "")); } else { // show blog-entry/entries @@ -191,7 +191,7 @@ public class BlogComments { prop.putHTML("mode_allow_author", UTF8.String(author)); prop.put("mode_comments", page.getCommentsSize()); prop.put("mode_date", dateString(page.getDate())); - prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_page", page.getPage()); + prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_page", page.getPage()); if (hasRights) { prop.put("mode_admin", "1"); prop.put("mode_admin_pageid", page.getKey()); @@ -234,7 +234,7 @@ public class BlogComments { if (!xml) { prop.putHTML("mode_entries_"+count+"_subject", UTF8.String(entry.getSubject())); prop.putHTML("mode_entries_"+count+"_author", UTF8.String(entry.getAuthor())); - prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_entries_"+count+"_page", entry.getPage()); + prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_entries_"+count+"_page", entry.getPage()); } else { prop.putHTML("mode_entries_"+count+"_subject", UTF8.String(entry.getSubject())); prop.putHTML("mode_entries_"+count+"_author", UTF8.String(entry.getAuthor())); diff --git a/htroot/ConfigNetwork_p.java b/htroot/ConfigNetwork_p.java index 7cebf0e79..2ca5a194b 100644 --- a/htroot/ConfigNetwork_p.java +++ b/htroot/ConfigNetwork_p.java @@ -174,10 +174,10 @@ public class ConfigNetwork_p prop.putHTML("cluster.peers.ipport", sb.getConfig("cluster.peers.ipport", "")); prop.putHTML("cluster.peers.yacydomain", sb.getConfig("cluster.peers.yacydomain", "")); StringBuilder hashes = new StringBuilder(); - for ( final byte[] h : sb.clusterhashes.keySet() ) { + for (final byte[] h: sb.clusterhashes) { hashes.append(", ").append(ASCII.String(h)); } - if ( hashes.length() > 2 ) { + if (hashes.length() > 2) { hashes = hashes.delete(0, 2); } diff --git a/htroot/CrawlStartScanner_p.java b/htroot/CrawlStartScanner_p.java index 09072404a..a8651b240 100644 --- a/htroot/CrawlStartScanner_p.java +++ b/htroot/CrawlStartScanner_p.java @@ -97,7 +97,7 @@ public class CrawlStartScanner_p } else { ip = Domains.myPublicLocalIP(); if ( Domains.isThisHostIP(ip) ) { - ip = sb.peers.mySeed().getInetAddress(); + ip = Domains.dnsResolve(sb.peers.mySeed().getIP()); } } if ( ip != null ) { diff --git a/htroot/MessageSend_p.java b/htroot/MessageSend_p.java index f192dd59e..443b8b88c 100644 --- a/htroot/MessageSend_p.java +++ b/htroot/MessageSend_p.java @@ -106,7 +106,7 @@ public class MessageSend_p { prop.putXML("mode_permission_message", message); prop.putHTML("mode_permission_hash", hash); if (post.containsKey("preview")) { - prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_permission_previewmessage", message); + prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_permission_previewmessage", message); } diff --git a/htroot/Messages_p.java b/htroot/Messages_p.java index 45c59a749..ac05dc53f 100644 --- a/htroot/Messages_p.java +++ b/htroot/Messages_p.java @@ -162,7 +162,7 @@ public class Messages_p { prop.putXML("mode_subject", message.subject()); String theMessage = null; theMessage = UTF8.String(message.message()); - prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_message", theMessage); + prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_message", theMessage); prop.put("mode_hash", message.authorHash()); prop.putXML("mode_key", key); } diff --git a/htroot/Network.html b/htroot/Network.html index f75b1a760..a96c2888c 100644 --- a/htroot/Network.html +++ b/htroot/Network.html @@ -162,7 +162,7 @@ document.getElementById("apilink").setAttribute("href", "Network.xml?" + window. #[rU]# #(complete)# :: - http://#[ip]#:#[port]# + http://#[ip]#:#[port]# #[ips]# #[hash]# #[age]# #[seeds]# diff --git a/htroot/Network.java b/htroot/Network.java index fed15aab5..8b070dd22 100644 --- a/htroot/Network.java +++ b/htroot/Network.java @@ -191,7 +191,7 @@ public class Network { if (sb.peers.mySeed() != null) { prop.put("table_my-hash", sb.peers.mySeed().hash ); - prop.put("table_my-ip", sb.peers.mySeed().getIP() ); + prop.put("table_my-ip", sb.peers.mySeed().getIPs().toString()); prop.put("table_my-port", sb.peers.mySeed().getPort() ); } @@ -210,11 +210,11 @@ public class Network { Seed peer = new Seed(post.get("peerHash"), map); sb.updateMySeed(); - final int added = Protocol.hello(sb.peers.mySeed(), sb.peers.peerActions, peer.getPublicAddress(), peer.hash, peer.getName()); + final int added = Protocol.hello(sb.peers.mySeed(), sb.peers.peerActions, peer); if (added <= 0) { prop.put("table_comment",1); - prop.putHTML("table_comment_status","publish: disconnected peer '" + peer.getName() + "/" + post.get("peerHash") + "' from " + peer.getPublicAddress()); + prop.putHTML("table_comment_status","publish: disconnected peer '" + peer.getName() + "/" + post.get("peerHash") + "' from " + peer.getIPs()); } else { peer = sb.peers.getConnected(peer.hash); if (peer == null) { @@ -222,7 +222,7 @@ public class Network { prop.putHTML("table_comment_status","publish: disconnected peer 'UNKNOWN/" + post.get("peerHash") + "' from UNKNOWN"); } else { prop.put("table_comment",2); - prop.putHTML("table_comment_status","publish: handshaked " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + peer.getName() + "' at " + peer.getPublicAddress()); + prop.putHTML("table_comment_status","publish: handshaked " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + peer.getName() + "' at " + peer.getIPs()); prop.putHTML("table_comment_details",peer.toString()); } } @@ -397,7 +397,8 @@ public class Network { prop.putHTML(STR_TABLE_LIST + conCount + "_location", location); if (complete) { prop.put(STR_TABLE_LIST + conCount + "_complete", 1); - prop.put(STR_TABLE_LIST + conCount + "_complete_ip", seed.getIP() ); + prop.putHTML(STR_TABLE_LIST + conCount + "_complete_ip", seed.getIP() ); + prop.putHTML(STR_TABLE_LIST + conCount + "_complete_ips", seed.getIPs().toString() ); prop.put(STR_TABLE_LIST + conCount + "_complete_port", seed.get(Seed.PORT, "-") ); prop.put(STR_TABLE_LIST + conCount + "_complete_hash", seed.hash); prop.put(STR_TABLE_LIST + conCount + "_complete_age", seed.getAge()); @@ -447,7 +448,7 @@ public class Network { } } prop.put(STR_TABLE_LIST + conCount + "_nodestate", seed.getFlagRootNode() ? 1 : 0); - prop.put(STR_TABLE_LIST + conCount + "_nodestate_ip", seed.getIP() ); + prop.put(STR_TABLE_LIST + conCount + "_nodestate_ip", seed.getIP()); prop.put(STR_TABLE_LIST + conCount + "_nodestate_port", seed.get(Seed.PORT, "-") ); if (seed.getFlagAcceptRemoteIndex()) { prop.put(STR_TABLE_LIST + conCount + "_dhtreceive_peertags", ""); diff --git a/htroot/SettingsAck_p.java b/htroot/SettingsAck_p.java index fc72eea58..3251b384d 100644 --- a/htroot/SettingsAck_p.java +++ b/htroot/SettingsAck_p.java @@ -177,7 +177,7 @@ public class SettingsAck_p { } else if (staticIP.startsWith("https://")) { if (staticIP.length() > 8) { staticIP = staticIP.substring(8); } else { staticIP = ""; } } - String error = Seed.isProperIP(staticIP); + String error = Seed.isProperIP(staticIP) ? null : "ip not proper: " + staticIP; if (error == null) { serverCore.useStaticIP = true; sb.peers.mySeed().setIP(staticIP); diff --git a/htroot/Status.java b/htroot/Status.java index 3d43644df..42a5d7a47 100644 --- a/htroot/Status.java +++ b/htroot/Status.java @@ -27,8 +27,8 @@ // javac -classpath .:../Classes Status.java // if the shell's current path is HTROOT -import java.net.InetAddress; import java.util.Date; +import java.util.Set; import net.yacy.cora.protocol.ConnectionInfo; import net.yacy.cora.protocol.Domains; @@ -189,8 +189,9 @@ public class Status } else { prop.put("extPortFormat", "0"); } - final InetAddress hostIP = Domains.myPublicLocalIP(); - prop.put("host", hostIP != null ? hostIP.getHostAddress() : "Unkown IP"); + + Set ips = Domains.myPublicIPs(); + prop.put("host", ips.toString()); // ssl support prop.put("sslSupport", sb.getConfig("keyStore", "").isEmpty() || !sb.getConfigBool("server.https", false) ? 0 : 1); @@ -227,7 +228,7 @@ public class Status prop.putNum("peerStatistics_disconnects", sb.peers.peerActions.disconnects); prop.put("peerStatistics_connects", Formatter.number(sb.peers.mySeed().get(Seed.CCOUNT, "0"))); thisHash = sb.peers.mySeed().hash; - if ( sb.peers.mySeed().getPublicAddress() == null ) { + if ( sb.peers.mySeed().getIPs().size() == 0 ) { prop.put("peerAddress", "0"); // not assigned + instructions prop.put("warningGoOnline", "1"); } else { diff --git a/htroot/Surftips.java b/htroot/Surftips.java index 0926a8728..dad98d129 100644 --- a/htroot/Surftips.java +++ b/htroot/Surftips.java @@ -277,7 +277,7 @@ public class Surftips { Seed seed = sb.peers.getConnected(record.originator()); if (seed == null) seed = sb.peers.getDisconnected(record.originator()); if (seed != null) { - url = "http://" + seed.getPublicAddress() + "/Wiki.html?page=" + record.attribute("page", ""); + url = "http://" + seed.getPublicAddress(seed.getIP()) + "/Wiki.html?page=" + record.attribute("page", ""); entry = rowdef.newEntry(new byte[][]{ UTF8.getBytes(url), UTF8.getBytes(record.attribute("author", "Anonymous") + ": " + record.attribute("page", "")), @@ -292,7 +292,7 @@ public class Surftips { Seed seed = sb.peers.getConnected(record.originator()); if (seed == null) seed = sb.peers.getDisconnected(record.originator()); if (seed != null) { - url = "http://" + seed.getPublicAddress() + "/Blog.html?page=" + record.attribute("page", ""); + url = "http://" + seed.getPublicAddress(seed.getIP()) + "/Blog.html?page=" + record.attribute("page", ""); entry = rowdef.newEntry(new byte[][]{ UTF8.getBytes(url), UTF8.getBytes(record.attribute("author", "Anonymous") + ": " + record.attribute("page", "")), diff --git a/htroot/ViewProfile.java b/htroot/ViewProfile.java index ad96150f4..7a56c5ca8 100644 --- a/htroot/ViewProfile.java +++ b/htroot/ViewProfile.java @@ -113,7 +113,6 @@ public class ViewProfile { } // try to get the profile from remote peer - if (sb.clusterhashes != null) seed.setAlternativeAddress(sb.clusterhashes.get(seed.hash.getBytes())); profile = Protocol.getProfile(seed); // if profile did not arrive, say that peer is disconnected @@ -170,7 +169,7 @@ public class ViewProfile { prop.put("success_" + key, "1"); // only comments get "wikified" if(key.equals("comment")){ - prop.putWiki(sb.peers.mySeed().getClusterAddress(), + prop.putWiki(sb.peers.mySeed().getPublicAddress(), "success_" + key + "_value", entry.getValue().replaceAll("\r", "").replaceAll("\\\\n", "\n")); prop.put("success_" + key + "_b64value", Base64Order.standardCoder.encodeString(entry.getValue())); diff --git a/htroot/Wiki.java b/htroot/Wiki.java index b74c76140..7ca1185fe 100644 --- a/htroot/Wiki.java +++ b/htroot/Wiki.java @@ -147,7 +147,7 @@ public class Wiki { prop.putHTML("mode_pagename", pagename); prop.putHTML("mode_author", author); prop.put("mode_date", dateString(new Date())); - prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_page", post.get("content", "")); + prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_page", post.get("content", "")); prop.putHTML("mode_page-code", post.get("content", "")); } //end contrib of [MN] @@ -237,7 +237,7 @@ public class Wiki { prop.putHTML("mode_versioning_pagename", pagename); prop.putHTML("mode_versioning_author", oentry.author()); prop.put("mode_versioning_date", dateString(oentry.date())); - prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_versioning_page", oentry.page()); + prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_versioning_page", oentry.page()); prop.putHTML("mode_versioning_page-code", UTF8.String(oentry.page())); } } catch (final IOException e) { @@ -252,7 +252,7 @@ public class Wiki { prop.putHTML("mode_pagename", pagename); prop.putHTML("mode_author", page.author()); prop.put("mode_date", dateString(page.date())); - prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_page", page.page()); + prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_page", page.page()); prop.put("controls", "0"); prop.putHTML("controls_pagename", pagename); diff --git a/htroot/goto_p.java b/htroot/goto_p.java index 387665ac4..b958742d9 100644 --- a/htroot/goto_p.java +++ b/htroot/goto_p.java @@ -43,7 +43,7 @@ public class goto_p { * hash= of remote peer * path= path part to forward to */ - public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { + public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) { final Switchboard sb = (Switchboard) env; final serverObjects prop = new serverObjects(); diff --git a/htroot/mediawiki_p.java b/htroot/mediawiki_p.java index 3a6bf3864..140c9e115 100644 --- a/htroot/mediawiki_p.java +++ b/htroot/mediawiki_p.java @@ -70,7 +70,7 @@ public class mediawiki_p { page = page.substring(p, q); prop.putHTML("title", title); - prop.putWiki(sb.peers.mySeed().getClusterAddress(), "page", page); + prop.putWiki(sb.peers.mySeed().getPublicAddress(), "page", page); return prop; } diff --git a/htroot/yacy/hello.java b/htroot/yacy/hello.java index 7dadac23f..2659c14a1 100644 --- a/htroot/yacy/hello.java +++ b/htroot/yacy/hello.java @@ -30,17 +30,18 @@ import java.io.IOException; import java.net.InetAddress; import java.util.Iterator; +import java.util.Set; import java.util.concurrent.ConcurrentMap; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; +import net.yacy.cora.util.ConcurrentLog; import net.yacy.peers.Network; import net.yacy.peers.DHTSelection; import net.yacy.peers.Protocol; import net.yacy.peers.Seed; import net.yacy.peers.graphics.ProfilingGraph; -import net.yacy.peers.operation.yacyVersion; import net.yacy.search.EventTracker; import net.yacy.search.Switchboard; import net.yacy.server.serverCore; @@ -58,6 +59,7 @@ public final class hello { final long start = System.currentTimeMillis(); prop.put("message", "none"); final String clientip = header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, ""); // read an artificial header addendum + ConcurrentLog.info("**hello-DEBUG**", "client request from = " + clientip); final InetAddress ias = Domains.dnsResolve(clientip); long time = System.currentTimeMillis(); final long time_dnsResolve = System.currentTimeMillis() - time; @@ -78,13 +80,13 @@ public final class hello { return prop; } -// final String iam = (String) post.get("iam", ""); // complete seed of the requesting peer +// final String iam = (String) post.get("iam", ""); // complete seed of the requesting peer // final String mytime = (String) post.get(MYTIME, ""); // final String key = post.get("key", ""); // transmission key for response final String seed = post.get("seed", ""); int count = post.getInt("count", 0); - final long magic = post.getLong("magic", 0); -// final Date remoteTime = yacyCore.parseUniversalDate(post.get(MYTIME)); // read remote time + // final long magic = post.getLong("magic", 0); + // final Date remoteTime = yacyCore.parseUniversalDate(post.get(MYTIME)); // read remote time if (seed.length() > Seed.maxsize) { Network.log.info("hello/server: rejected contacting seed; too large (" + seed.length() + " > " + Seed.maxsize + ", time_dnsResolve=" + time_dnsResolve + ")"); prop.put("message", "your seed is too long (" + seed.length() + ")"); @@ -105,21 +107,21 @@ public final class hello { return prop; } -// final String properTest = remoteSeed.isProper(); - // The remote peer might not know its IP yet, so don't abort if the IP check fails -// if ((properTest != null) && (! properTest.substring(0,1).equals("IP"))) { return null; } - // we easily know the caller's IP: final String userAgent = header.get(HeaderFramework.USER_AGENT, ""); sb.peers.peerActions.setUserAgent(clientip, userAgent); - final String reportedip = remoteSeed.getIP(); + final Set reportedips = remoteSeed.getIPs(); final String reportedPeerType = remoteSeed.get(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR); - final double clientversion = remoteSeed.getVersion(); + //final double clientversion = remoteSeed.getVersion(); - if ((reportedip + ':' + remoteSeed.getPort()).equals(sb.peers.mySeed().getPublicAddress())) { - // reject a self-ping - prop.put("message", "I am I"); - return prop; + if (remoteSeed.getPort() == sb.peers.mySeed().getPort()) { + for (String reportedip: reportedips) { + if (sb.peers.mySeed().getIPs().contains(reportedip)) { + // reject a self-ping + prop.put("message", "I am I"); + return prop; + } + } } if (remoteSeed.hash.equals(sb.peers.mySeed().hash)) { // reject a ping with my own hash @@ -140,58 +142,42 @@ public final class hello { } long[] callback = new long[]{-1, -1}; - if (sb.clusterhashes != null) remoteSeed.setAlternativeAddress(sb.clusterhashes.get(remoteSeed.hash.getBytes())); // if the remote client has reported its own IP address and the client supports // the port forwarding feature (if client version >= 0.383) then we try to // connect to the reported IP address first long time_backping = 0; String backping_method = "none"; - if (reportedip.length() > 0 && - !clientip.equals(reportedip) && - clientversion >= yacyVersion.YACY_SUPPORTS_PORT_FORWARDING && - magic != 0) { - - // try first the reportedip, since this may be a connect from a port-forwarding host - prop.put("yourip", reportedip); - remoteSeed.setIP(reportedip); - time = System.currentTimeMillis(); - callback = Protocol.queryRWICount(remoteSeed, "Tq418bNZd6AO"); - time_backping = System.currentTimeMillis() - time; - backping_method = "reportedip=" + reportedip; - } else { - prop.put("yourip", ias.getHostAddress()); - remoteSeed.setIP(ias.getHostAddress()); + boolean success = false; + // TODO: make this a concurrent process + if (!serverCore.useStaticIP || !ias.isSiteLocalAddress()) { + reportedips.add(ias.getHostAddress()); } - - // if the previous attempt (using the reported ip address) was not successful, - // then try the ip where the request came from - if (callback[0] < 0 || (magic != 0 && magic != callback[1])) { - boolean isNotLocal = true; - - // we are only allowed to connect to the client IP address if it's not our own address - if (serverCore.useStaticIP) { - isNotLocal = !ias.isSiteLocalAddress(); - } - - if (isNotLocal) { - - prop.put("yourip", clientip); - remoteSeed.setIP(clientip); - time = System.currentTimeMillis(); - callback = Protocol.queryRWICount(remoteSeed, "Tq418bNZd6AO"); // hash for "www"; the actual count is irrelevant, we just want to know if this works - time_backping = System.currentTimeMillis() - time; - backping_method = "clientip=" + clientip; + final int connectedBefore = sb.peers.sizeConnected(); + ConcurrentLog.info("**hello-DEBUG**", "peer " + remoteSeed.getName() + " challenged us with IPs " + reportedips); + int callbackRemain = Math.min(5, reportedips.size()); + long callbackStart = System.currentTimeMillis(); + if (reportedips.size() > 0) { + for (String reportedip: reportedips) { + int partialtimeout = ((int) (callbackStart + 6500 - System.currentTimeMillis())) / callbackRemain; // bad hack until a concurrent version is implemented + if (partialtimeout <= 0) break; + ConcurrentLog.info("**hello-DEBUG**", "reportedip = " + reportedip + " is handled"); + if (Seed.isProperIP(reportedip)) { + ConcurrentLog.info("**hello-DEBUG**", "starting callback to reportedip = " + reportedip + ", timeout = " + partialtimeout); + prop.put("yourip", reportedip); + remoteSeed.setIP(reportedip); + time = System.currentTimeMillis(); + callback = Protocol.queryRWICount(remoteSeed.getPublicAddress(reportedip), remoteSeed.hash, partialtimeout); + ConcurrentLog.info("**hello-DEBUG**", "reportedip = " + reportedip + " returns callback " + (callback == null ? "NULL" : callback[0])); + time_backping = System.currentTimeMillis() - time; + backping_method = "reportedip=" + reportedip; + if (callback[0] >= 0) { success = true; break; } + if (callbackRemain-- <= 0) break; // no more tries left / restrict to a limited number of ips + } } } - -// System.out.println("YACYHELLO: YOUR IP=" + clientip); - // set lastseen value (we have seen that peer, it contacted us!) - remoteSeed.setLastSeenUTC(); - - // assign status - final int connectedBefore = sb.peers.sizeConnected(); - if (callback[0] >= 0) { + if (success) { + ConcurrentLog.info("**hello-DEBUG**", "success for IP(s) " + remoteSeed.getIPs() + ", port " + remoteSeed.getPort()); if (remoteSeed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) == null) { prop.put(Seed.YOURTYPE, Seed.PEERTYPE_SENIOR); remoteSeed.put(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR); @@ -202,26 +188,27 @@ public final class hello { remoteSeed.put(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR); } // connect the seed - Network.log.info("hello/server: responded remote senior peer '" + remoteSeed.getName() + "' from " + reportedip + ", time_dnsResolve=" + time_dnsResolve + ", time_backping=" + time_backping + ", method=" + backping_method + ", urls=" + callback[0]); + Network.log.info("hello/server: responded remote " + reportedPeerType + " peer '" + remoteSeed.getName() + "' from " + reportedips + ", time_dnsResolve=" + time_dnsResolve + ", time_backping=" + time_backping + ", method=" + backping_method + ", urls=" + callback[0]); sb.peers.peerActions.peerArrival(remoteSeed, true); } else { + ConcurrentLog.info("**hello-DEBUG**", "fail for IP(s) " + remoteSeed.getIPs() + ", port " + remoteSeed.getPort()); + prop.put("yourip", ias.getHostAddress()); + remoteSeed.setIP(ias.getHostAddress()); prop.put(Seed.YOURTYPE, Seed.PEERTYPE_JUNIOR); remoteSeed.put(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR); - Network.log.info("hello/server: responded remote junior peer '" + remoteSeed.getName() + "' from " + reportedip + ", time_dnsResolve=" + time_dnsResolve + ", time_backping=" + time_backping + ", method=" + backping_method + ", urls=" + callback[0]); + Network.log.info("hello/server: responded remote " + reportedPeerType + " peer '" + remoteSeed.getName() + "' from " + reportedips + ", time_dnsResolve=" + time_dnsResolve + ", time_backping=" + time_backping + ", method=" + backping_method + ", urls=" + callback[0]); // no connection here, instead store junior in connection cache if ((remoteSeed.hash != null) && (remoteSeed.isProper(false) == null)) { sb.peers.peerActions.peerPing(remoteSeed); } } + remoteSeed.setLastSeenUTC(); final int connectedAfter = sb.peers.sizeConnected(); // update event tracker EventTracker.update(EventTracker.EClass.PEERPING, new ProfilingGraph.EventPing(remoteSeed.getName(), sb.peers.myName(), false, connectedAfter - connectedBefore), false); if (!(prop.get(Seed.YOURTYPE)).equals(reportedPeerType)) { - Network.log.info("hello/server: changing remote peer '" + remoteSeed.getName() + - "' [" + reportedip + - "] peerType from '" + reportedPeerType + - "' to '" + prop.get(Seed.YOURTYPE) + "'."); + Network.log.info("hello/server: changing remote peer '" + remoteSeed.getName() + "' " + reportedips + " peerType from '" + reportedPeerType + "' to '" + prop.get(Seed.YOURTYPE) + "'."); } final StringBuilder seeds = new StringBuilder(768); @@ -262,7 +249,7 @@ public final class hello { prop.put("seedlist", seeds.toString()); // return rewrite properties prop.put("message", "ok " + seed.length()); - Network.log.info("hello/server: responded remote peer '" + remoteSeed.getName() + "' [" + reportedip + "] in " + (System.currentTimeMillis() - start) + " milliseconds"); + Network.log.info("hello/server: responded remote peer '" + remoteSeed.getName() + "' " + reportedips + " in " + (System.currentTimeMillis() - start) + " milliseconds"); return prop; } diff --git a/htroot/yacy/query.java b/htroot/yacy/query.java index a2f7d27c9..5e881c8ef 100644 --- a/htroot/yacy/query.java +++ b/htroot/yacy/query.java @@ -109,7 +109,7 @@ public final class query { if (obj.equals("lurlcount")) { // return the number of all available l-url's - prop.put("response", 1 /*sb.index.fulltext().collectionSize()*/); // patched to not call collectionSize() any more because the acutal size is not needed. Instead, rwicount should be called + prop.put("response", "1" /*sb.index.fulltext().collectionSize()*/); // patched to not call collectionSize() any more because the acutal size is not needed. Instead, rwicount should be called return prop; } diff --git a/installYaCyWindowsService.bat b/installYaCyWindowsService.bat index 38e701383..d91c9599d 100644 --- a/installYaCyWindowsService.bat +++ b/installYaCyWindowsService.bat @@ -3,7 +3,7 @@ title YaCy Windows Service Install :STARTJAVA REM set the Java options -set javaopts=-Xss256k;-XX:MaxPermSize=256m;-Djava.net.preferIPv4Stack=true;-Djava.awt.headless=true;-Dfile.encoding=UTF-8 +set javaopts=-Xss256k;-XX:MaxPermSize=256m;-Djava.awt.headless=true;-Dfile.encoding=UTF-8 REM set max Java heap memory (in MB) set jmx=800 diff --git a/source/net/yacy/cora/protocol/Domains.java b/source/net/yacy/cora/protocol/Domains.java index bf759fd2a..8a0d1ef21 100644 --- a/source/net/yacy/cora/protocol/Domains.java +++ b/source/net/yacy/cora/protocol/Domains.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.lang.reflect.Array; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.net.Inet4Address; import java.net.InetAddress; import java.net.NetworkInterface; import java.net.SocketException; @@ -68,15 +69,19 @@ public class Domains { private final static ConcurrentLog log = new ConcurrentLog(Domains.class.getName()); - public static final String LOCALHOST = "127.0.0.1"; // replace with IPv6 0:0:0:0:0:0:0:1 ? + public static final String LOCALHOST = "localhost"; // replace with IPv6 0:0:0:0:0:0:0:1 ? private static String LOCALHOST_NAME = LOCALHOST; // this will be replaced with the actual name of the local host private static Class InetAddressLocatorClass; private static Method InetAddressLocatorGetLocaleInetAddressMethod; private static final Set ccSLD_TLD = new HashSet(); private static final String PRESENT = ""; - private static final Pattern LOCALHOST_PATTERNS = Pattern.compile("(127\\..*)|(localhost)|(\\[?(fe80|0)\\:0\\:0\\:0\\:0\\:0\\:0\\:1.*)"); - private static final Pattern INTRANET_PATTERNS = Pattern.compile("(10\\..*)|(127\\..*)|(172\\.(1[6-9]|2[0-9]|3[0-1])\\..*)|(169\\.254\\..*)|(192\\.168\\..*)|(localhost)|(\\[?\\:\\:1/.*)|(\\[?fc.*\\:.*)|(\\[?fd.*\\:.*)|(\\[?(fe80|0)\\:0\\:0\\:0\\:0\\:0\\:0\\:1.*)", Pattern.CASE_INSENSITIVE); + private static final String LOCALHOST_IPv4_PATTERN = "(127\\..*)"; + private static final String LOCALHOST_IPv6_PATTERN = "(\\[?fe80\\:\\:(/.*|\\z))|(\\[?0\\:0\\:0\\:0\\:0\\:0\\:0\\:1.*)|(\\[?\\:\\:1(/.*|\\z))"; + private static final String INTRANET_IPv4_PATTERN = "(10\\..*)|(172\\.(1[6-9]|2[0-9]|3[0-1])\\..*)|(169\\.254\\..*)|(192\\.168\\..*)"; + private static final String INTRANET_IPv6_PATTERN = "(\\[?(fc|fd).*\\:.*)"; + private static final Pattern LOCALHOST_PATTERNS = Pattern.compile("(localhost)|" + LOCALHOST_IPv4_PATTERN + "|" + LOCALHOST_IPv6_PATTERN, Pattern.CASE_INSENSITIVE); + private static final Pattern INTRANET_PATTERNS = Pattern.compile(LOCALHOST_PATTERNS.pattern() + "|" + INTRANET_IPv4_PATTERN + "|" + INTRANET_IPv6_PATTERN, Pattern.CASE_INSENSITIVE); private static final int MAX_NAME_CACHE_HIT_SIZE = 10000; private static final int MAX_NAME_CACHE_MISS_SIZE = 1000; @@ -90,6 +95,92 @@ public class Domains { public static long cacheHit_Hit = 0, cacheHit_Miss = 0, cacheHit_Insert = 0; // for statistics only; do not write public static long cacheMiss_Hit = 0, cacheMiss_Miss = 0, cacheMiss_Insert = 0; // for statistics only; do not write + private static Set myHostAddresses = new HashSet(); + private static Set localHostAddresses = new HashSet(); // subset of myHostAddresses + private static Set publicIPv4HostAddresses = new HashSet(); // subset of myHostAddresses + private static Set publicIPv6HostAddresses = new HashSet(); // subset of myHostAddresses + private static Set myHostNames = new HashSet(); + private static Set localHostNames = new HashSet(); // subset of myHostNames + private static Set publicIPv4HostNames = new HashSet(); // subset of myHostNames + private static Set publicIPv6HostNames = new HashSet(); // subset of myHostNames + static { + myHostNames.add(LOCALHOST); + localHostNames.add(LOCALHOST); + try { + InetAddress localHostAddress = InetAddress.getLocalHost(); + if (localHostAddress != null) myHostAddresses.add(localHostAddress); + } catch (final UnknownHostException e) {} + try { + final InetAddress[] moreAddresses = InetAddress.getAllByName(LOCALHOST_NAME); + if (moreAddresses != null) myHostAddresses.addAll(Arrays.asList(moreAddresses)); + } catch (final UnknownHostException e) {} + + // to get the local host name, a dns lookup is necessary. + // if such a lookup blocks, it can cause that the static initiatializer does not finish fast + // therefore we start the host name lookup as concurrent thread + // meanwhile the host name is "127.0.0.1" which is not completely wrong + new Thread() { + @Override + public void run() { + Thread.currentThread().setName("Domains: init"); + // try to get local addresses from interfaces + try { + final Enumeration nis = NetworkInterface.getNetworkInterfaces(); + while (nis.hasMoreElements()) { + final NetworkInterface ni = nis.nextElement(); + final Enumeration addrs = ni.getInetAddresses(); + while (addrs.hasMoreElements()) { + final InetAddress addr = addrs.nextElement(); + if (addr != null) myHostAddresses.add(addr); + } + } + } catch (final SocketException e) { + } + + // now look up the host name + try { + LOCALHOST_NAME = getHostName(InetAddress.getLocalHost()); + } catch (final UnknownHostException e) {} + + // after the host name was resolved, we try to look up more local addresses + // using the host name: + try { + final InetAddress[] moreAddresses = InetAddress.getAllByName(LOCALHOST_NAME); + if (moreAddresses != null) myHostAddresses.addAll(Arrays.asList(moreAddresses)); + } catch (final UnknownHostException e) { + } + + // fill a cache of local host names + for (final InetAddress a: myHostAddresses) { + final String hostname = getHostName(a); + if (hostname != null) { + myHostNames.add(hostname); + myHostNames.add(a.getHostAddress()); + } + // we write the local tests into variables to be able to debug these values + boolean isAnyLocalAddress = a.isAnyLocalAddress(); + boolean isLinkLocalAddress = a.isLinkLocalAddress(); // true i.e. for localhost/fe80:0:0:0:0:0:0:1%1, myhost.local/fe80:0:0:0:223:dfff:fedf:30ce%7 + boolean isLoopbackAddress = a.isLoopbackAddress(); // true i.e. for localhost/0:0:0:0:0:0:0:1, localhost/127.0.0.1 + boolean isSiteLocalAddress = a.isSiteLocalAddress(); // true i.e. for myhost.local/192.168.1.33 + if (isAnyLocalAddress || isLinkLocalAddress || isLoopbackAddress || isSiteLocalAddress) { + ConcurrentLog.info("Domain Init", "local host address: " + a + " (local)"); + localHostAddresses.add(a); + if (hostname != null) {localHostNames.add(hostname); localHostNames.add(a.getHostAddress());} + } else { + ConcurrentLog.info("Domain Init", "local host address: " + a + " (public)"); + if (a instanceof Inet4Address) { + publicIPv4HostAddresses.add(a); + if (hostname != null) {publicIPv4HostNames.add(hostname); publicIPv4HostNames.add(a.getHostAddress());} + } else { + publicIPv6HostAddresses.add(a); + if (hostname != null) {publicIPv6HostNames.add(hostname); publicIPv6HostNames.add(a.getHostAddress());} + } + } + } + } + }.start(); + } + /** * ! ! ! A T T E N T I O N A T T E N T I O N A T T E N T I O N ! ! ! * @@ -810,7 +901,7 @@ public class Domains { // add also the isLocal host name caches final boolean localp = ip.isAnyLocalAddress() || ip.isLinkLocalAddress() || ip.isSiteLocalAddress(); if (localp) { - localHostNames.add(host); + myHostNames.add(host); } else { if (globalHosts != null) try { globalHosts.add(host); @@ -847,131 +938,54 @@ public class Domains { return nameCacheNoCachingPatterns.size(); } - private static Set localHostAddresses = new HashSet(); - private static Set localHostNames = new HashSet(); - static { - try { - final InetAddress localHostAddress = InetAddress.getLocalHost(); - if (localHostAddress != null) localHostAddresses.add(localHostAddress); - } catch (final UnknownHostException e) {} - try { - final InetAddress[] moreAddresses = InetAddress.getAllByName(LOCALHOST_NAME); - if (moreAddresses != null) localHostAddresses.addAll(Arrays.asList(moreAddresses)); - } catch (final UnknownHostException e) {} - - // to get the local host name, a dns lookup is necessary. - // if such a lookup blocks, it can cause that the static initiatializer does not finish fast - // therefore we start the host name lookup as concurrent thread - // meanwhile the host name is "127.0.0.1" which is not completely wrong - new Thread() { - @Override - public void run() { - Thread.currentThread().setName("Domains: init"); - // try to get local addresses from interfaces - try { - final Enumeration nis = NetworkInterface.getNetworkInterfaces(); - while (nis.hasMoreElements()) { - final NetworkInterface ni = nis.nextElement(); - final Enumeration addrs = ni.getInetAddresses(); - while (addrs.hasMoreElements()) { - final InetAddress addr = addrs.nextElement(); - if (addr != null) localHostAddresses.add(addr); - } - } - } catch (final SocketException e) { - } - - // now look up the host name - try { - LOCALHOST_NAME = getHostName(InetAddress.getLocalHost()); - } catch (final UnknownHostException e) {} - - // after the host name was resolved, we try to look up more local addresses - // using the host name: - try { - final InetAddress[] moreAddresses = InetAddress.getAllByName(LOCALHOST_NAME); - if (moreAddresses != null) localHostAddresses.addAll(Arrays.asList(moreAddresses)); - } catch (final UnknownHostException e) { - } - - // fill a cache of local host names - for (final InetAddress a: localHostAddresses) { - final String hostname = getHostName(a); - if (hostname != null) { - localHostNames.add(hostname); - localHostNames.add(a.getHostAddress()); - } - } - } - }.start(); + /** + * myPublicLocalIP() returns the IP of this host which is reachable in the public network under this address + * This is deprecated since it should be possible that the host is reachable with more than one IP + * That is particularly the case if the host supports IPv4 and IPv6. + * Please use myPublicIPv4() or (preferred) myPublicIPv6() instead. + * @return + */ + @Deprecated + public static InetAddress myPublicLocalIP() { + // for backward compatibility, we try to select a IPv4 address here. + // future methods should use myPublicIPs() and prefer IPv6 + if (publicIPv4HostAddresses.size() > 0) return publicIPv4HostAddresses.iterator().next(); + if (publicIPv6HostAddresses.size() > 0) return publicIPv6HostAddresses.iterator().next(); + return null; } - public static InetAddress myPublicLocalIP() { - // list all addresses - // for (int i = 0; i < localHostAddresses.length; i++) System.out.println("IP: " + localHostAddresses[i].getHostAddress()); // DEBUG - if (localHostAddresses.isEmpty()) { - return null; - } - if (localHostAddresses.size() == 1) { - // only one network connection available - return localHostAddresses.iterator().next(); - } - // we have more addresses, find an address that is not local - int b0, b1; - for (final InetAddress a: localHostAddresses) { - b0 = 0Xff & a.getAddress()[0]; - b1 = 0Xff & a.getAddress()[1]; - if (b0 != 10 && // class A reserved - b0 != 127 && // loopback - (b0 != 172 || b1 < 16 || b1 > 31) && // class B reserved - (b0 != 192 || b1 != 168) && // class C reserved - (a.getHostAddress().indexOf(':',0) < 0)) - return a; - } - // there is only a local address - // return that one that is returned with InetAddress.getLocalHost() - // if appropriate - try { - final InetAddress localHostAddress = InetAddress.getLocalHost(); - if (localHostAddress != null && - (0Xff & localHostAddress.getAddress()[0]) != 127 && - localHostAddress.getHostAddress().indexOf(':',0) < 0) return localHostAddress; - } catch (final UnknownHostException e) { - } - // we filter out the loopback address 127.0.0.1 and all addresses without a name - for (final InetAddress a: localHostAddresses) { - if ((0Xff & a.getAddress()[0]) != 127 && - a.getHostAddress().indexOf(':',0) < 0 && - a.getHostName() != null && - !a.getHostName().isEmpty()) return a; - } - // if no address has a name, then take any other than the loopback - for (final InetAddress a: localHostAddresses) { - if ((0Xff & a.getAddress()[0]) != 127 && - a.getHostAddress().indexOf(':',0) < 0) return a; - } - // if all fails, give back whatever we have - for (final InetAddress a: localHostAddresses) { - if (a.getHostAddress().indexOf(':',0) < 0) return a; - } - // finally, just get any - return localHostAddresses.iterator().next(); + public static Set myPublicIPs() { + Set h = new HashSet<>(publicIPv4HostAddresses.size() + publicIPv6HostAddresses.size()); + for (InetAddress i: publicIPv4HostAddresses) h.add(i.getHostAddress()); + for (InetAddress i: publicIPv6HostAddresses) h.add(i.getHostAddress()); + return h; + } + + /** + * Get all IPv4 addresses which are assigned to the local host but are public IP addresses. + * These should be the possible addresses which can be used to access this peer. + * @return the public IPv4 Addresses of this peer + */ + public static Set myPublicIPv4() { + return publicIPv4HostAddresses; + } + + /** + * Get all IPv6 addresses which are assigned to the local host but are public IP addresses. + * These should be the possible addresses which can be used to access this peer. + * @return the public IPv6 addresses of this peer + */ + public static Set myPublicIPv6() { + return publicIPv6HostAddresses; } /** - * generate a list of intranet InetAddresses without the loopback address 127.0.0.1 + * generate a list of intranet InetAddresses * @return list of all intranet addresses */ public static Set myIntranetIPs() { - // list all local addresses - if (localHostAddresses.size() < 1) try {Thread.sleep(1000);} catch (final InterruptedException e) {} - final Set list = new HashSet(); - if (localHostAddresses.isEmpty()) return list; // give up - for (final InetAddress a: localHostAddresses) { - if ((0Xff & a.getAddress()[0]) == 127) continue; - list.add(a); - } - return list; + if (myHostAddresses.size() < 1) try {Thread.sleep(1000);} catch (final InterruptedException e) {} + return localHostAddresses; } public static boolean isThisHostIP(final String hostName) { @@ -982,7 +996,7 @@ public class Domains { final InetAddress clientAddress = Domains.dnsResolve(hostName); if (clientAddress == null) return false; if (clientAddress.isAnyLocalAddress() || clientAddress.isLoopbackAddress()) return true; - for (final InetAddress a: localHostAddresses) { + for (final InetAddress a: myHostAddresses) { if (a.equals(clientAddress)) { isThisHostIP = true; break; @@ -999,7 +1013,7 @@ public class Domains { try { if (clientAddress.isAnyLocalAddress() || clientAddress.isLoopbackAddress()) return true; - for (final InetAddress a: localHostAddresses) { + for (final InetAddress a: myHostAddresses) { if (a.equals(clientAddress)) { isThisHostIP = true; break; @@ -1024,7 +1038,9 @@ public class Domains { * @return true if the host from the string is the localhost */ public static boolean isLocalhost(final String host) { - return (host != null && LOCALHOST_PATTERNS.matcher(host).matches()); + return host == null || // filesystems do not have host names + LOCALHOST_PATTERNS.matcher(host).matches() || + localHostNames.contains(host); } /** @@ -1038,7 +1054,9 @@ public class Domains { */ public static boolean isIntranet(final String host) { return (noLocalCheck || // DO NOT REMOVE THIS! it is correct to return true if the check is off - (host != null && INTRANET_PATTERNS.matcher(host).matches())); + host == null || // filesystems do not have host names + INTRANET_PATTERNS.matcher(host).matches()) || + localHostNames.contains(host); } /** @@ -1060,10 +1078,7 @@ public class Domains { // check local ip addresses if (isIntranet(host)) return true; - if (hostaddress != null && ( - isIntranet(hostaddress.getHostAddress()) || - isLocal(hostaddress) - )) return true; + if (hostaddress != null && (isIntranet(hostaddress.getHostAddress()) || isLocal(hostaddress))) return true; // check if there are other local IP addresses that are not in // the standard IP range diff --git a/source/net/yacy/cora/protocol/RequestHeader.java b/source/net/yacy/cora/protocol/RequestHeader.java index 8ec7be751..56404cf79 100644 --- a/source/net/yacy/cora/protocol/RequestHeader.java +++ b/source/net/yacy/cora/protocol/RequestHeader.java @@ -134,6 +134,7 @@ public class RequestHeader extends HeaderFramework { return false; } final String refererHost = this.refererHost(); - return refererHost == null || refererHost.isEmpty() || Domains.isLocalhost(refererHost); + if (refererHost == null || refererHost.isEmpty() || Domains.isLocalhost(refererHost)) return true; + return false; } } diff --git a/source/net/yacy/cora/protocol/http/HTTPClient.java b/source/net/yacy/cora/protocol/http/HTTPClient.java index a45c05844..4082cb486 100644 --- a/source/net/yacy/cora/protocol/http/HTTPClient.java +++ b/source/net/yacy/cora/protocol/http/HTTPClient.java @@ -37,6 +37,11 @@ import java.security.cert.X509Certificate; import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; import javax.net.ssl.SSLContext; @@ -58,6 +63,7 @@ import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.ClientProtocolException; import org.apache.http.client.CredentialsProvider; import org.apache.http.client.config.CookieSpecs; import org.apache.http.client.config.RequestConfig; @@ -77,7 +83,6 @@ import org.apache.http.conn.routing.HttpRoute; import org.apache.http.conn.socket.ConnectionSocketFactory; import org.apache.http.conn.socket.PlainConnectionSocketFactory; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.entity.InputStreamEntity; import org.apache.http.entity.mime.MultipartEntityBuilder; import org.apache.http.entity.mime.content.ContentBody; import org.apache.http.impl.client.BasicCredentialsProvider; @@ -112,7 +117,7 @@ public class HTTPClient { private long upbytes = 0L; private String host = null; private final long timeout; - + private static ExecutorService executor = Executors.newFixedThreadPool(200); public HTTPClient(final ClientIdentification.Agent agent) { super(); @@ -236,23 +241,6 @@ public class HTTPClient { } } -// public static void setAuth(final String host, final int port, final String user, final String pw) { -// final UsernamePasswordCredentials creds = new UsernamePasswordCredentials(user, pw); -// final AuthScope scope = new AuthScope(host, port); -// credsProvider.setCredentials(scope, creds); -// httpClient.getParams().setParameter(ClientContext.CREDS_PROVIDER, credsProvider); -// } - -// /** -// * this method sets a host on which more than the default of 2 router per host are allowed -// * -// * @param the host to be raised in 'route per host' -// */ -// public static void setMaxRouteHost(final String host) { -// final HttpHost mHost = new HttpHost(host); -// ((PoolingClientConnectionManager) httpClient.getConnectionManager()).setMaxPerRoute(new HttpRoute(mHost), 50); -// } - /** * This method sets the Header used for the request * @@ -454,10 +442,12 @@ public class HTTPClient { * @param length the contentlength * @throws IOException */ + /* public void POST(final String uri, final InputStream instream, final long length, final boolean concurrent) throws IOException { POST(new MultiProtocolURL(uri), instream, length, concurrent); } - + */ + /** * This method POSTs a page from the server. * to be used for streaming out @@ -490,11 +480,13 @@ public class HTTPClient { * @return content bytes * @throws IOException */ + /* public byte[] POSTbytes(final String uri, final Map parts, final boolean usegzip, final boolean concurrent) throws IOException { final MultiProtocolURL url = new MultiProtocolURL(uri); return POSTbytes(url, url.getHost(), parts, usegzip, concurrent); } - + */ + /** * send data to the server named by vhost * @@ -512,8 +504,7 @@ public class HTTPClient { if (vhost == null) setHost(Domains.LOCALHOST); final MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create(); - for (final Entry part : post.entrySet()) - entityBuilder.addPart(part.getKey(), part.getValue()); + for (final Entry part : post.entrySet()) entityBuilder.addPart(part.getKey(), part.getValue()); final HttpEntity multipartEntity = entityBuilder.build(); // statistics this.upbytes = multipartEntity.getContentLength(); @@ -536,6 +527,7 @@ public class HTTPClient { * @return content bytes * @throws IOException */ + /* public byte[] POSTbytes(final String uri, final InputStream instream, final long length, final boolean concurrent) throws IOException { final MultiProtocolURL url = new MultiProtocolURL(uri); final HttpPost httpPost = new HttpPost(url.toNormalform(true)); @@ -549,8 +541,9 @@ public class HTTPClient { httpPost.setEntity(inputStreamEntity); return getContentBytes(httpPost, Integer.MAX_VALUE, concurrent); } + */ - /** + /** * * @return HttpResponse from call */ @@ -682,30 +675,26 @@ public class HTTPClient { assert !hrequest.expectContinue(); } - Thread.currentThread().setName("HTTPClient-" + httpUriRequest.getURI().getHost()); + Thread.currentThread().setName("HTTPClient-" + httpUriRequest.getURI()); final long time = System.currentTimeMillis(); try { final CloseableHttpClient client = clientBuilder.build(); if (concurrent) { - final CloseableHttpResponse[] thr = new CloseableHttpResponse[]{null}; - final Throwable[] te = new Throwable[]{null}; - Thread t = new Thread() { + FutureTask t = new FutureTask(new Callable() { @Override - public void run() { - this.setName("HTTPClient.execute(" + httpUriRequest.getURI() + ")"); - try { - thr[0] = client.execute(httpUriRequest, context); - } catch (Throwable e) { - te[0] = e; - } + public CloseableHttpResponse call() throws ClientProtocolException, IOException { + CloseableHttpResponse response = client.execute(httpUriRequest, context); + return response; } - }; - t.start(); - try {t.join(this.timeout);} catch (InterruptedException e) {} - if (t.isAlive()) try {t.interrupt();} catch (Throwable e) {} - if (te[0] != null) throw te[0]; - if (thr[0] == null) throw new IOException("timout to client after " + this.timeout + "ms"); - this.httpResponse = thr[0]; + }); + executor.execute(t); + try { + this.httpResponse = t.get(this.timeout, TimeUnit.MILLISECONDS); + } catch (ExecutionException e) { + throw e.getCause(); + } catch (Throwable e) {} + try {t.cancel(true);} catch (Throwable e) {} + if (this.httpResponse == null) throw new IOException("timout to client after " + this.timeout + "ms"); } else { this.httpResponse = client.execute(httpUriRequest, context); } diff --git a/source/net/yacy/cora/util/CommonPattern.java b/source/net/yacy/cora/util/CommonPattern.java index 1b1894e87..93f7a1f99 100644 --- a/source/net/yacy/cora/util/CommonPattern.java +++ b/source/net/yacy/cora/util/CommonPattern.java @@ -43,6 +43,7 @@ public class CommonPattern { public final static Pattern SEMICOLON = Pattern.compile(";"); public final static Pattern DOUBLEPOINT = Pattern.compile(":"); public final static Pattern SLASH = Pattern.compile("/"); + public final static Pattern PIPE = Pattern.compile("\\|"); public final static Pattern BACKSLASH = Pattern.compile("\\\\"); public final static Pattern QUESTION = Pattern.compile("\\?"); public final static Pattern AMP = Pattern.compile("&"); diff --git a/source/net/yacy/crawler/CrawlStacker.java b/source/net/yacy/crawler/CrawlStacker.java index a2bb4ab80..4c11e08d4 100644 --- a/source/net/yacy/crawler/CrawlStacker.java +++ b/source/net/yacy/crawler/CrawlStacker.java @@ -36,7 +36,6 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.atomic.AtomicInteger; import net.yacy.contentcontrol.ContentControlFilterUpdateThread; -import net.yacy.cora.document.analysis.Classification.ContentDomain; import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.document.id.AnchorURL; diff --git a/source/net/yacy/http/YacyDomainHandler.java b/source/net/yacy/http/YacyDomainHandler.java index 7041b5c35..5df7f3b13 100644 --- a/source/net/yacy/http/YacyDomainHandler.java +++ b/source/net/yacy/http/YacyDomainHandler.java @@ -82,7 +82,7 @@ public class YacyDomainHandler extends AbstractHandler implements Handler { newHost = hostPort; newPort = 80; } - if (newHost.equals(alternativeResolvers.myIP())) return; + if (alternativeResolvers.myIPs().contains(newHost)) return; if (Domains.isLocal(newHost, null)) return; RequestDispatcher dispatcher = request.getRequestDispatcher(path + target); dispatcher.forward(new DomainRequestWrapper(request, newHost, newPort), response); diff --git a/source/net/yacy/migration.java b/source/net/yacy/migration.java index 630c22692..8d83e428b 100644 --- a/source/net/yacy/migration.java +++ b/source/net/yacy/migration.java @@ -37,7 +37,6 @@ import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; import com.google.common.io.Files; -import java.util.concurrent.ExecutionException; import net.yacy.cora.lod.vocabulary.Tagging; import net.yacy.cora.protocol.TimeoutRequest; import net.yacy.cora.storage.Configuration.Entry; diff --git a/source/net/yacy/peers/DHTSelection.java b/source/net/yacy/peers/DHTSelection.java index 689d2e755..e40be93c4 100644 --- a/source/net/yacy/peers/DHTSelection.java +++ b/source/net/yacy/peers/DHTSelection.java @@ -31,10 +31,9 @@ import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; -import java.util.Map; import java.util.Random; import java.util.Set; -import java.util.SortedMap; +import java.util.SortedSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -55,18 +54,12 @@ import net.yacy.peers.operation.yacyVersion; */ public class DHTSelection { - public static Set selectClusterPeers(final SeedDB seedDB, final SortedMap peerhashes) { - final Iterator> i = peerhashes.entrySet().iterator(); + public static Set selectClusterPeers(final SeedDB seedDB, final SortedSet peerhashes) { final Set l = new HashSet(); - Map.Entry entry; Seed s; - while (i.hasNext()) { - entry = i.next(); - s = seedDB.get(ASCII.String(entry.getKey())); // should be getConnected; get only during testing time - if (s != null) { - s.setAlternativeAddress(entry.getValue()); - l.add(s); - } + for (byte[] hashb: peerhashes) { + s = seedDB.get(ASCII.String(hashb)); // should be getConnected; get only during testing time + if (s != null) l.add(s); } return l; } diff --git a/source/net/yacy/peers/Network.java b/source/net/yacy/peers/Network.java index 97dfa6c43..e6197b686 100644 --- a/source/net/yacy/peers/Network.java +++ b/source/net/yacy/peers/Network.java @@ -46,6 +46,7 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Semaphore; @@ -101,7 +102,7 @@ public class Network // ensure that correct IP is used final String staticIP = sb.getConfig("staticIP", ""); - if ( staticIP.length() != 0 && Seed.isProperIP(staticIP) == null ) { + if (staticIP.length() != 0 && Seed.isProperIP(staticIP)) { serverCore.useStaticIP = true; sb.peers.mySeed().setIP(staticIP); log.info("staticIP set to " + staticIP); @@ -118,9 +119,7 @@ public class Network } public final void publishSeedList() { - if ( log.isFine() ) { - log.fine("yacyCore.publishSeedList: Triggered Seed Publish"); - } + if (log.isFine()) log.fine("yacyCore.publishSeedList: Triggered Seed Publish"); /* if (oldIPStamp.equals((String) seedDB.mySeed.get(yacySeed.IP, "127.0.0.1"))) @@ -131,14 +130,11 @@ public class Network yacyCore.log.logDebug("***DEBUG publishSeedList: I can reach myself"); */ - if ( (this.sb.peers.lastSeedUpload_myIP.equals(this.sb.peers.mySeed().getIP())) + if ((this.sb.peers.mySeed().getIPs().contains(this.sb.peers.lastSeedUpload_myIP)) && (this.sb.peers.lastSeedUpload_seedDBSize == this.sb.peers.sizeConnected()) && (System.currentTimeMillis() - this.sb.peers.lastSeedUpload_timeStamp < 1000 * 60 * 60 * 24) && (this.sb.peers.mySeed().isPrincipal()) ) { - if ( log.isFine() ) { - log - .fine("yacyCore.publishSeedList: not necessary to publish: oldIP is equal, sizeConnected is equal and I can reach myself under the old IP."); - } + if (log.isFine()) log.fine("yacyCore.publishSeedList: not necessary to publish: oldIP is equal, sizeConnected is equal and I can reach myself under the old IP."); return; } @@ -162,9 +158,7 @@ public class Network if ( seedUploadMethod.equals("") ) { this.sb.setConfig("seedUploadMethod", "none"); } - if ( log.isFine() ) { - log.fine("yacyCore.publishSeedList: No uploading method configured"); - } + if (log.isFine()) log.fine("yacyCore.publishSeedList: No uploading method configured"); return; } } @@ -231,13 +225,7 @@ public class Network @Override public final void run() { try { - this.added = - Protocol.hello( - Network.this.sb.peers.mySeed(), - Network.this.sb.peers.peerActions, - this.seed.getClusterAddress(), - this.seed.hash, - this.seed.getName()); + this.added = Protocol.hello(Network.this.sb.peers.mySeed(), Network.this.sb.peers.peerActions, this.seed); if ( this.added < 0 ) { // no or wrong response, delete that address final String cause = "peer ping to peer resulted in error response (added < 0)"; @@ -246,7 +234,7 @@ public class Network + " peer '" + this.seed.getName() + "' from " - + this.seed.getPublicAddress() + + this.seed.getIPs() + ": " + cause); Network.this.sb.peers.peerActions.peerDeparture(this.seed, cause); @@ -258,7 +246,7 @@ public class Network + " peer '" + this.seed.getName() + "' at " - + this.seed.getPublicAddress()); + + this.seed.getIPs()); // check if seed's lastSeen has been updated final Seed newSeed = Network.this.sb.peers.getConnected(this.seed.hash); if ( newSeed != null ) { @@ -269,7 +257,7 @@ public class Network + " peer '" + this.seed.getName() + "' at " - + this.seed.getPublicAddress() + + this.seed.getIPs() + " is not online." + " Removing Peer from connected"); } @@ -284,7 +272,7 @@ public class Network + " peer '" + this.seed.getName() + "' at " - + this.seed.getPublicAddress() + + this.seed.getIPs() + " with old LastSeen: '" + my_SHORT_SECOND_FORMATTER.format(new Date(newSeed .getLastSeenUTC())) + "'"); @@ -299,7 +287,7 @@ public class Network + " peer '" + this.seed.getName() + "' at " - + this.seed.getPublicAddress() + + this.seed.getIPs() + " with old LastSeen: '" + my_SHORT_SECOND_FORMATTER.format(new Date(newSeed .getLastSeenUTC())) @@ -319,7 +307,7 @@ public class Network + " peer '" + this.seed.getName() + "' at " - + this.seed.getPublicAddress() + + this.seed.getIPs() + " not in connectedDB"); } } @@ -364,25 +352,21 @@ public class Network if ( attempts > PING_INITIAL ) { attempts = PING_INITIAL; } - final Map ch = Switchboard.getSwitchboard().clusterhashes; + final Set ch = Switchboard.getSwitchboard().clusterhashes; seeds = DHTSelection.seedsByAge(this.sb.peers, true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection // add also all peers from cluster if this is a public robinson cluster if ( ch != null ) { - final Iterator> i = ch.entrySet().iterator(); String hash; - Map.Entry entry; Seed seed; - while ( i.hasNext() ) { - entry = i.next(); - hash = ASCII.String(entry.getKey()); + for (byte[] hashb: ch) { + hash = ASCII.String(hashb); seed = seeds.get(hash); - if ( seed == null ) { + if (seed == null) { seed = this.sb.peers.get(hash); if ( seed == null ) { continue; } } - seed.setAlternativeAddress(entry.getValue()); seeds.put(hash, seed); } } @@ -441,17 +425,14 @@ public class Network } i++; - final String address = seed.getClusterAddress(); + final String address = seed.getPublicAddress(seed.getIP()); if ( log.isFine() ) { log.fine("HELLO #" + i + " to peer '" + seed.get(Seed.NAME, "") + "' at " + address); // debug } final String seederror = seed.isProper(false); if ( (address == null) || (seederror != null) ) { // we don't like that address, delete it - this.sb.peers.peerActions.peerDeparture(seed, "peer ping to peer resulted in address = " - + address - + "; seederror = " - + seederror); + this.sb.peers.peerActions.peerDeparture(seed, "peer ping to peer resulted in address = " + address + "; seederror = " + seederror); sync.acquire(); } else { // starting a new publisher thread @@ -567,16 +548,13 @@ public class Network //if (ip.equals("")) ip = natLib.retrieveIP(DI604use, DI604pw); // yacyCore.log.logDebug("DEBUG: new IP=" + ip); - if ( Seed.isProperIP(ip) == null ) { + if (Seed.isProperIP(ip)) { this.sb.peers.mySeed().setIP(ip); } if ( this.sb.peers.mySeed().get(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR).equals(Seed.PEERTYPE_JUNIOR) ) { this.sb.peers.mySeed().put(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR); // to start bootstraping, we need to be recognised as PEERTYPE_SENIOR peer } - log.info("publish: no recipient found, our address is " - + ((this.sb.peers.mySeed().getPublicAddress() == null) ? "unknown" : this.sb.peers - .mySeed() - .getPublicAddress())); + log.info("publish: no recipient found, our address is " + this.sb.peers.mySeed().getIPs()); this.sb.peers.saveMySeed(); return 0; } catch (final InterruptedException e ) { diff --git a/source/net/yacy/peers/PeerActions.java b/source/net/yacy/peers/PeerActions.java index 845b10946..c79f6ab82 100644 --- a/source/net/yacy/peers/PeerActions.java +++ b/source/net/yacy/peers/PeerActions.java @@ -28,6 +28,7 @@ import java.util.Map; import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.document.feed.RSSMessage; +import net.yacy.cora.protocol.Domains; import net.yacy.cora.storage.ConcurrentARC; import net.yacy.kelondro.util.MapTools; import net.yacy.peers.operation.yacyVersion; @@ -66,7 +67,7 @@ public class PeerActions { return false; } if ((this.seedDB.mySeedIsDefined()) && (seed.hash.equals(this.seedDB.mySeed().hash))) { - Network.log.info("connect: SELF reference " + seed.getPublicAddress()); + Network.log.info("connect: SELF reference " + seed.getIPs()); return false; } final String peerType = seed.get(Seed.PEERTYPE, Seed.PEERTYPE_VIRGIN); @@ -82,7 +83,7 @@ public class PeerActions { return false; } - final Seed doubleSeed = this.seedDB.lookupByIP(seed.getInetAddress(), seed.getPort(), true, false, false); + final Seed doubleSeed = this.seedDB.lookupByIP(Domains.dnsResolve(seed.getIP()), seed.getPort(), true, false, false); if ((doubleSeed != null) && (doubleSeed.getPort() == seed.getPort()) && (!(doubleSeed.hash.equals(seed.hash)))) { // a user frauds with his peer different peer hashes if (Network.log.isFine()) Network.log.fine("connect: rejecting FRAUD (double hashes " + doubleSeed.hash + "/" + seed.hash + " on same port " + seed.getPort() + ") peer " + seed.getName()); @@ -107,7 +108,7 @@ public class PeerActions { } if (Math.abs(nowUTC0Time - ctimeUTC0) / 1000 / 60 > 60 * 6 ) { // the new connection is out-of-age, we reject the connection - if (Network.log.isFine()) Network.log.fine("connect: rejecting out-dated peer '" + seed.getName() + "' from " + seed.getPublicAddress() + "; nowUTC0=" + nowUTC0Time + ", seedUTC0=" + ctimeUTC0 + ", TimeDiff=" + formatInterval(Math.abs(nowUTC0Time - ctimeUTC0))); + if (Network.log.isFine()) Network.log.fine("connect: rejecting out-dated peer '" + seed.getName() + "' from " + seed.getIPs() + "; nowUTC0=" + nowUTC0Time + ", seedUTC0=" + ctimeUTC0 + ", TimeDiff=" + formatInterval(Math.abs(nowUTC0Time - ctimeUTC0))); return false; } @@ -144,13 +145,13 @@ public class PeerActions { if (!direct) { if (ctimeUTC0 < dtimeUTC0) { // the disconnection was later, we reject the connection - if (Network.log.isFine()) Network.log.fine("connect: rejecting disconnected peer '" + seed.getName() + "' from " + seed.getPublicAddress()); + if (Network.log.isFine()) Network.log.fine("connect: rejecting disconnected peer '" + seed.getName() + "' from " + seed.getIPs()); return false; } } // this is a return of a lost peer - if (Network.log.isFine()) Network.log.fine("connect: returned KNOWN " + peerType + " peer '" + seed.getName() + "' from " + seed.getPublicAddress()); + if (Network.log.isFine()) Network.log.fine("connect: returned KNOWN " + peerType + " peer '" + seed.getName() + "' from " + seed.getIPs()); this.seedDB.addConnected(seed); return true; } @@ -169,10 +170,10 @@ public class PeerActions { // TODO: update seed name lookup cache }*/ } catch (final NumberFormatException e) { - if (Network.log.isFine()) Network.log.fine("connect: rejecting wrong peer '" + seed.getName() + "' from " + seed.getPublicAddress() + ". Cause: " + e.getMessage()); + if (Network.log.isFine()) Network.log.fine("connect: rejecting wrong peer '" + seed.getName() + "' from " + seed.getIPs() + ". Cause: " + e.getMessage()); return false; } - if (Network.log.isFine()) Network.log.fine("connect: updated KNOWN " + ((direct) ? "direct " : "") + peerType + " peer '" + seed.getName() + "' from " + seed.getPublicAddress()); + if (Network.log.isFine()) Network.log.fine("connect: updated KNOWN " + ((direct) ? "direct " : "") + peerType + " peer '" + seed.getName() + "' from " + seed.getIPs()); this.seedDB.addConnected(seed); return true; } @@ -181,10 +182,10 @@ public class PeerActions { if ((this.seedDB.mySeedIsDefined()) && (seed.getIP().equals(this.seedDB.mySeed().getIP()))) { // seed from the same IP as the calling client: can be // the case if there runs another one over a NAT - if (Network.log.isFine()) Network.log.fine("connect: saved NEW seed (myself IP) " + seed.getPublicAddress()); + if (Network.log.isFine()) Network.log.fine("connect: saved NEW seed (myself IP) " + seed.getIPs()); } else { // completely new seed - if (Network.log.isFine()) Network.log.fine("connect: saved NEW " + peerType + " peer '" + seed.getName() + "' from " + seed.getPublicAddress()); + if (Network.log.isFine()) Network.log.fine("connect: saved NEW " + peerType + " peer '" + seed.getName() + "' from " + seed.getIPs()); } this.seedDB.addConnected(seed); return true; @@ -204,7 +205,7 @@ public class PeerActions { public void peerDeparture(final Seed peer, final String cause) { if (peer == null) return; // we do this if we did not get contact with the other peer - if (Network.log.isFine()) Network.log.fine("connect: no contact to a " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_VIRGIN) + " peer '" + peer.getName() + "' at " + peer.getPublicAddress() + ". Cause: " + cause); + if (Network.log.isFine()) Network.log.fine("connect: no contact to a " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_VIRGIN) + " peer '" + peer.getName() + "' at " + peer.getIPs() + ". Cause: " + cause); synchronized (this.seedDB) { if (!this.seedDB.hasDisconnected(ASCII.getBytes(peer.hash))) { this.disconnects++; } peer.put(Seed.DCT, Long.toString(System.currentTimeMillis())); diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java index 0692af33c..e2a79fa6a 100644 --- a/source/net/yacy/peers/Protocol.java +++ b/source/net/yacy/peers/Protocol.java @@ -51,6 +51,7 @@ import java.net.InetAddress; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -128,14 +129,7 @@ import org.apache.solr.common.SolrInputDocument; public final class Protocol { - private static byte[] postToFile( - final Seed target, - final String filename, - final Map parts, - final int timeout) throws IOException { - return postToFile(target.getClusterAddress(), target.hash, filename, parts, timeout); - } - + @Deprecated private static byte[] postToFile( final SeedDB seedDB, final String targetHash, @@ -176,9 +170,8 @@ public final class Protocol { public static int hello( final Seed mySeed, final PeerActions peerActions, - final String address, - final String otherHash, - final String otherName) { + final Seed otherSeed) { + final String address = otherSeed.getPublicAddress(otherSeed.getIP()); Map result = null; final String salt = crypt.randomSalt(); @@ -198,7 +191,7 @@ public final class Protocol { content = httpClient.POSTbytes( new MultiProtocolURL("http://" + address + "/yacy/hello.html"), - Seed.b64Hash2hexHash(otherHash) + ".yacyh", + Seed.b64Hash2hexHash(otherSeed.hash) + ".yacyh", parts, false, true); responseTime = System.currentTimeMillis() - start; @@ -210,12 +203,7 @@ public final class Protocol { + "' interrupted."); return -1; } - Network.log.info("yacyClient.hello thread '" - + Thread.currentThread().getName() - + "', peer " - + address - + "; exception: " - + e.getMessage()); + Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "', peer " + address + "; exception: " + e.getMessage()); // try again (go into loop) result = null; } @@ -238,35 +226,25 @@ public final class Protocol { // check consistency with expectation Seed otherPeer = null; String seed; - if ( (otherHash != null) && (otherHash.length() > 0) && ((seed = result.get("seed0")) != null) ) { + if ( (otherSeed.hash != null) && (otherSeed.hash.length() > 0) && ((seed = result.get("seed0")) != null) ) { if ( seed.length() > Seed.maxsize ) { - Network.log.info("hello/client 0: rejected contacting seed; too large (" - + seed.length() - + " > " - + Seed.maxsize - + ")"); + Network.log.info("hello/client 0: rejected contacting seed; too large (" + seed.length() + " > " + Seed.maxsize + ")"); } else { try { - final int p = address.indexOf(':'); - if ( p < 0 ) { - return -1; - } + // patch the remote peer address to avoid that remote peers spoof the network with wrong addresses + final int p = address.lastIndexOf(':'); + if ( p < 0 ) return -1; String h = address.substring(0, p); + if (h.charAt(0) == '[') h = h.substring(1); + if (h.charAt(h.length() - 1) == ']') h = h.substring(0, h.length() - 1); InetAddress ie = Domains.dnsResolve(h); - final String host = ie == null ? h : ie.getHostAddress(); // hack to prevent NPEs - otherPeer = Seed.genRemoteSeed(seed, false, host); - if ( !otherPeer.hash.equals(otherHash) ) { - Network.log.info("yacyClient.hello: consistency error: otherPeer.hash = " - + otherPeer.hash - + ", otherHash = " - + otherHash); + otherPeer = Seed.genRemoteSeed(seed, false, ie.getHostAddress()); + if ( !otherPeer.hash.equals(otherSeed.hash) ) { + Network.log.info("yacyClient.hello: consistency error: otherPeer.hash = " + otherPeer.hash + ", otherHash = " + otherSeed.hash); return -1; // no success } } catch (final IOException e ) { - Network.log.info("yacyClient.hello: consistency error: other seed bad:" - + e.getMessage() - + ", seed=" - + seed); + Network.log.info("yacyClient.hello: consistency error: other seed bad:" + e.getMessage() + ", seed=" + seed); return -1; // no success } } @@ -281,11 +259,15 @@ public final class Protocol { // set my own seed according to new information // we overwrite our own IP number only if ( serverCore.useStaticIP ) { - mySeed.setIP(Switchboard.getSwitchboard().myPublicIP()); + mySeed.setIPs(Switchboard.getSwitchboard().myPublicIPs()); } else { final String myIP = result.get("yourip"); - final String properIP = Seed.isProperIP(myIP); - if ( properIP == null ) mySeed.setIP(myIP); + // with the IPv6 extension, this may contain several ips, separated by comma ',' + HashSet h = new HashSet<>(); + for (String s: myIP.split(",")) { + if (s.length() > 0 && Seed.isProperIP(s)) h.add(s); + } + if (h.size() > 0) mySeed.setIPs(h); } mySeed.setFlagRootNode( (mytype.equals(Seed.PEERTYPE_SENIOR) || mytype.equals(Seed.PEERTYPE_PRINCIPAL)) && @@ -303,7 +285,7 @@ public final class Protocol { accessible.IWasAccessed = false; } accessible.lastUpdated = System.currentTimeMillis(); - Network.amIAccessibleDB.put(otherHash, accessible); + Network.amIAccessibleDB.put(otherSeed.hash, accessible); /* * If we were reported as junior we have to check if your port forwarding channel is broken @@ -387,52 +369,11 @@ public final class Protocol { final int connectedAfter = peerActions.sizeConnected(); // update event tracker - EventTracker.update(EventTracker.EClass.PEERPING, new ProfilingGraph.EventPing( - mySeed.getName(), - otherName, - true, - connectedAfter - connectedBefore), false); + EventTracker.update(EventTracker.EClass.PEERPING, new ProfilingGraph.EventPing(mySeed.getName(), otherSeed.getName(), true, connectedAfter - connectedBefore), false); return count; } - /* - private int readSeeds(String prefix) { - String seedStr; - while ( (seedStr = result.get("seed" + i++)) != null ) { - // integrate new seed into own database - // the first seed, "seed0" is the seed of the responding peer - if ( seedStr.length() > Seed.maxsize ) { - Network.log.logInfo("hello/client: rejected contacting seed; too large (" - + seedStr.length() - + " > " - + Seed.maxsize - + ")"); - } else { - try { - if ( i == 1 ) { - final int p = address.indexOf(':'); - if ( p < 0 ) { - return -1; - } - InetAddress ia = Domains.dnsResolve(address.substring(0, p)); - if (ia == null) continue; - final String host = ia.getHostAddress(); - s = Seed.genRemoteSeed(seedStr, false, host); - } else { - s = Seed.genRemoteSeed(seedStr, false, null); - } - if ( peerActions.peerArrival(s, (i == 1)) ) { - count++; - } - } catch (final IOException e ) { - Network.log.logInfo("hello/client: rejected contacting seed; bad (" - + e.getMessage() - + ")"); - } - } - } - } -*/ + public static Seed querySeed(final Seed target, final String seedHash) { // prepare request final String salt = crypt.randomSalt(); @@ -443,7 +384,7 @@ public final class Protocol { basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt); parts.put("object", UTF8.StringBody("seed")); parts.put("env", UTF8.StringBody(seedHash)); - final byte[] content = postToFile(target, "query.html", parts, 10000); + final byte[] content = postToFile(target.getPublicAddress(target.getIP()), target.hash, "query.html", parts, 10000); final Map result = FileUtils.table(content); if ( result == null || result.isEmpty() ) { @@ -457,22 +398,24 @@ public final class Protocol { } } - public static long[] queryRWICount(final Seed target, final String wordHash) { - if (target == null) return new long[] {-1, -1}; - + public static long[] queryRWICount(final String targetAddress, final String targetHash, int timeout) { // prepare request final String salt = crypt.randomSalt(); // send request try { - final Map parts = basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt); + final Map parts = basicRequestParts(Switchboard.getSwitchboard(), targetHash, salt); parts.put("object", UTF8.StringBody("rwicount")); parts.put("ttl", UTF8.StringBody("0")); - parts.put("env", UTF8.StringBody(wordHash)); - final byte[] content = postToFile(target, "query.html", parts, 6000); + parts.put("env", UTF8.StringBody("")); + ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "posting request to " + targetAddress); + final byte[] content = postToFile(targetAddress, targetHash, "query.html", parts, timeout); + ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received CONTENT from requesting " + targetAddress + (content == null ? "NULL" : (": length = " + content.length))); final Map result = FileUtils.table(content); if (result == null || result.isEmpty()) return new long[] {-1, -1}; + ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESULT from requesting " + targetAddress + " : result = " + result.toString()); final String resp = result.get("response"); + ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESPONSE from requesting " + targetAddress + " : response = " + resp); if (resp == null) return new long[] {-1, -1}; String magic = result.get("magic"); if (magic == null) magic = "0"; @@ -482,6 +425,7 @@ public final class Protocol { return new long[] {-1, -1}; } } catch (final Exception e ) { + ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received EXCEPTION from requesting " + targetAddress + ": " + e.getMessage()); if (Network.log.isFine()) Network.log.fine("yacyClient.queryRWICount error:" + e.getMessage()); return new long[] {-1, -1}; } @@ -516,10 +460,7 @@ public final class Protocol { parts.put("time", UTF8.StringBody(Long.toString(maxTime))); // final byte[] result = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/urls.xml"), (int) maxTime, target.getHexHash() + ".yacyh", parts); final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, (int) maxTime); - final byte[] result = - httpClient.POSTbytes(new MultiProtocolURL("http://" - + target.getClusterAddress() - + "/yacy/urls.xml"), target.getHexHash() + ".yacyh", parts, false, true); + final byte[] result = httpClient.POSTbytes(new MultiProtocolURL("http://" + target.getPublicAddress(target.getIP()) + "/yacy/urls.xml"), target.getHexHash() + ".yacyh", parts, false, true); final RSSReader reader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result); if ( reader == null ) { Network.log.warn("yacyClient.queryRemoteCrawlURLs failed asking peer '" @@ -586,8 +527,8 @@ public final class Protocol { final long timestamp = System.currentTimeMillis(); event.addExpectedRemoteReferences(count); SearchResult result; - String clusteraddress = target.getClusterAddress(); - if (clusteraddress.equals(event.peers.mySeed().getClusterAddress())) clusteraddress = "localhost:" + event.peers.mySeed().getPort(); + String clusteraddress = target.getPublicAddress(target.getIP()); + if (target.clash(event.peers.mySeed().getIPs())) clusteraddress = "localhost:" + event.peers.mySeed().getPort(); try { result = new SearchResult( @@ -680,7 +621,7 @@ public final class Protocol { maxDistance, partitions, target.getHexHash() + ".yacyh", - target.getClusterAddress(), + target.getPublicAddress(), null ); } catch (final IOException e ) { @@ -1033,7 +974,7 @@ public final class Protocol { Network.log.info("SEARCH skip (solr), remote Solr interface not accessible, peer=" + target.getName()); return -1; } - final String address = myseed ? "localhost:" + target.getPort() : target.getPublicAddress(); + final String address = myseed ? "localhost:" + target.getPort() : target.getPublicAddress(target.getIP()); final int solrtimeout = Switchboard.getSwitchboard().getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 6000); Thread remoteRequest = new Thread() { @Override @@ -1285,7 +1226,7 @@ public final class Protocol { final String salt = crypt.randomSalt(); // determining target address - final String address = target.getClusterAddress(); + final String address = target.getPublicAddress(target.getIP()); if ( address == null ) { return null; } @@ -1435,7 +1376,7 @@ public final class Protocol { final ReferenceContainerCache indexes, boolean gzipBody, final int timeout) { - final String address = targetSeed.getPublicAddress(); + final String address = targetSeed.getPublicAddress(targetSeed.getIP()); if ( address == null ) { Network.log.warn("no address for transferRWI"); return null; @@ -1507,7 +1448,7 @@ public final class Protocol { boolean gzipBody, final int timeout) { // this post a message to the remote message board - final String address = targetSeed.getPublicAddress(); + final String address = targetSeed.getPublicAddress(targetSeed.getIP()); if ( address == null ) { return null; } @@ -1577,7 +1518,7 @@ public final class Protocol { // this post a message to the remote message board final String salt = crypt.randomSalt(); - String address = targetSeed.getClusterAddress(); + String address = targetSeed.getPublicAddress(targetSeed.getIP()); if ( address == null ) { address = "localhost:8090"; } @@ -1619,7 +1560,7 @@ public final class Protocol { final Map parts = basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt); parts.put("object", UTF8.StringBody("host")); - final byte[] content = postToFile(target, "idx.json", parts, 30000); + final byte[] content = postToFile(target.getPublicAddress(target.getIP()), target.hash, "idx.json", parts, 30000); if ( content == null || content.length == 0 ) { Network.log.warn("yacyClient.loadIDXHosts error: empty result"); return null; @@ -1702,64 +1643,45 @@ public final class Protocol { return false; } - private static final LinkedHashMap basicRequestParts( - final Switchboard sb, - final String targetHash, - final String salt) { - // put in all the essentials for routing and network authentication - // generate a session key - final LinkedHashMap parts = - basicRequestParts( - sb.peers.mySeed().hash, - targetHash, - Switchboard.getSwitchboard().getConfig( - SwitchboardConstants.NETWORK_NAME, - Seed.DFLT_NETWORK_UNIT)); - parts.put("key", UTF8.StringBody(salt)); - - // authentication essentials - final String authenticationControl = sb.getConfig("network.unit.protocol.control", "uncontrolled"); - final String authenticationMethod = - sb.getConfig("network.unit.protocol.request.authentication.method", ""); - if ( (authenticationControl.equals("controlled")) && (authenticationMethod.length() > 0) ) { - if ( authenticationMethod.equals("salted-magic-sim") ) { - // generate an authentication essential using the salt, the iam-hash and the network magic - final String magic = - sb.getConfig("network.unit.protocol.request.authentication.essentials", ""); - final String md5 = Digest.encodeMD5Hex(salt + sb.peers.mySeed().hash + magic); - parts.put("magicmd5", UTF8.StringBody(md5)); - } - } - - return parts; - } - - private static final LinkedHashMap basicRequestParts( - final String myHash, - final String targetHash, - final String networkName) { - // put in all the essentials for routing and network authentication - // generate a session key + /** + * put in all the essentials for routing and network authentication + * @param sb + * @param targetHash + * @param salt + * @return + */ + private static final LinkedHashMap basicRequestParts(final Switchboard sb, final String targetHash, final String salt) { final LinkedHashMap parts = new LinkedHashMap(); - + // just standard identification essentials - if ( myHash != null ) { - parts.put("iam", UTF8.StringBody(myHash)); - if ( targetHash != null ) { - parts.put("youare", UTF8.StringBody(targetHash)); - } - + if ( sb.peers.mySeed().hash != null ) { + parts.put("iam", UTF8.StringBody(sb.peers.mySeed().hash)); + if ( targetHash != null ) parts.put("youare", UTF8.StringBody(targetHash)); + // time information for synchronization // use our own formatter to prevent concurrency locks with other processes - final GenericFormatter my_SHORT_SECOND_FORMATTER = - new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); + final GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); parts.put("mytime", UTF8.StringBody(my_SHORT_SECOND_FORMATTER.format())); parts.put("myUTC", UTF8.StringBody(Long.toString(System.currentTimeMillis()))); - + // network identification - parts.put(SwitchboardConstants.NETWORK_NAME, UTF8.StringBody(networkName)); + parts.put(SwitchboardConstants.NETWORK_NAME, UTF8.StringBody(Switchboard.getSwitchboard().getConfig( + SwitchboardConstants.NETWORK_NAME, + Seed.DFLT_NETWORK_UNIT))); } + parts.put("key", UTF8.StringBody(salt)); + // authentication essentials + final String authenticationControl = sb.getConfig("network.unit.protocol.control", "uncontrolled"); + final String authenticationMethod = sb.getConfig("network.unit.protocol.request.authentication.method", ""); + if ((authenticationControl.equals("controlled")) && (authenticationMethod.length() > 0) ) { + if (authenticationMethod.equals("salted-magic-sim") ) { + // generate an authentication essential using the salt, the iam-hash and the network magic + final String magic = sb.getConfig("network.unit.protocol.request.authentication.essentials", ""); + final String md5 = Digest.encodeMD5Hex(salt + sb.peers.mySeed().hash + magic); + parts.put("magicmd5", UTF8.StringBody(md5)); + } + } return parts; } diff --git a/source/net/yacy/peers/RemoteSearch.java b/source/net/yacy/peers/RemoteSearch.java index facd3a25b..f74de4609 100644 --- a/source/net/yacy/peers/RemoteSearch.java +++ b/source/net/yacy/peers/RemoteSearch.java @@ -30,7 +30,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.Random; import java.util.Set; -import java.util.SortedMap; +import java.util.SortedSet; import org.apache.solr.client.solrj.SolrQuery; @@ -144,7 +144,7 @@ public class RemoteSearch extends Thread { final int start, final int count, final long time, final Blacklist blacklist, - final SortedMap clusterselection) { + final SortedSet clusterselection) { // check own peer status //if (wordIndex.seedDB.mySeed() == null || wordIndex.seedDB.mySeed().getPublicAddress() == null) { return null; } @@ -277,14 +277,13 @@ public class RemoteSearch extends Thread { final Blacklist blacklist) { // check own peer status - if (event.peers.mySeed() == null || event.peers.mySeed().getPublicAddress() == null) { return null; } + if (event.peers.mySeed() == null || event.peers.mySeed().getIPs().size() == 0) { return null; } assert urlhashes != null; assert urlhashes.length() > 0; // prepare seed targets and threads final Seed targetPeer = event.peers.getConnected(targethash); if (targetPeer == null || targetPeer.hash == null) return null; - if (event.preselectedPeerHashes != null) targetPeer.setAlternativeAddress(event.preselectedPeerHashes.get(ASCII.getBytes(targetPeer.hash))); Thread secondary = new Thread() { @Override public void run() { @@ -332,12 +331,8 @@ public class RemoteSearch extends Thread { assert solrQuery != null; // check own peer status - if (event.peers.mySeed() == null || event.peers.mySeed().getPublicAddress() == null) { return null; } - // prepare seed targets and threads - if (targetPeer != null && targetPeer.hash != null && event.preselectedPeerHashes != null) { - if (!targetPeer.getFlagSolrAvailable()) return null; // solr interface not avail. - targetPeer.setAlternativeAddress(event.preselectedPeerHashes.get(ASCII.getBytes(targetPeer.hash))); - } + if (event.peers.mySeed() == null || event.peers.mySeed().getIPs().size() == 0) { return null; } + // prepare threads Thread solr = new Thread() { @Override public void run() { diff --git a/source/net/yacy/peers/Seed.java b/source/net/yacy/peers/Seed.java index b7a31df9f..07ba8d5c9 100644 --- a/source/net/yacy/peers/Seed.java +++ b/source/net/yacy/peers/Seed.java @@ -44,13 +44,16 @@ import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; +import java.net.Inet6Address; import java.net.InetAddress; import java.net.MalformedURLException; import java.net.URL; import java.text.ParseException; import java.util.Comparator; import java.util.Date; +import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.Random; import java.util.Set; import java.util.TreeMap; @@ -132,6 +135,7 @@ public class Seed implements Cloneable, Comparable, Comparator private static final String IPTYPE = "IPType"; private static final String FLAGS = "Flags"; public static final String FLAGSZERO = " "; + /** the applications version */ public static final String VERSION = "Version"; @@ -142,31 +146,52 @@ public class Seed implements Cloneable, Comparable, Comparator /** the name of the peer (user-set) */ public static final String NAME = "Name"; public static final String HASH = "Hash"; + /** Birthday - first startup */ private static final String BDATE = "BDate"; + /** UTC-Offset */ public static final String UTC = "UTC"; private static final String PEERTAGS = "Tags"; /** the speed of indexing (pages/minute) of the peer */ public static final String ISPEED = "ISpeed"; + /** the speed of retrieval (queries/minute) of the peer */ public static final String RSPEED = "RSpeed"; + /** the number of minutes that the peer is up in minutes/day (moving average MA30) */ public static final String UPTIME = "Uptime"; + /** the number of links that the peer has stored (LURL's) */ public static final String LCOUNT = "LCount"; + /** the number of links that the peer has noticed, but not loaded (NURL's) */ public static final String NCOUNT = "NCount"; + /** the number of links that the peer provides for remote crawls (ZURL's) */ public static final String RCOUNT = "RCount"; + /** the number of different words the peer has indexed */ public static final String ICOUNT = "ICount"; + /** the number of seeds that the peer has stored */ public static final String SCOUNT = "SCount"; + /** the number of clients that the peer connects (connects/hour as double) */ public static final String CCOUNT = "CCount"; + + /** the public IP of this peer (old field, will be used to carry the IPv4) */ public static final String IP = "IP"; + + /** more public IPs of this peer, containing only IPv6 addresses. This list of of IPv6 addresses is separated with a vertical bar/pipe '|'. + * This list may have zero entries if the host does not have a IPv6 address. It may have more than one IPv6 address if the + * Host detects more than one public IPv6 locally but did not get a feedback from other peers if any of these addresses are + * reachable. The list may contain only one address, if a 'hello' with backping to another peer was successful and the other peer + * peer confirmed one of the IPv6 IPs which is then the only entry in this field. + */ + public static final String IP6 = "IP6"; + public static final String PORT = "Port"; public static final String SEEDLISTURL = "seedURL"; public static final String NEWS = "news"; // news attachment @@ -192,9 +217,8 @@ public class Seed implements Cloneable, Comparable, Comparator public final String hash; /** a set of identity founding values, eg. IP, name of the peer, YaCy-version, ... */ private final ConcurrentMap dna; - private String alternativeIP = null; private long birthdate; // keep this value in ram since it is often used and may cause lockings in concurrent situations. - + public Seed(final String theHash, final ConcurrentMap theDna) { // create a seed with a pre-defined hash map assert theHash != null; @@ -297,36 +321,87 @@ public class Seed implements Cloneable, Comparable, Comparator } /** - * used when doing routing within a cluster; this can assign a ip and a port that is used instead the - * address stored in the seed DNA + * try to get the public IP
+ * + * @return the IP or localhost IP (127.0.0.1) */ - public void setAlternativeAddress(final String ipport) { - if ( ipport == null ) { - return; + @Deprecated + public final String getIP() { + final String ipx = this.dna.get(Seed.IP); // may contain both, IPv4 or IPv6 + final String ip6 = this.dna.get(Seed.IP6); + Set ip6s = MapTools.string2set(ip6, "|"); + + if (ip6s == null || ip6s.size() == 0) { + if (ipx != null && !ipx.isEmpty()) return chopZoneID(ipx); } - final int p = ipport.lastIndexOf(':'); - if ( p < 0 ) { - this.alternativeIP = ipport; - } else { - this.alternativeIP = ipport.substring(0, p); + if (ip6s != null && ip6s.size() == 1) { + // We prefer IPv6 + for (String s: ip6s) if (s.length() > 0) return chopZoneID(s); + if (ipx != null && !ipx.isEmpty()) return chopZoneID(ipx); } + + // if we have more than one IPv6, then chances are high that one of them do not work. + // in that case we prefer the IPv4 + if (ipx != null && !ipx.isEmpty()) return chopZoneID(ipx); + if (ip6s != null) for (String s: ip6s) if (s.length() > 0) return chopZoneID(s); + + // in case that we don't have any address using the dna (i.e. a fresh peer), then use all locally known addresses + for (InetAddress i: Domains.myPublicIPv4()) return chopZoneID(i.getHostAddress()); + for (InetAddress i: Domains.myPublicIPv6()) return chopZoneID(i.getHostAddress()); - if (this.alternativeIP.charAt(0) == '[' && this.alternativeIP.charAt(this.alternativeIP.length() - 1) == ']') { - // IPv6 patch - this.alternativeIP = this.alternativeIP.substring(1, this.alternativeIP.length() - 1); - } + // final chance + return Domains.LOCALHOST; } /** - * try to get the public IP
- * - * @return the IP or localhost IP (127.0.0.1) + * Get all my public IPs. If there was a static IP assignment, only one, that IP is returned. + * If no feedback from other peers exist, then all locally determined IPs are returned. + * If a feedback from other peers exist, then return at most two IPs: + * the latest IPv4 and the latest IPv6 which was returned during a hello process from a remote peer + * @return a set of IPs which are supposed to be my own public IPs */ - public final String getIP() { - final String ip = this.dna.get(Seed.IP); - return (ip == null || ip.isEmpty()) ? Domains.LOCALHOST : ip; // not public (but leave as is for now 2014-08-24) + public final Set getIPs() { + Set h = new LinkedHashSet<>(); + final String ipx = this.dna.get(Seed.IP); // may contain both, IPv4 or IPv6 + final String ip6 = this.dna.get(Seed.IP6); + Set ip6s = MapTools.string2set(ip6, "|"); + + if (ip6s == null || ip6s.size() == 0) { + if (ipx != null && !ipx.isEmpty()) h.add(chopZoneID(ipx)); + } else if (ip6s != null && ip6s.size() == 1) { + // We add IPv6 first because then those addresses appear first + // in the LinkedHashSet and are preferred by methods using only the first one. + for (String s: ip6s) if (s.length() > 0) h.add(chopZoneID(s)); + if (ipx != null && !ipx.isEmpty()) h.add(chopZoneID(ipx)); + } else { + // if we have more than one IPv6, then chances are high that one of them do not work. + // in that case we prefer the IPv4 + if (ipx != null && !ipx.isEmpty()) h.add(chopZoneID(ipx)); + if (ip6s != null) for (String s: ip6s) if (s.length() > 0) h.add(chopZoneID(s)); + } + + // in case that we don't have any address using the dna (i.e. a fresh peer), then use all locally known addresses + if (h.size() == 0) { + for (InetAddress i: Domains.myPublicIPv4()) h.add(chopZoneID(i.getHostAddress())); + for (InetAddress i: Domains.myPublicIPv6()) h.add(chopZoneID(i.getHostAddress())); + h.add(Domains.LOCALHOST); + } + return h; + } + + private String chopZoneID(String ip) { + int i = ip.indexOf('%'); + return i < 0 ? ip : ip.substring(0, i); } + public boolean clash(Set ips) { + Set myIPs = getIPs(); + for (String s: ips) { + if (myIPs.contains(s) && isProperIP(s)) return true; + } + return false; + } + /** * try to get the peertype
* @@ -423,9 +498,55 @@ public class Seed implements Cloneable, Comparable, Comparator return dflt; } } - + + /** + * set the Peer ip. + * This sets the IP and IP6 field according to the current fill state of that fields: + * - if no field has a content, then IP is filled with the given ip, even if that ip is of type IPv6 + * - if IP is already set then check if this is equivalent with the given ip. If both are equal, nothing is done. + * If they are not equal, the IP and IPv6 field is set according to the type if the given ip: if the given ip + * is of type IPv4, then IP is set with ip, otherwise IP6 is set with the ip. + * ATTENTION: if the given IP is IPv6, then after the call that IP is the only one assigned to the peer! + * @param ip + */ public final void setIP(final String ip) { - this.dna.put(Seed.IP, ip); + if (!isProperIP(ip)) return; + String oldIP = this.dna.get(Seed.IP); + String oldIP6 = this.dna.get(Seed.IP6); + if ((oldIP == null || oldIP.length() == 0) && (oldIP6 == null || oldIP6.length() == 0)) { + this.dna.put(Seed.IP, ip); + } else { + if (oldIP == null || !oldIP.equals(ip)) { + if (oldIP == null || oldIP.length() == 0 || ip.indexOf(':') == 0) this.dna.put(Seed.IP, ip); else this.dna.put(Seed.IP6, ip); + } + } + } + + /** + * Set several local IPs which are good to access this peer. + * This OVERWRITES ALL ips stored before! + * @param ips list of IPs + */ + public final void setIPs(final Set ips) { + // we must sort IPv4 and IPv6 here + Set ipv6 = new HashSet<>(); + Set ipv4 = new HashSet<>(); + for (String ip: ips) if (isProperIP(ip)) { + if (ip.indexOf(':') >= 0) ipv6.add(ip); else ipv4.add(ip); + } + if (ipv4.size() == 0) { + if (ipv6.size() == 1) { + // if the only IP we have is IPv6, then put this into IP field + this.dna.put(Seed.IP, ipv6.iterator().next()); + this.dna.put(Seed.IP6, ""); + } else if (ipv6.size() > 1) { + this.dna.put(Seed.IP, ""); + this.dna.put(Seed.IP6, MapTools.set2string(ipv6, "|", false)); + } + } else { + this.dna.put(Seed.IP, ipv4.iterator().next()); + this.dna.put(Seed.IP6, MapTools.set2string(ipv6, "|", false)); + } } public final void setPort(final String port) { @@ -575,78 +696,65 @@ public class Seed implements Cloneable, Comparable, Comparator } /** + * deprecated, use getIPs() instead * @return the public address of the peer as IP:port string or null if no valid values for * either the IP or the port could be retrieved from this yacySeed object */ + @Deprecated public final String getPublicAddress() { - String ip = getIP(); - if (ip == null) ip = Domains.LOCALHOST; // that should not happen - - int p = ip.lastIndexOf(':'); - if (p > 0 && (ip.indexOf(':') == p || "]:".equals(ip.substring(p - 1, p + 1)))) return ip; // includes already the port - - final String port = this.dna.get(Seed.PORT); + return getPublicAddress(getIP()); + } + + /** + * generate a public address using a given ip. This combines the ip with the port and encloses the ip + * with square brackets if the ip is of typeIPv6 + * @param ip + * @return an address string which can be used as host:port part of an url + */ + public final String getPublicAddress(final InetAddress ip) { + // we do not use getPublicAddress(String ip) here to be able to check IPv6 with instanceof Inet6Address which is faster than indexOf(':') + if (ip == null) throw new RuntimeException("ip == NULL"); // that should not happen + final String port = this.dna.get(Seed.PORT); // we do not use getPort() here to avoid String->Integer->toString() conversion if ( port == null || port.length() < 2 || port.length() > 5 ) { - return null; + throw new RuntimeException("port not wellformed: " + port); // that should not happen } - - final StringBuilder sb = new StringBuilder(ip.length() + port.length() + 3); - if (ip.indexOf(':') >= 0) { - // IPv6 Address!, see: http://en.wikipedia.org/wiki/IPv6_address#Literal_IPv6_addresses_in_network_resource_identifiers - if (!ip.startsWith("[")) sb.append('['); - sb.append(ip); - if (!ip.endsWith("]")) sb.append(']'); - sb.append(':'); - sb.append(port); + final StringBuilder sb = new StringBuilder(); + if (ip instanceof Inet6Address) { + sb.append('[').append(ip.getHostAddress()).append(']'); } else { - sb.append(ip); - sb.append(':'); - sb.append(port); + sb.append(ip.getHostAddress()); } + sb.append(':'); + sb.append(port); + return sb.toString(); } - + /** - * If this seed is part of a cluster, the peer has probably the {@linkplain #alternativeIP} object set to - * a local IP. If this is present and the public IP of this peer is identical to the public IP of the own - * seed, construct an address using this IP; otherwise return the public address - * - * @see #getPublicAddress() - * @return the alternative IP:port if present, else the public address + * generate a public address using a given ip. This combines the ip with the port and encloses the ip + * with square brackets if the ip is of typeIPv6 + * @param ip + * @return an address string which can be used as host:port part of an url */ - public final String getClusterAddress() { - if ( this.alternativeIP == null ) { - return getPublicAddress(); - } - - final String port = this.dna.get(Seed.PORT); + public final String getPublicAddress(final String ip) { + if (ip == null) throw new RuntimeException("ip == NULL"); // that should not happen + final String port = this.dna.get(Seed.PORT); // we do not use getPort() here to avoid String->Integer->toString() conversion if ( port == null || port.length() < 2 || port.length() > 5 ) { - return null; + throw new RuntimeException("port not wellformed: " + port); // that should not happen } - - final StringBuilder sb = new StringBuilder(this.alternativeIP.length() + port.length() + 3); - if (this.alternativeIP.indexOf(':') >= 0) { - // IPv6 Address!, see: http://en.wikipedia.org/wiki/IPv6_address#Literal_IPv6_addresses_in_network_resource_identifiers - sb.append('['); - sb.append(this.alternativeIP); - sb.append(']'); - sb.append(':'); - sb.append(port); + final StringBuilder sb = new StringBuilder(ip.length() + port.length() + 3); + if (ip.indexOf(':') >= 0) { + if (!ip.startsWith("[")) sb.append('['); + sb.append(ip); + if (!ip.endsWith("]")) sb.append(']'); } else { - sb.append(this.alternativeIP); - sb.append(':'); - sb.append(port); + sb.append(ip); } + sb.append(':'); + sb.append(port); return sb.toString(); } - /** - * @return the IP address of the peer represented by this yacySeed object as {@link InetAddress} - */ - public final InetAddress getInetAddress() { - return Domains.dnsResolve(getIP()); - } - /** @return the port number of this seed or -1 if not present */ public final int getPort() { final String port = this.dna.get(Seed.PORT); @@ -662,12 +770,8 @@ public class Seed implements Cloneable, Comparable, Comparator // because java thinks it must apply the UTC offset to the current time, // to create a string that looks like our current time, it adds the local UTC offset to the // time. To create a corrected UTC Date string, we first subtract the local UTC offset. - final GenericFormatter my_SHORT_SECOND_FORMATTER = - new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); // use our own formatter to prevent concurrency locks with other processes - final String ls = - my_SHORT_SECOND_FORMATTER - .format(new Date(System.currentTimeMillis() /*- DateFormatter.UTCDiff()*/)); - //System.out.println("SETTING LAST-SEEN of " + this.getName() + " to " + ls); + final GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); // use our own formatter to prevent concurrency locks with other processes + final String ls = my_SHORT_SECOND_FORMATTER.format(new Date(System.currentTimeMillis() /*- DateFormatter.UTCDiff()*/)); this.dna.put(Seed.LASTSEEN, ls); } @@ -1101,10 +1205,8 @@ public class Seed implements Cloneable, Comparable, Comparator // check IP if ( !checkOwnIP ) { // checking of IP is omitted if we read the own seed file - final String ipCheck = isProperIP(getIP()); - if ( ipCheck != null ) { - return ipCheck; - } + final String ip = getIP(); + if (!isProperIP(ip)) return "not a proper IP " + ip; } // seedURL @@ -1134,23 +1236,19 @@ public class Seed implements Cloneable, Comparable, Comparator return null; } - public static final String isProperIP(final String ipString) { - // returns null if ipString is proper, a string with the cause otherwise - if ( ipString == null ) { - return ipString + " -> IP is null"; - } - if ( ipString.length() < 8 ) { - return ipString + " -> IP is too short: "; - } - if ( Switchboard.getSwitchboard().isAllIPMode() ) { - return null; - } + /** + * check if the given string containing an IP is proper. This checks also if the IP is within the given + * range of the network definition + * @param ipString + * @return true iff the IP is proper + */ + public static final boolean isProperIP(final String ipString) { + if (ipString == null) return false; + if (ipString.length() < 3) return false; + if (Switchboard.getSwitchboard().isAllIPMode()) return true; // accept everyting final boolean islocal = Domains.isLocal(ipString, null); //if (islocal && Switchboard.getSwitchboard().isGlobalMode()) return ipString + " - local IP for global mode rejected"; - if ( !islocal && Switchboard.getSwitchboard().isIntranetMode() ) { - return ipString + " - global IP for intranet mode rejected"; - } - return null; + return islocal == Switchboard.getSwitchboard().isIntranetMode(); } @Override diff --git a/source/net/yacy/peers/SeedDB.java b/source/net/yacy/peers/SeedDB.java index 295663643..d20a2d304 100644 --- a/source/net/yacy/peers/SeedDB.java +++ b/source/net/yacy/peers/SeedDB.java @@ -35,7 +35,8 @@ import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.TreeMap; +import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -208,11 +209,7 @@ public final class SeedDB implements AlternativeDomainNames { System.exit(-1); } } - if (Switchboard.getSwitchboard().isIntranetMode()) { - this.mySeed.setIP(Domains.myPublicLocalIP().getHostAddress()); // in intranet mode host address is best choice (to become senior peer) - } else { - this.mySeed.setIP("");// we delete the old information to see what we have now - } + this.mySeed.setIPs(Switchboard.getSwitchboard().myPublicIPs()); this.mySeed.put(Seed.PEERTYPE, Seed.PEERTYPE_VIRGIN); // markup startup condition } @@ -229,10 +226,6 @@ public final class SeedDB implements AlternativeDomainNames { if (this.mySeed == null) { if (sizeConnected() == 0) try {Thread.sleep(5000);} catch (final InterruptedException e) {} // wait for init initMySeed(); - // check if my seed has an IP assigned - if (myIP() == null || myIP().isEmpty()) { - this.mySeed.setIP(Domains.myPublicLocalIP().getHostAddress()); - } } return this.mySeed; } @@ -246,10 +239,15 @@ public final class SeedDB implements AlternativeDomainNames { return mySeed().getName() + ".yacy"; } - @Override + @Deprecated public String myIP() { return mySeed().getIP(); } + + @Override + public Set myIPs() { + return mySeed().getIPs(); + } @Override public int myPort() { @@ -349,7 +347,7 @@ public final class SeedDB implements AlternativeDomainNames { return new seedEnum(up, field, this.seedPotentialDB); } - public TreeMap /* peer-b64-hashes/ipport */ clusterHashes(final String clusterdefinition) { + public TreeSet /* peer-b64-hashes/ipport */ clusterHashes(final String clusterdefinition) { // collects seeds according to cluster definition string, which consists of // comma-separated .yacy or .yacyh-domains // the domain may be extended by an alternative address specification of the form @@ -359,18 +357,16 @@ public final class SeedDB implements AlternativeDomainNames { // address ::= ('.yacy'|'.yacyh'){'='{':' clustermap = new TreeMap(Base64Order.enhancedCoder); + final TreeSet clustermap = new TreeSet<>(Base64Order.enhancedCoder); Seed seed; - String hash, yacydom, ipport; + String hash, yacydom; int p; for (final String addresse : addresses) { p = addresse.indexOf('='); if (p >= 0) { yacydom = addresse.substring(0, p); - ipport = addresse.substring(p + 1); } else { yacydom = addresse; - ipport = null; } if (yacydom.endsWith(".yacyh")) { // find a peer with its hexhash @@ -379,7 +375,7 @@ public final class SeedDB implements AlternativeDomainNames { if (seed == null) { Network.log.warn("cluster peer '" + yacydom + "' was not found."); } else { - clustermap.put(ASCII.getBytes(hash), ipport); + clustermap.add(ASCII.getBytes(hash)); } } else if (yacydom.endsWith(".yacy")) { // find a peer with its name @@ -387,7 +383,7 @@ public final class SeedDB implements AlternativeDomainNames { if (seed == null) { Network.log.warn("cluster peer '" + yacydom + "' was not found."); } else { - clustermap.put(ASCII.getBytes(seed.hash), ipport); + clustermap.add(ASCII.getBytes(seed.hash)); } } else { Network.log.warn("cluster peer '" + addresse + "' has wrong syntax. the name must end with .yacy or .yacyh"); @@ -732,9 +728,7 @@ public final class SeedDB implements AlternativeDomainNames { } // check local seed - if (this.mySeed == null) return null; - String s = this.mySeed.getIP(); - if (s == null || !ipString.equals(s)) return null; + if (this.mySeed == null || !this.mySeed.getIPs().contains(ipString)) return null; int p = this.mySeed.getPort(); if (port > 0 && p != port) return null; //System.out.println("*** found lookupByIP as my seed: " + peerIP.toString() + " -> " + this.mySeed.getName()); @@ -915,7 +909,7 @@ public final class SeedDB implements AlternativeDomainNames { seed = this.mySeed; else return null; } - return seed.getPublicAddress() + ((subdom == null) ? "" : ("/" + subdom)); + return seed.getPublicAddress(seed.getIP()) + ((subdom == null) ? "" : ("/" + subdom)); } else if (host.endsWith(".yacy")) { // identify subdomain p = host.indexOf('.'); @@ -932,7 +926,7 @@ public final class SeedDB implements AlternativeDomainNames { // take local ip instead of external return Switchboard.getSwitchboard().myPublicIP() + ":" + Switchboard.getSwitchboard().getConfig("port", "8090") + ((subdom == null) ? "" : ("/" + subdom)); } - return seed.getPublicAddress() + ((subdom == null) ? "" : ("/" + subdom)); + return seed.getPublicAddress(seed.getIP()) + ((subdom == null) ? "" : ("/" + subdom)); } else { return null; } @@ -942,11 +936,11 @@ public final class SeedDB implements AlternativeDomainNames { // find target address String address; if (targetHash.equals(mySeed().hash)) { - address = mySeed().getClusterAddress(); + address = mySeed().getPublicAddress(mySeed().getIP()); } else { final Seed targetSeed = getConnected(targetHash); if (targetSeed == null) { return null; } - address = targetSeed.getClusterAddress(); + address = targetSeed.getPublicAddress(targetSeed.getIP()); } if (address == null) address = "localhost" + (this.mySeed.getPort() > 0 ? ":" + this.mySeed.getPort() : ""); return address; diff --git a/source/net/yacy/peers/Transmission.java b/source/net/yacy/peers/Transmission.java index 970c4532d..65662f295 100644 --- a/source/net/yacy/peers/Transmission.java +++ b/source/net/yacy/peers/Transmission.java @@ -268,8 +268,7 @@ public class Transmission { // get possibly newer target Info final Seed newTarget = Transmission.this.seeds.get(this.dhtTarget.hash); if (newTarget != null) { - final String oldAddress = this.dhtTarget.getPublicAddress(); - if ((oldAddress != null) && (oldAddress.equals(newTarget.getPublicAddress()))) { + if (this.dhtTarget.clash(newTarget.getIPs())) { newTarget.setFlagAcceptRemoteIndex(false); Transmission.this.seeds.update(newTarget.hash, newTarget); } else { diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index e9c783eca..c809727c6 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -64,7 +64,6 @@ import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; -import java.util.SortedMap; import java.util.SortedSet; import java.util.TreeMap; import java.util.TreeSet; @@ -265,7 +264,7 @@ public final class Switchboard extends serverSwitch { public int searchQueriesRobinsonFromLocal = 0; // absolute counter of all local queries submitted on this peer from a local or autheticated used public int searchQueriesRobinsonFromRemote = 0; // absolute counter of all local queries submitted on this peer from a remote IP without authentication public float searchQueriesGlobal = 0f; // partial counter of remote queries (1/number-of-requested-peers) - public SortedMap clusterhashes; // map of peerhash(String)/alternative-local-address as ip:port or only ip (String) or null if address in seed should be used + public SortedSet clusterhashes; // a set of cluster hashes public List networkWhitelist, networkBlacklist; public FilterEngine domainList; private Dispatcher dhtDispatcher; @@ -1592,7 +1591,7 @@ public final class Switchboard extends serverSwitch { getConfig(SwitchboardConstants.CLUSTER_MODE, SwitchboardConstants.CLUSTER_MODE_PUBLIC_PEER); if ( clustermode.equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER) ) { // check if we got the request from a peer in the public cluster - return this.clusterhashes.containsKey(ASCII.getBytes(peer)); + return this.clusterhashes.contains(ASCII.getBytes(peer)); } return false; } @@ -1610,7 +1609,7 @@ public final class Switchboard extends serverSwitch { getConfig(SwitchboardConstants.CLUSTER_MODE, SwitchboardConstants.CLUSTER_MODE_PUBLIC_PEER); if ( clustermode.equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER) ) { // check if we got the request from a peer in the public cluster - return this.clusterhashes.containsKey(ASCII.getBytes(seed.hash)); + return this.clusterhashes.contains(ASCII.getBytes(seed.hash)); } return false; } @@ -2888,9 +2887,6 @@ public final class Switchboard extends serverSwitch { if ( (processCase == EventOrigin.GLOBAL_CRAWLING) && (queueEntry.initiator() != null) ) { final Seed initiatorPeer = this.peers.get(ASCII.String(queueEntry.initiator())); if ( initiatorPeer != null ) { - if ( this.clusterhashes != null ) { - initiatorPeer.setAlternativeAddress(this.clusterhashes.get(queueEntry.initiator())); - } // start a thread for receipt sending to avoid a blocking here SolrDocument sd = this.index.fulltext().getDefaultConfiguration().toSolrDocument(newEntry); new Thread(new receiptSending(initiatorPeer, new URIMetadataNode(sd)), "sending receipt to " + ASCII.String(queueEntry.initiator())).start(); @@ -3712,6 +3708,11 @@ public final class Switchboard extends serverSwitch { mySeed.setFlagAcceptRemoteCrawl(getConfigBool("crawlResponse", true)); mySeed.setFlagAcceptRemoteIndex(getConfigBool("allowReceiveIndex", true)); mySeed.setFlagSSLAvailable(this.getHttpServer() != null && this.getHttpServer().withSSL() && getConfigBool("server.https", false)); + + // set local ips + String staticIP = this.getConfig("staticIP", ""); + if (staticIP.length() > 0) mySeed.setIP(staticIP); + mySeed.setIPs(Switchboard.getSwitchboard().myPublicIPs()); } public void loadSeedLists() { diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.java index 7ac36955c..dd64b2f72 100644 --- a/source/net/yacy/search/query/SearchEvent.java +++ b/source/net/yacy/search/query/SearchEvent.java @@ -37,6 +37,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.SortedMap; +import java.util.SortedSet; import java.util.TreeMap; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; @@ -131,7 +132,7 @@ public final class SearchEvent { public final List primarySearchThreadsL; public final List nodeSearchThreads; public Thread[] secondarySearchThreads; - public final SortedMap preselectedPeerHashes; + public final SortedSet preselectedPeerHashes; private final SortedMap IACount; private final SortedMap IAResults; private final SortedMap heuristics; @@ -198,7 +199,7 @@ public final class SearchEvent { final QueryParams query, final SeedDB peers, final WorkTables workTables, - final SortedMap preselectedPeerHashes, + final SortedSet preselectedPeerHashes, final boolean generateAbstracts, final LoaderDispatcher loader, final int remote_maxcount, diff --git a/source/net/yacy/search/query/SearchEventCache.java b/source/net/yacy/search/query/SearchEventCache.java index b55ec8682..9458bca07 100644 --- a/source/net/yacy/search/query/SearchEventCache.java +++ b/source/net/yacy/search/query/SearchEventCache.java @@ -29,7 +29,7 @@ package net.yacy.search.query; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; -import java.util.SortedMap; +import java.util.SortedSet; import net.yacy.cora.util.ConcurrentLog; import net.yacy.data.WorkTables; @@ -135,7 +135,7 @@ public class SearchEventCache { final QueryParams query, final SeedDB peers, final WorkTables workTables, - final SortedMap preselectedPeerHashes, + final SortedSet preselectedPeerHashes, final boolean generateAbstracts, final LoaderDispatcher loader, final int remote_maxcount, diff --git a/source/net/yacy/search/snippet/TextSnippet.java b/source/net/yacy/search/snippet/TextSnippet.java index 57882eee6..767b9ab8f 100644 --- a/source/net/yacy/search/snippet/TextSnippet.java +++ b/source/net/yacy/search/snippet/TextSnippet.java @@ -26,7 +26,6 @@ package net.yacy.search.snippet; import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.Comparator; import java.util.HashSet; import java.util.Iterator; @@ -48,7 +47,6 @@ import net.yacy.crawler.retrieval.Request; import net.yacy.crawler.retrieval.Response; import net.yacy.document.Document; import net.yacy.document.Parser; -import net.yacy.document.SentenceReader; import net.yacy.document.SnippetExtractor; import net.yacy.document.WordTokenizer; import net.yacy.document.parser.html.CharacterCoding; diff --git a/source/net/yacy/server/http/AlternativeDomainNames.java b/source/net/yacy/server/http/AlternativeDomainNames.java index 0fabf5bc5..1d7025b9f 100644 --- a/source/net/yacy/server/http/AlternativeDomainNames.java +++ b/source/net/yacy/server/http/AlternativeDomainNames.java @@ -24,6 +24,8 @@ package net.yacy.server.http; +import java.util.Set; + public interface AlternativeDomainNames { /** @@ -42,10 +44,10 @@ public interface AlternativeDomainNames { public String myAlternativeAddress(); /** - * return the IP as string of my server address - * @return IP as string of this server + * return a set of IPs of this server + * @return IP as set of strings of this server */ - public String myIP(); + public Set myIPs(); /** * return the port of my server address diff --git a/source/net/yacy/server/serverAccessTracker.java b/source/net/yacy/server/serverAccessTracker.java index d4f13817d..185fc995e 100644 --- a/source/net/yacy/server/serverAccessTracker.java +++ b/source/net/yacy/server/serverAccessTracker.java @@ -140,7 +140,9 @@ public class serverAccessTracker { track.add(new Track(System.currentTimeMillis(), accessPath)); clearTooOldAccess(track); } - if (Domains.isLocalhost(host)) lastLocalhostAccess = System.currentTimeMillis(); + if (Domains.isLocalhost(host)) lastLocalhostAccess = System.currentTimeMillis(); else { + System.out.println("******** Access not from localhost: " + host); + } } public static Collection accessTrack(final String host) { diff --git a/source/net/yacy/server/serverSwitch.java b/source/net/yacy/server/serverSwitch.java index 8c71614ae..ea81d3892 100644 --- a/source/net/yacy/server/serverSwitch.java +++ b/source/net/yacy/server/serverSwitch.java @@ -31,10 +31,13 @@ import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; import java.net.InetAddress; +import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.Map; import java.util.NavigableMap; import java.util.Random; +import java.util.Set; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -50,6 +53,7 @@ import net.yacy.http.YaCyHttpServer; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.workflow.BusyThread; import net.yacy.kelondro.workflow.WorkflowThread; +import net.yacy.peers.Seed; import net.yacy.search.SwitchboardConstants; public class serverSwitch @@ -139,8 +143,11 @@ public class serverSwitch /** * get my public IP, either set statically or figure out dynamic - * @return + * This method is deprecated because there may be more than one public IPs of this peer, + * i.e. one IPv4 and one IPv6. Please use myPublicIPs() instead + * @return the public IP of this peer, if known */ + @Deprecated public String myPublicIP() { // if a static IP was configured, we have to return it here ... final String staticIP = getConfig("staticIP", ""); @@ -152,6 +159,31 @@ public class serverSwitch return null; } + /** + * Get all my public IPs. If there was a static IP assignment, only one, that IP is returned. + * @return a set of IPs which are supposed to be my own public IPs + */ + public Set myPublicIPs() { + // if a static IP was configured, we have to return it here ... + final String staticIP = getConfig("staticIP", ""); + if ( staticIP.length() > 0 ) { + HashSet h = new HashSet<>(); + h.add(staticIP); + return h; + } + + Set h = new LinkedHashSet<>(); + for (InetAddress i: Domains.myPublicIPv6()) { + String s = i.getHostAddress(); + if (Seed.isProperIP(s)) h.add(s); + } + for (InetAddress i: Domains.myPublicIPv4()) { + String s = i.getHostAddress(); + if (Seed.isProperIP(s)) h.add(s); + } + return h; + } + // a logger for this switchboard public void setLog(final ConcurrentLog log) { this.log = log; diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java index 1b58a5e4b..36adfcbc4 100644 --- a/source/net/yacy/yacy.java +++ b/source/net/yacy/yacy.java @@ -530,7 +530,7 @@ public final class yacy { p.load(fis); fis.close(); // Test for server access restriction (is implemented using Jetty IPaccessHandler which does not support IPv6 - // try to disavle IPv6 + // try to disable IPv6 String teststr = p.getProperty("serverClient", "*"); if (!teststr.equals("*")) { // testing on Win-8 showed this property has to be set befor Switchboard starts @@ -569,7 +569,6 @@ public final class yacy { } } - /** * Main-method which is started by java. Checks for special arguments or * starts up the application. @@ -601,8 +600,7 @@ public final class yacy { if (OS.isWindows) headless = false; if (args.length >= 1 && args[0].toLowerCase().equals("-gui")) headless = false; System.setProperty("java.awt.headless", headless ? "true" : "false"); - // System.setProperty("java.net.preferIPv4Stack", "true"); // DO NOT PREFER IPv6, i.e. freifunk uses ipv6 only and host resolving does not work - + String s = ""; for (final String a: args) s += a + " "; yacyRelease.startParameter = s.trim(); diff --git a/startYACY.sh b/startYACY.sh index ede26fd7e..f8a20e9cb 100755 --- a/startYACY.sh +++ b/startYACY.sh @@ -6,7 +6,7 @@ PIDFILE="yacy.pid" OS="`uname`" #get javastart args -JAVA_ARGS="-server -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Dfile.encoding=UTF-8"; +JAVA_ARGS="-server -Djava.awt.headless=true -Dfile.encoding=UTF-8"; #JAVA_ARGS="-verbose:gc -XX:+PrintGCTimeStamps -XX:+PrintGCDetails $JAVA_ARGS"; #check if OS is Sun Solaris or one of the OpenSolaris distributions and use different version of id if necessary diff --git a/startYACY_debug.bat b/startYACY_debug.bat index 3af009b59..28bafe8be 100644 --- a/startYACY_debug.bat +++ b/startYACY_debug.bat @@ -17,7 +17,7 @@ if exist DATA\SETTINGS\httpProxy.conf GoTo :RENAMEINDEX if exist DATA\SETTINGS\yacy.conf GoTo :GETSTARTOPTS :STARTJAVA -set javacmd=%javacmd% -XX:-UseGCOverheadLimit -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Dfile.encoding=UTF-8 +set javacmd=%javacmd% -XX:-UseGCOverheadLimit -Djava.awt.headless=true -Dfile.encoding=UTF-8 Rem Starting YaCy Echo Generated classpath:%CLASSPATH% Echo JRE Parameters:%javacmd%