large IPv6 redesign of peer ping methods!

removed preferred IPv4 in start options and added a new field IP6 in
peer seeds which will contain one or more IPv6 addresses. Now every peer
has one or more IP addresses assigned, even several IPv6 addresses are
possible. The peer-ping process must check all given and possible IP
addresses for a backping and return the one IP which was successful when
pinging the peer. The ping-ing peer must be able to recognize which of
the given IPs are available for outside access of the peer and store
this accordingly. If only one IPv6 address is available and no IPv4,
then the IPv6 is stored in the old IP field of the seed DNA.
Many methods in Seed.java are now marked as @deprecated because they had
been used for a single IP only. There is still a large construction site
left in YaCy now where all these deprecated methods must be replaced
with new method calls. The 'extra'-IPs, used by cluster assignment had
been removed since that can be replaced with IPv6 usage in p2p clusters.
All clusters must now use IPv6 if they want an intranet-routing.
pull/1/head
Michael Peter Christen 10 years ago
parent 67cd4c37bd
commit 6491270b3a

@ -25,7 +25,7 @@
<key>Java</key> <key>Java</key>
<dict> <dict>
<key>VMOptions</key> <key>VMOptions</key>
<string>-Xmx600m -Xms600m -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8 -Dsolr.directoryFactory=solr.MMapDirectoryFactory</string> <string>-Xmx600m -Xms600m -Dfile.encoding=UTF-8 -Dsolr.directoryFactory=solr.MMapDirectoryFactory</string>
<key>WorkingDirectory</key> <key>WorkingDirectory</key>
<string>$APP_PACKAGE/Contents/Resources/Java</string> <string>$APP_PACKAGE/Contents/Resources/Java</string>
<key>MainClass</key> <key>MainClass</key>

@ -57,7 +57,7 @@ SHUTDOWN_TIMEOUT=50
# Default niceness if not set in config file # Default niceness if not set in config file
NICE_VAL=0 NICE_VAL=0
JAVA_ARGS="-server -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Dsolr.directoryFactory=solr.MMapDirectoryFactory" JAVA_ARGS="-server -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Dsolr.directoryFactory=solr.MMapDirectoryFactory"
ifdef(`openSUSE', `dnl ifdef(`openSUSE', `dnl
. /etc/rc.status . /etc/rc.status

@ -192,7 +192,7 @@ public class Blog {
prop.putHTML("mode_author", UTF8.String(author)); prop.putHTML("mode_author", UTF8.String(author));
prop.putHTML("mode_subject", post.get("subject","")); prop.putHTML("mode_subject", post.get("subject",""));
prop.put("mode_date", dateString(new Date())); prop.put("mode_date", dateString(new Date()));
prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_page", post.get("content", "")); prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_page", post.get("content", ""));
prop.putHTML("mode_page-code", post.get("content", "")); prop.putHTML("mode_page-code", post.get("content", ""));
} }
else { else {
@ -327,7 +327,7 @@ public class Blog {
prop.put("mode_entries_" + number + "_page", entry.getPage()); prop.put("mode_entries_" + number + "_page", entry.getPage());
prop.put("mode_entries_" + number + "_timestamp", entry.getTimestamp()); prop.put("mode_entries_" + number + "_timestamp", entry.getTimestamp());
} else { } else {
prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_entries_" + number + "_page", entry.getPage()); prop.putWiki(sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()), "mode_entries_" + number + "_page", entry.getPage());
} }
if (hasRights) { if (hasRights) {

@ -175,7 +175,7 @@ public class BlogComments {
prop.putHTML("mode_allow_author", UTF8.String(author)); prop.putHTML("mode_allow_author", UTF8.String(author));
prop.putHTML("mode_subject", post.get("subject","")); prop.putHTML("mode_subject", post.get("subject",""));
prop.put("mode_date", dateString(new Date())); prop.put("mode_date", dateString(new Date()));
prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_page", post.get("content", "")); prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_page", post.get("content", ""));
prop.put("mode_page-code", post.get("content", "")); prop.put("mode_page-code", post.get("content", ""));
} else { } else {
// show blog-entry/entries // show blog-entry/entries
@ -191,7 +191,7 @@ public class BlogComments {
prop.putHTML("mode_allow_author", UTF8.String(author)); prop.putHTML("mode_allow_author", UTF8.String(author));
prop.put("mode_comments", page.getCommentsSize()); prop.put("mode_comments", page.getCommentsSize());
prop.put("mode_date", dateString(page.getDate())); prop.put("mode_date", dateString(page.getDate()));
prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_page", page.getPage()); prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_page", page.getPage());
if (hasRights) { if (hasRights) {
prop.put("mode_admin", "1"); prop.put("mode_admin", "1");
prop.put("mode_admin_pageid", page.getKey()); prop.put("mode_admin_pageid", page.getKey());
@ -234,7 +234,7 @@ public class BlogComments {
if (!xml) { if (!xml) {
prop.putHTML("mode_entries_"+count+"_subject", UTF8.String(entry.getSubject())); prop.putHTML("mode_entries_"+count+"_subject", UTF8.String(entry.getSubject()));
prop.putHTML("mode_entries_"+count+"_author", UTF8.String(entry.getAuthor())); prop.putHTML("mode_entries_"+count+"_author", UTF8.String(entry.getAuthor()));
prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_entries_"+count+"_page", entry.getPage()); prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_entries_"+count+"_page", entry.getPage());
} else { } else {
prop.putHTML("mode_entries_"+count+"_subject", UTF8.String(entry.getSubject())); prop.putHTML("mode_entries_"+count+"_subject", UTF8.String(entry.getSubject()));
prop.putHTML("mode_entries_"+count+"_author", UTF8.String(entry.getAuthor())); prop.putHTML("mode_entries_"+count+"_author", UTF8.String(entry.getAuthor()));

@ -174,7 +174,7 @@ public class ConfigNetwork_p
prop.putHTML("cluster.peers.ipport", sb.getConfig("cluster.peers.ipport", "")); prop.putHTML("cluster.peers.ipport", sb.getConfig("cluster.peers.ipport", ""));
prop.putHTML("cluster.peers.yacydomain", sb.getConfig("cluster.peers.yacydomain", "")); prop.putHTML("cluster.peers.yacydomain", sb.getConfig("cluster.peers.yacydomain", ""));
StringBuilder hashes = new StringBuilder(); StringBuilder hashes = new StringBuilder();
for ( final byte[] h : sb.clusterhashes.keySet() ) { for (final byte[] h: sb.clusterhashes) {
hashes.append(", ").append(ASCII.String(h)); hashes.append(", ").append(ASCII.String(h));
} }
if (hashes.length() > 2) { if (hashes.length() > 2) {

@ -97,7 +97,7 @@ public class CrawlStartScanner_p
} else { } else {
ip = Domains.myPublicLocalIP(); ip = Domains.myPublicLocalIP();
if ( Domains.isThisHostIP(ip) ) { if ( Domains.isThisHostIP(ip) ) {
ip = sb.peers.mySeed().getInetAddress(); ip = Domains.dnsResolve(sb.peers.mySeed().getIP());
} }
} }
if ( ip != null ) { if ( ip != null ) {

@ -106,7 +106,7 @@ public class MessageSend_p {
prop.putXML("mode_permission_message", message); prop.putXML("mode_permission_message", message);
prop.putHTML("mode_permission_hash", hash); prop.putHTML("mode_permission_hash", hash);
if (post.containsKey("preview")) { if (post.containsKey("preview")) {
prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_permission_previewmessage", message); prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_permission_previewmessage", message);
} }

@ -162,7 +162,7 @@ public class Messages_p {
prop.putXML("mode_subject", message.subject()); prop.putXML("mode_subject", message.subject());
String theMessage = null; String theMessage = null;
theMessage = UTF8.String(message.message()); theMessage = UTF8.String(message.message());
prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_message", theMessage); prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_message", theMessage);
prop.put("mode_hash", message.authorHash()); prop.put("mode_hash", message.authorHash());
prop.putXML("mode_key", key); prop.putXML("mode_key", key);
} }

@ -162,7 +162,7 @@ document.getElementById("apilink").setAttribute("href", "Network.xml?" + window.
<td align="right">#[rU]#</td> <td align="right">#[rU]#</td>
#(complete)# #(complete)#
:: ::
<td><a href="http://#[ip]#:#[port]#/Network.html?page=1&amp;ip=">http://#[ip]#:#[port]#</a></td> <td><a href="http://#[ip]#:#[port]#/Network.html?page=1&amp;ip=">http://#[ip]#:#[port]#</a> #[ips]#</td>
<td >#[hash]#</td> <td >#[hash]#</td>
<td >#[age]#</td> <td >#[age]#</td>
<td align="right">#[seeds]#</td> <td align="right">#[seeds]#</td>

@ -191,7 +191,7 @@ public class Network {
if (sb.peers.mySeed() != null) { if (sb.peers.mySeed() != null) {
prop.put("table_my-hash", sb.peers.mySeed().hash ); prop.put("table_my-hash", sb.peers.mySeed().hash );
prop.put("table_my-ip", sb.peers.mySeed().getIP() ); prop.put("table_my-ip", sb.peers.mySeed().getIPs().toString());
prop.put("table_my-port", sb.peers.mySeed().getPort() ); prop.put("table_my-port", sb.peers.mySeed().getPort() );
} }
@ -210,11 +210,11 @@ public class Network {
Seed peer = new Seed(post.get("peerHash"), map); Seed peer = new Seed(post.get("peerHash"), map);
sb.updateMySeed(); sb.updateMySeed();
final int added = Protocol.hello(sb.peers.mySeed(), sb.peers.peerActions, peer.getPublicAddress(), peer.hash, peer.getName()); final int added = Protocol.hello(sb.peers.mySeed(), sb.peers.peerActions, peer);
if (added <= 0) { if (added <= 0) {
prop.put("table_comment",1); prop.put("table_comment",1);
prop.putHTML("table_comment_status","publish: disconnected peer '" + peer.getName() + "/" + post.get("peerHash") + "' from " + peer.getPublicAddress()); prop.putHTML("table_comment_status","publish: disconnected peer '" + peer.getName() + "/" + post.get("peerHash") + "' from " + peer.getIPs());
} else { } else {
peer = sb.peers.getConnected(peer.hash); peer = sb.peers.getConnected(peer.hash);
if (peer == null) { if (peer == null) {
@ -222,7 +222,7 @@ public class Network {
prop.putHTML("table_comment_status","publish: disconnected peer 'UNKNOWN/" + post.get("peerHash") + "' from UNKNOWN"); prop.putHTML("table_comment_status","publish: disconnected peer 'UNKNOWN/" + post.get("peerHash") + "' from UNKNOWN");
} else { } else {
prop.put("table_comment",2); prop.put("table_comment",2);
prop.putHTML("table_comment_status","publish: handshaked " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + peer.getName() + "' at " + peer.getPublicAddress()); prop.putHTML("table_comment_status","publish: handshaked " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + peer.getName() + "' at " + peer.getIPs());
prop.putHTML("table_comment_details",peer.toString()); prop.putHTML("table_comment_details",peer.toString());
} }
} }
@ -397,7 +397,8 @@ public class Network {
prop.putHTML(STR_TABLE_LIST + conCount + "_location", location); prop.putHTML(STR_TABLE_LIST + conCount + "_location", location);
if (complete) { if (complete) {
prop.put(STR_TABLE_LIST + conCount + "_complete", 1); prop.put(STR_TABLE_LIST + conCount + "_complete", 1);
prop.put(STR_TABLE_LIST + conCount + "_complete_ip", seed.getIP() ); prop.putHTML(STR_TABLE_LIST + conCount + "_complete_ip", seed.getIP() );
prop.putHTML(STR_TABLE_LIST + conCount + "_complete_ips", seed.getIPs().toString() );
prop.put(STR_TABLE_LIST + conCount + "_complete_port", seed.get(Seed.PORT, "-") ); prop.put(STR_TABLE_LIST + conCount + "_complete_port", seed.get(Seed.PORT, "-") );
prop.put(STR_TABLE_LIST + conCount + "_complete_hash", seed.hash); prop.put(STR_TABLE_LIST + conCount + "_complete_hash", seed.hash);
prop.put(STR_TABLE_LIST + conCount + "_complete_age", seed.getAge()); prop.put(STR_TABLE_LIST + conCount + "_complete_age", seed.getAge());

@ -177,7 +177,7 @@ public class SettingsAck_p {
} else if (staticIP.startsWith("https://")) { } else if (staticIP.startsWith("https://")) {
if (staticIP.length() > 8) { staticIP = staticIP.substring(8); } else { staticIP = ""; } if (staticIP.length() > 8) { staticIP = staticIP.substring(8); } else { staticIP = ""; }
} }
String error = Seed.isProperIP(staticIP); String error = Seed.isProperIP(staticIP) ? null : "ip not proper: " + staticIP;
if (error == null) { if (error == null) {
serverCore.useStaticIP = true; serverCore.useStaticIP = true;
sb.peers.mySeed().setIP(staticIP); sb.peers.mySeed().setIP(staticIP);

@ -27,8 +27,8 @@
// javac -classpath .:../Classes Status.java // javac -classpath .:../Classes Status.java
// if the shell's current path is HTROOT // if the shell's current path is HTROOT
import java.net.InetAddress;
import java.util.Date; import java.util.Date;
import java.util.Set;
import net.yacy.cora.protocol.ConnectionInfo; import net.yacy.cora.protocol.ConnectionInfo;
import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.Domains;
@ -189,8 +189,9 @@ public class Status
} else { } else {
prop.put("extPortFormat", "0"); prop.put("extPortFormat", "0");
} }
final InetAddress hostIP = Domains.myPublicLocalIP();
prop.put("host", hostIP != null ? hostIP.getHostAddress() : "Unkown IP"); Set<String> ips = Domains.myPublicIPs();
prop.put("host", ips.toString());
// ssl support // ssl support
prop.put("sslSupport", sb.getConfig("keyStore", "").isEmpty() || !sb.getConfigBool("server.https", false) ? 0 : 1); prop.put("sslSupport", sb.getConfig("keyStore", "").isEmpty() || !sb.getConfigBool("server.https", false) ? 0 : 1);
@ -227,7 +228,7 @@ public class Status
prop.putNum("peerStatistics_disconnects", sb.peers.peerActions.disconnects); prop.putNum("peerStatistics_disconnects", sb.peers.peerActions.disconnects);
prop.put("peerStatistics_connects", Formatter.number(sb.peers.mySeed().get(Seed.CCOUNT, "0"))); prop.put("peerStatistics_connects", Formatter.number(sb.peers.mySeed().get(Seed.CCOUNT, "0")));
thisHash = sb.peers.mySeed().hash; thisHash = sb.peers.mySeed().hash;
if ( sb.peers.mySeed().getPublicAddress() == null ) { if ( sb.peers.mySeed().getIPs().size() == 0 ) {
prop.put("peerAddress", "0"); // not assigned + instructions prop.put("peerAddress", "0"); // not assigned + instructions
prop.put("warningGoOnline", "1"); prop.put("warningGoOnline", "1");
} else { } else {

@ -277,7 +277,7 @@ public class Surftips {
Seed seed = sb.peers.getConnected(record.originator()); Seed seed = sb.peers.getConnected(record.originator());
if (seed == null) seed = sb.peers.getDisconnected(record.originator()); if (seed == null) seed = sb.peers.getDisconnected(record.originator());
if (seed != null) { if (seed != null) {
url = "http://" + seed.getPublicAddress() + "/Wiki.html?page=" + record.attribute("page", ""); url = "http://" + seed.getPublicAddress(seed.getIP()) + "/Wiki.html?page=" + record.attribute("page", "");
entry = rowdef.newEntry(new byte[][]{ entry = rowdef.newEntry(new byte[][]{
UTF8.getBytes(url), UTF8.getBytes(url),
UTF8.getBytes(record.attribute("author", "Anonymous") + ": " + record.attribute("page", "")), UTF8.getBytes(record.attribute("author", "Anonymous") + ": " + record.attribute("page", "")),
@ -292,7 +292,7 @@ public class Surftips {
Seed seed = sb.peers.getConnected(record.originator()); Seed seed = sb.peers.getConnected(record.originator());
if (seed == null) seed = sb.peers.getDisconnected(record.originator()); if (seed == null) seed = sb.peers.getDisconnected(record.originator());
if (seed != null) { if (seed != null) {
url = "http://" + seed.getPublicAddress() + "/Blog.html?page=" + record.attribute("page", ""); url = "http://" + seed.getPublicAddress(seed.getIP()) + "/Blog.html?page=" + record.attribute("page", "");
entry = rowdef.newEntry(new byte[][]{ entry = rowdef.newEntry(new byte[][]{
UTF8.getBytes(url), UTF8.getBytes(url),
UTF8.getBytes(record.attribute("author", "Anonymous") + ": " + record.attribute("page", "")), UTF8.getBytes(record.attribute("author", "Anonymous") + ": " + record.attribute("page", "")),

@ -113,7 +113,6 @@ public class ViewProfile {
} }
// try to get the profile from remote peer // try to get the profile from remote peer
if (sb.clusterhashes != null) seed.setAlternativeAddress(sb.clusterhashes.get(seed.hash.getBytes()));
profile = Protocol.getProfile(seed); profile = Protocol.getProfile(seed);
// if profile did not arrive, say that peer is disconnected // if profile did not arrive, say that peer is disconnected
@ -170,7 +169,7 @@ public class ViewProfile {
prop.put("success_" + key, "1"); prop.put("success_" + key, "1");
// only comments get "wikified" // only comments get "wikified"
if(key.equals("comment")){ if(key.equals("comment")){
prop.putWiki(sb.peers.mySeed().getClusterAddress(), prop.putWiki(sb.peers.mySeed().getPublicAddress(),
"success_" + key + "_value", "success_" + key + "_value",
entry.getValue().replaceAll("\r", "").replaceAll("\\\\n", "\n")); entry.getValue().replaceAll("\r", "").replaceAll("\\\\n", "\n"));
prop.put("success_" + key + "_b64value", Base64Order.standardCoder.encodeString(entry.getValue())); prop.put("success_" + key + "_b64value", Base64Order.standardCoder.encodeString(entry.getValue()));

@ -147,7 +147,7 @@ public class Wiki {
prop.putHTML("mode_pagename", pagename); prop.putHTML("mode_pagename", pagename);
prop.putHTML("mode_author", author); prop.putHTML("mode_author", author);
prop.put("mode_date", dateString(new Date())); prop.put("mode_date", dateString(new Date()));
prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_page", post.get("content", "")); prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_page", post.get("content", ""));
prop.putHTML("mode_page-code", post.get("content", "")); prop.putHTML("mode_page-code", post.get("content", ""));
} }
//end contrib of [MN] //end contrib of [MN]
@ -237,7 +237,7 @@ public class Wiki {
prop.putHTML("mode_versioning_pagename", pagename); prop.putHTML("mode_versioning_pagename", pagename);
prop.putHTML("mode_versioning_author", oentry.author()); prop.putHTML("mode_versioning_author", oentry.author());
prop.put("mode_versioning_date", dateString(oentry.date())); prop.put("mode_versioning_date", dateString(oentry.date()));
prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_versioning_page", oentry.page()); prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_versioning_page", oentry.page());
prop.putHTML("mode_versioning_page-code", UTF8.String(oentry.page())); prop.putHTML("mode_versioning_page-code", UTF8.String(oentry.page()));
} }
} catch (final IOException e) { } catch (final IOException e) {
@ -252,7 +252,7 @@ public class Wiki {
prop.putHTML("mode_pagename", pagename); prop.putHTML("mode_pagename", pagename);
prop.putHTML("mode_author", page.author()); prop.putHTML("mode_author", page.author());
prop.put("mode_date", dateString(page.date())); prop.put("mode_date", dateString(page.date()));
prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_page", page.page()); prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_page", page.page());
prop.put("controls", "0"); prop.put("controls", "0");
prop.putHTML("controls_pagename", pagename); prop.putHTML("controls_pagename", pagename);

@ -43,7 +43,7 @@ public class goto_p {
* hash= of remote peer * hash= of remote peer
* path= path part to forward to * path= path part to forward to
*/ */
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env; final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();

@ -70,7 +70,7 @@ public class mediawiki_p {
page = page.substring(p, q); page = page.substring(p, q);
prop.putHTML("title", title); prop.putHTML("title", title);
prop.putWiki(sb.peers.mySeed().getClusterAddress(), "page", page); prop.putWiki(sb.peers.mySeed().getPublicAddress(), "page", page);
return prop; return prop;
} }

@ -30,17 +30,18 @@
import java.io.IOException; import java.io.IOException;
import java.net.InetAddress; import java.net.InetAddress;
import java.util.Iterator; import java.util.Iterator;
import java.util.Set;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.peers.Network; import net.yacy.peers.Network;
import net.yacy.peers.DHTSelection; import net.yacy.peers.DHTSelection;
import net.yacy.peers.Protocol; import net.yacy.peers.Protocol;
import net.yacy.peers.Seed; import net.yacy.peers.Seed;
import net.yacy.peers.graphics.ProfilingGraph; import net.yacy.peers.graphics.ProfilingGraph;
import net.yacy.peers.operation.yacyVersion;
import net.yacy.search.EventTracker; import net.yacy.search.EventTracker;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.server.serverCore; import net.yacy.server.serverCore;
@ -58,6 +59,7 @@ public final class hello {
final long start = System.currentTimeMillis(); final long start = System.currentTimeMillis();
prop.put("message", "none"); prop.put("message", "none");
final String clientip = header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "<unknown>"); // read an artificial header addendum final String clientip = header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "<unknown>"); // read an artificial header addendum
ConcurrentLog.info("**hello-DEBUG**", "client request from = " + clientip);
final InetAddress ias = Domains.dnsResolve(clientip); final InetAddress ias = Domains.dnsResolve(clientip);
long time = System.currentTimeMillis(); long time = System.currentTimeMillis();
final long time_dnsResolve = System.currentTimeMillis() - time; final long time_dnsResolve = System.currentTimeMillis() - time;
@ -83,7 +85,7 @@ public final class hello {
final String key = post.get("key", ""); // transmission key for response final String key = post.get("key", ""); // transmission key for response
final String seed = post.get("seed", ""); final String seed = post.get("seed", "");
int count = post.getInt("count", 0); int count = post.getInt("count", 0);
final long magic = post.getLong("magic", 0); // final long magic = post.getLong("magic", 0);
// final Date remoteTime = yacyCore.parseUniversalDate(post.get(MYTIME)); // read remote time // final Date remoteTime = yacyCore.parseUniversalDate(post.get(MYTIME)); // read remote time
if (seed.length() > Seed.maxsize) { if (seed.length() > Seed.maxsize) {
Network.log.info("hello/server: rejected contacting seed; too large (" + seed.length() + " > " + Seed.maxsize + ", time_dnsResolve=" + time_dnsResolve + ")"); Network.log.info("hello/server: rejected contacting seed; too large (" + seed.length() + " > " + Seed.maxsize + ", time_dnsResolve=" + time_dnsResolve + ")");
@ -105,22 +107,22 @@ public final class hello {
return prop; return prop;
} }
// final String properTest = remoteSeed.isProper();
// The remote peer might not know its IP yet, so don't abort if the IP check fails
// if ((properTest != null) && (! properTest.substring(0,1).equals("IP"))) { return null; }
// we easily know the caller's IP: // we easily know the caller's IP:
final String userAgent = header.get(HeaderFramework.USER_AGENT, "<unknown>"); final String userAgent = header.get(HeaderFramework.USER_AGENT, "<unknown>");
sb.peers.peerActions.setUserAgent(clientip, userAgent); sb.peers.peerActions.setUserAgent(clientip, userAgent);
final String reportedip = remoteSeed.getIP(); final Set<String> reportedips = remoteSeed.getIPs();
final String reportedPeerType = remoteSeed.get(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR); final String reportedPeerType = remoteSeed.get(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR);
final double clientversion = remoteSeed.getVersion(); //final double clientversion = remoteSeed.getVersion();
if ((reportedip + ':' + remoteSeed.getPort()).equals(sb.peers.mySeed().getPublicAddress())) { if (remoteSeed.getPort() == sb.peers.mySeed().getPort()) {
for (String reportedip: reportedips) {
if (sb.peers.mySeed().getIPs().contains(reportedip)) {
// reject a self-ping // reject a self-ping
prop.put("message", "I am I"); prop.put("message", "I am I");
return prop; return prop;
} }
}
}
if (remoteSeed.hash.equals(sb.peers.mySeed().hash)) { if (remoteSeed.hash.equals(sb.peers.mySeed().hash)) {
// reject a ping with my own hash // reject a ping with my own hash
prop.put("message", "You are using my peer hash"); prop.put("message", "You are using my peer hash");
@ -140,58 +142,42 @@ public final class hello {
} }
long[] callback = new long[]{-1, -1}; long[] callback = new long[]{-1, -1};
if (sb.clusterhashes != null) remoteSeed.setAlternativeAddress(sb.clusterhashes.get(remoteSeed.hash.getBytes()));
// if the remote client has reported its own IP address and the client supports // if the remote client has reported its own IP address and the client supports
// the port forwarding feature (if client version >= 0.383) then we try to // the port forwarding feature (if client version >= 0.383) then we try to
// connect to the reported IP address first // connect to the reported IP address first
long time_backping = 0; long time_backping = 0;
String backping_method = "none"; String backping_method = "none";
if (reportedip.length() > 0 && boolean success = false;
!clientip.equals(reportedip) && // TODO: make this a concurrent process
clientversion >= yacyVersion.YACY_SUPPORTS_PORT_FORWARDING && if (!serverCore.useStaticIP || !ias.isSiteLocalAddress()) {
magic != 0) { reportedips.add(ias.getHostAddress());
}
// try first the reportedip, since this may be a connect from a port-forwarding host final int connectedBefore = sb.peers.sizeConnected();
ConcurrentLog.info("**hello-DEBUG**", "peer " + remoteSeed.getName() + " challenged us with IPs " + reportedips);
int callbackRemain = Math.min(5, reportedips.size());
long callbackStart = System.currentTimeMillis();
if (reportedips.size() > 0) {
for (String reportedip: reportedips) {
int partialtimeout = ((int) (callbackStart + 6500 - System.currentTimeMillis())) / callbackRemain; // bad hack until a concurrent version is implemented
if (partialtimeout <= 0) break;
ConcurrentLog.info("**hello-DEBUG**", "reportedip = " + reportedip + " is handled");
if (Seed.isProperIP(reportedip)) {
ConcurrentLog.info("**hello-DEBUG**", "starting callback to reportedip = " + reportedip + ", timeout = " + partialtimeout);
prop.put("yourip", reportedip); prop.put("yourip", reportedip);
remoteSeed.setIP(reportedip); remoteSeed.setIP(reportedip);
time = System.currentTimeMillis(); time = System.currentTimeMillis();
callback = Protocol.queryRWICount(remoteSeed, "Tq418bNZd6AO"); callback = Protocol.queryRWICount(remoteSeed.getPublicAddress(reportedip), remoteSeed.hash, partialtimeout);
ConcurrentLog.info("**hello-DEBUG**", "reportedip = " + reportedip + " returns callback " + (callback == null ? "NULL" : callback[0]));
time_backping = System.currentTimeMillis() - time; time_backping = System.currentTimeMillis() - time;
backping_method = "reportedip=" + reportedip; backping_method = "reportedip=" + reportedip;
} else { if (callback[0] >= 0) { success = true; break; }
prop.put("yourip", ias.getHostAddress()); if (callbackRemain-- <= 0) break; // no more tries left / restrict to a limited number of ips
remoteSeed.setIP(ias.getHostAddress());
} }
// if the previous attempt (using the reported ip address) was not successful,
// then try the ip where the request came from
if (callback[0] < 0 || (magic != 0 && magic != callback[1])) {
boolean isNotLocal = true;
// we are only allowed to connect to the client IP address if it's not our own address
if (serverCore.useStaticIP) {
isNotLocal = !ias.isSiteLocalAddress();
} }
if (isNotLocal) {
prop.put("yourip", clientip);
remoteSeed.setIP(clientip);
time = System.currentTimeMillis();
callback = Protocol.queryRWICount(remoteSeed, "Tq418bNZd6AO"); // hash for "www"; the actual count is irrelevant, we just want to know if this works
time_backping = System.currentTimeMillis() - time;
backping_method = "clientip=" + clientip;
} }
} if (success) {
ConcurrentLog.info("**hello-DEBUG**", "success for IP(s) " + remoteSeed.getIPs() + ", port " + remoteSeed.getPort());
// System.out.println("YACYHELLO: YOUR IP=" + clientip);
// set lastseen value (we have seen that peer, it contacted us!)
remoteSeed.setLastSeenUTC();
// assign status
final int connectedBefore = sb.peers.sizeConnected();
if (callback[0] >= 0) {
if (remoteSeed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) == null) { if (remoteSeed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) == null) {
prop.put(Seed.YOURTYPE, Seed.PEERTYPE_SENIOR); prop.put(Seed.YOURTYPE, Seed.PEERTYPE_SENIOR);
remoteSeed.put(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR); remoteSeed.put(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR);
@ -202,26 +188,27 @@ public final class hello {
remoteSeed.put(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR); remoteSeed.put(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR);
} }
// connect the seed // connect the seed
Network.log.info("hello/server: responded remote senior peer '" + remoteSeed.getName() + "' from " + reportedip + ", time_dnsResolve=" + time_dnsResolve + ", time_backping=" + time_backping + ", method=" + backping_method + ", urls=" + callback[0]); Network.log.info("hello/server: responded remote " + reportedPeerType + " peer '" + remoteSeed.getName() + "' from " + reportedips + ", time_dnsResolve=" + time_dnsResolve + ", time_backping=" + time_backping + ", method=" + backping_method + ", urls=" + callback[0]);
sb.peers.peerActions.peerArrival(remoteSeed, true); sb.peers.peerActions.peerArrival(remoteSeed, true);
} else { } else {
ConcurrentLog.info("**hello-DEBUG**", "fail for IP(s) " + remoteSeed.getIPs() + ", port " + remoteSeed.getPort());
prop.put("yourip", ias.getHostAddress());
remoteSeed.setIP(ias.getHostAddress());
prop.put(Seed.YOURTYPE, Seed.PEERTYPE_JUNIOR); prop.put(Seed.YOURTYPE, Seed.PEERTYPE_JUNIOR);
remoteSeed.put(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR); remoteSeed.put(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR);
Network.log.info("hello/server: responded remote junior peer '" + remoteSeed.getName() + "' from " + reportedip + ", time_dnsResolve=" + time_dnsResolve + ", time_backping=" + time_backping + ", method=" + backping_method + ", urls=" + callback[0]); Network.log.info("hello/server: responded remote " + reportedPeerType + " peer '" + remoteSeed.getName() + "' from " + reportedips + ", time_dnsResolve=" + time_dnsResolve + ", time_backping=" + time_backping + ", method=" + backping_method + ", urls=" + callback[0]);
// no connection here, instead store junior in connection cache // no connection here, instead store junior in connection cache
if ((remoteSeed.hash != null) && (remoteSeed.isProper(false) == null)) { if ((remoteSeed.hash != null) && (remoteSeed.isProper(false) == null)) {
sb.peers.peerActions.peerPing(remoteSeed); sb.peers.peerActions.peerPing(remoteSeed);
} }
} }
remoteSeed.setLastSeenUTC();
final int connectedAfter = sb.peers.sizeConnected(); final int connectedAfter = sb.peers.sizeConnected();
// update event tracker // update event tracker
EventTracker.update(EventTracker.EClass.PEERPING, new ProfilingGraph.EventPing(remoteSeed.getName(), sb.peers.myName(), false, connectedAfter - connectedBefore), false); EventTracker.update(EventTracker.EClass.PEERPING, new ProfilingGraph.EventPing(remoteSeed.getName(), sb.peers.myName(), false, connectedAfter - connectedBefore), false);
if (!(prop.get(Seed.YOURTYPE)).equals(reportedPeerType)) { if (!(prop.get(Seed.YOURTYPE)).equals(reportedPeerType)) {
Network.log.info("hello/server: changing remote peer '" + remoteSeed.getName() + Network.log.info("hello/server: changing remote peer '" + remoteSeed.getName() + "' " + reportedips + " peerType from '" + reportedPeerType + "' to '" + prop.get(Seed.YOURTYPE) + "'.");
"' [" + reportedip +
"] peerType from '" + reportedPeerType +
"' to '" + prop.get(Seed.YOURTYPE) + "'.");
} }
final StringBuilder seeds = new StringBuilder(768); final StringBuilder seeds = new StringBuilder(768);
@ -262,7 +249,7 @@ public final class hello {
prop.put("seedlist", seeds.toString()); prop.put("seedlist", seeds.toString());
// return rewrite properties // return rewrite properties
prop.put("message", "ok " + seed.length()); prop.put("message", "ok " + seed.length());
Network.log.info("hello/server: responded remote peer '" + remoteSeed.getName() + "' [" + reportedip + "] in " + (System.currentTimeMillis() - start) + " milliseconds"); Network.log.info("hello/server: responded remote peer '" + remoteSeed.getName() + "' " + reportedips + " in " + (System.currentTimeMillis() - start) + " milliseconds");
return prop; return prop;
} }

@ -109,7 +109,7 @@ public final class query {
if (obj.equals("lurlcount")) { if (obj.equals("lurlcount")) {
// return the number of all available l-url's // return the number of all available l-url's
prop.put("response", 1 /*sb.index.fulltext().collectionSize()*/); // patched to not call collectionSize() any more because the acutal size is not needed. Instead, rwicount should be called prop.put("response", "1" /*sb.index.fulltext().collectionSize()*/); // patched to not call collectionSize() any more because the acutal size is not needed. Instead, rwicount should be called
return prop; return prop;
} }

@ -3,7 +3,7 @@ title YaCy Windows Service Install
:STARTJAVA :STARTJAVA
REM set the Java options REM set the Java options
set javaopts=-Xss256k;-XX:MaxPermSize=256m;-Djava.net.preferIPv4Stack=true;-Djava.awt.headless=true;-Dfile.encoding=UTF-8 set javaopts=-Xss256k;-XX:MaxPermSize=256m;-Djava.awt.headless=true;-Dfile.encoding=UTF-8
REM set max Java heap memory (in MB) REM set max Java heap memory (in MB)
set jmx=800 set jmx=800

@ -29,6 +29,7 @@ import java.io.IOException;
import java.lang.reflect.Array; import java.lang.reflect.Array;
import java.lang.reflect.InvocationTargetException; import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method; import java.lang.reflect.Method;
import java.net.Inet4Address;
import java.net.InetAddress; import java.net.InetAddress;
import java.net.NetworkInterface; import java.net.NetworkInterface;
import java.net.SocketException; import java.net.SocketException;
@ -68,15 +69,19 @@ public class Domains {
private final static ConcurrentLog log = new ConcurrentLog(Domains.class.getName()); private final static ConcurrentLog log = new ConcurrentLog(Domains.class.getName());
public static final String LOCALHOST = "127.0.0.1"; // replace with IPv6 0:0:0:0:0:0:0:1 ? public static final String LOCALHOST = "localhost"; // replace with IPv6 0:0:0:0:0:0:0:1 ?
private static String LOCALHOST_NAME = LOCALHOST; // this will be replaced with the actual name of the local host private static String LOCALHOST_NAME = LOCALHOST; // this will be replaced with the actual name of the local host
private static Class<?> InetAddressLocatorClass; private static Class<?> InetAddressLocatorClass;
private static Method InetAddressLocatorGetLocaleInetAddressMethod; private static Method InetAddressLocatorGetLocaleInetAddressMethod;
private static final Set<String> ccSLD_TLD = new HashSet<String>(); private static final Set<String> ccSLD_TLD = new HashSet<String>();
private static final String PRESENT = ""; private static final String PRESENT = "";
private static final Pattern LOCALHOST_PATTERNS = Pattern.compile("(127\\..*)|(localhost)|(\\[?(fe80|0)\\:0\\:0\\:0\\:0\\:0\\:0\\:1.*)"); private static final String LOCALHOST_IPv4_PATTERN = "(127\\..*)";
private static final Pattern INTRANET_PATTERNS = Pattern.compile("(10\\..*)|(127\\..*)|(172\\.(1[6-9]|2[0-9]|3[0-1])\\..*)|(169\\.254\\..*)|(192\\.168\\..*)|(localhost)|(\\[?\\:\\:1/.*)|(\\[?fc.*\\:.*)|(\\[?fd.*\\:.*)|(\\[?(fe80|0)\\:0\\:0\\:0\\:0\\:0\\:0\\:1.*)", Pattern.CASE_INSENSITIVE); private static final String LOCALHOST_IPv6_PATTERN = "(\\[?fe80\\:\\:(/.*|\\z))|(\\[?0\\:0\\:0\\:0\\:0\\:0\\:0\\:1.*)|(\\[?\\:\\:1(/.*|\\z))";
private static final String INTRANET_IPv4_PATTERN = "(10\\..*)|(172\\.(1[6-9]|2[0-9]|3[0-1])\\..*)|(169\\.254\\..*)|(192\\.168\\..*)";
private static final String INTRANET_IPv6_PATTERN = "(\\[?(fc|fd).*\\:.*)";
private static final Pattern LOCALHOST_PATTERNS = Pattern.compile("(localhost)|" + LOCALHOST_IPv4_PATTERN + "|" + LOCALHOST_IPv6_PATTERN, Pattern.CASE_INSENSITIVE);
private static final Pattern INTRANET_PATTERNS = Pattern.compile(LOCALHOST_PATTERNS.pattern() + "|" + INTRANET_IPv4_PATTERN + "|" + INTRANET_IPv6_PATTERN, Pattern.CASE_INSENSITIVE);
private static final int MAX_NAME_CACHE_HIT_SIZE = 10000; private static final int MAX_NAME_CACHE_HIT_SIZE = 10000;
private static final int MAX_NAME_CACHE_MISS_SIZE = 1000; private static final int MAX_NAME_CACHE_MISS_SIZE = 1000;
@ -90,6 +95,92 @@ public class Domains {
public static long cacheHit_Hit = 0, cacheHit_Miss = 0, cacheHit_Insert = 0; // for statistics only; do not write public static long cacheHit_Hit = 0, cacheHit_Miss = 0, cacheHit_Insert = 0; // for statistics only; do not write
public static long cacheMiss_Hit = 0, cacheMiss_Miss = 0, cacheMiss_Insert = 0; // for statistics only; do not write public static long cacheMiss_Hit = 0, cacheMiss_Miss = 0, cacheMiss_Insert = 0; // for statistics only; do not write
private static Set<InetAddress> myHostAddresses = new HashSet<InetAddress>();
private static Set<InetAddress> localHostAddresses = new HashSet<InetAddress>(); // subset of myHostAddresses
private static Set<InetAddress> publicIPv4HostAddresses = new HashSet<InetAddress>(); // subset of myHostAddresses
private static Set<InetAddress> publicIPv6HostAddresses = new HashSet<InetAddress>(); // subset of myHostAddresses
private static Set<String> myHostNames = new HashSet<String>();
private static Set<String> localHostNames = new HashSet<String>(); // subset of myHostNames
private static Set<String> publicIPv4HostNames = new HashSet<String>(); // subset of myHostNames
private static Set<String> publicIPv6HostNames = new HashSet<String>(); // subset of myHostNames
static {
myHostNames.add(LOCALHOST);
localHostNames.add(LOCALHOST);
try {
InetAddress localHostAddress = InetAddress.getLocalHost();
if (localHostAddress != null) myHostAddresses.add(localHostAddress);
} catch (final UnknownHostException e) {}
try {
final InetAddress[] moreAddresses = InetAddress.getAllByName(LOCALHOST_NAME);
if (moreAddresses != null) myHostAddresses.addAll(Arrays.asList(moreAddresses));
} catch (final UnknownHostException e) {}
// to get the local host name, a dns lookup is necessary.
// if such a lookup blocks, it can cause that the static initiatializer does not finish fast
// therefore we start the host name lookup as concurrent thread
// meanwhile the host name is "127.0.0.1" which is not completely wrong
new Thread() {
@Override
public void run() {
Thread.currentThread().setName("Domains: init");
// try to get local addresses from interfaces
try {
final Enumeration<NetworkInterface> nis = NetworkInterface.getNetworkInterfaces();
while (nis.hasMoreElements()) {
final NetworkInterface ni = nis.nextElement();
final Enumeration<InetAddress> addrs = ni.getInetAddresses();
while (addrs.hasMoreElements()) {
final InetAddress addr = addrs.nextElement();
if (addr != null) myHostAddresses.add(addr);
}
}
} catch (final SocketException e) {
}
// now look up the host name
try {
LOCALHOST_NAME = getHostName(InetAddress.getLocalHost());
} catch (final UnknownHostException e) {}
// after the host name was resolved, we try to look up more local addresses
// using the host name:
try {
final InetAddress[] moreAddresses = InetAddress.getAllByName(LOCALHOST_NAME);
if (moreAddresses != null) myHostAddresses.addAll(Arrays.asList(moreAddresses));
} catch (final UnknownHostException e) {
}
// fill a cache of local host names
for (final InetAddress a: myHostAddresses) {
final String hostname = getHostName(a);
if (hostname != null) {
myHostNames.add(hostname);
myHostNames.add(a.getHostAddress());
}
// we write the local tests into variables to be able to debug these values
boolean isAnyLocalAddress = a.isAnyLocalAddress();
boolean isLinkLocalAddress = a.isLinkLocalAddress(); // true i.e. for localhost/fe80:0:0:0:0:0:0:1%1, myhost.local/fe80:0:0:0:223:dfff:fedf:30ce%7
boolean isLoopbackAddress = a.isLoopbackAddress(); // true i.e. for localhost/0:0:0:0:0:0:0:1, localhost/127.0.0.1
boolean isSiteLocalAddress = a.isSiteLocalAddress(); // true i.e. for myhost.local/192.168.1.33
if (isAnyLocalAddress || isLinkLocalAddress || isLoopbackAddress || isSiteLocalAddress) {
ConcurrentLog.info("Domain Init", "local host address: " + a + " (local)");
localHostAddresses.add(a);
if (hostname != null) {localHostNames.add(hostname); localHostNames.add(a.getHostAddress());}
} else {
ConcurrentLog.info("Domain Init", "local host address: " + a + " (public)");
if (a instanceof Inet4Address) {
publicIPv4HostAddresses.add(a);
if (hostname != null) {publicIPv4HostNames.add(hostname); publicIPv4HostNames.add(a.getHostAddress());}
} else {
publicIPv6HostAddresses.add(a);
if (hostname != null) {publicIPv6HostNames.add(hostname); publicIPv6HostNames.add(a.getHostAddress());}
}
}
}
}
}.start();
}
/** /**
* ! ! ! A T T E N T I O N A T T E N T I O N A T T E N T I O N ! ! ! * ! ! ! A T T E N T I O N A T T E N T I O N A T T E N T I O N ! ! !
* *
@ -810,7 +901,7 @@ public class Domains {
// add also the isLocal host name caches // add also the isLocal host name caches
final boolean localp = ip.isAnyLocalAddress() || ip.isLinkLocalAddress() || ip.isSiteLocalAddress(); final boolean localp = ip.isAnyLocalAddress() || ip.isLinkLocalAddress() || ip.isSiteLocalAddress();
if (localp) { if (localp) {
localHostNames.add(host); myHostNames.add(host);
} else { } else {
if (globalHosts != null) try { if (globalHosts != null) try {
globalHosts.add(host); globalHosts.add(host);
@ -847,131 +938,54 @@ public class Domains {
return nameCacheNoCachingPatterns.size(); return nameCacheNoCachingPatterns.size();
} }
private static Set<InetAddress> localHostAddresses = new HashSet<InetAddress>(); /**
private static Set<String> localHostNames = new HashSet<String>(); * myPublicLocalIP() returns the IP of this host which is reachable in the public network under this address
static { * This is deprecated since it should be possible that the host is reachable with more than one IP
try { * That is particularly the case if the host supports IPv4 and IPv6.
final InetAddress localHostAddress = InetAddress.getLocalHost(); * Please use myPublicIPv4() or (preferred) myPublicIPv6() instead.
if (localHostAddress != null) localHostAddresses.add(localHostAddress); * @return
} catch (final UnknownHostException e) {} */
try { @Deprecated
final InetAddress[] moreAddresses = InetAddress.getAllByName(LOCALHOST_NAME); public static InetAddress myPublicLocalIP() {
if (moreAddresses != null) localHostAddresses.addAll(Arrays.asList(moreAddresses)); // for backward compatibility, we try to select a IPv4 address here.
} catch (final UnknownHostException e) {} // future methods should use myPublicIPs() and prefer IPv6
if (publicIPv4HostAddresses.size() > 0) return publicIPv4HostAddresses.iterator().next();
// to get the local host name, a dns lookup is necessary. if (publicIPv6HostAddresses.size() > 0) return publicIPv6HostAddresses.iterator().next();
// if such a lookup blocks, it can cause that the static initiatializer does not finish fast return null;
// therefore we start the host name lookup as concurrent thread
// meanwhile the host name is "127.0.0.1" which is not completely wrong
new Thread() {
@Override
public void run() {
Thread.currentThread().setName("Domains: init");
// try to get local addresses from interfaces
try {
final Enumeration<NetworkInterface> nis = NetworkInterface.getNetworkInterfaces();
while (nis.hasMoreElements()) {
final NetworkInterface ni = nis.nextElement();
final Enumeration<InetAddress> addrs = ni.getInetAddresses();
while (addrs.hasMoreElements()) {
final InetAddress addr = addrs.nextElement();
if (addr != null) localHostAddresses.add(addr);
}
}
} catch (final SocketException e) {
} }
// now look up the host name public static Set<String> myPublicIPs() {
try { Set<String> h = new HashSet<>(publicIPv4HostAddresses.size() + publicIPv6HostAddresses.size());
LOCALHOST_NAME = getHostName(InetAddress.getLocalHost()); for (InetAddress i: publicIPv4HostAddresses) h.add(i.getHostAddress());
} catch (final UnknownHostException e) {} for (InetAddress i: publicIPv6HostAddresses) h.add(i.getHostAddress());
return h;
// after the host name was resolved, we try to look up more local addresses
// using the host name:
try {
final InetAddress[] moreAddresses = InetAddress.getAllByName(LOCALHOST_NAME);
if (moreAddresses != null) localHostAddresses.addAll(Arrays.asList(moreAddresses));
} catch (final UnknownHostException e) {
} }
// fill a cache of local host names /**
for (final InetAddress a: localHostAddresses) { * Get all IPv4 addresses which are assigned to the local host but are public IP addresses.
final String hostname = getHostName(a); * These should be the possible addresses which can be used to access this peer.
if (hostname != null) { * @return the public IPv4 Addresses of this peer
localHostNames.add(hostname); */
localHostNames.add(a.getHostAddress()); public static Set<InetAddress> myPublicIPv4() {
} return publicIPv4HostAddresses;
}
}
}.start();
} }
public static InetAddress myPublicLocalIP() { /**
// list all addresses * Get all IPv6 addresses which are assigned to the local host but are public IP addresses.
// for (int i = 0; i < localHostAddresses.length; i++) System.out.println("IP: " + localHostAddresses[i].getHostAddress()); // DEBUG * These should be the possible addresses which can be used to access this peer.
if (localHostAddresses.isEmpty()) { * @return the public IPv6 addresses of this peer
return null; */
} public static Set<InetAddress> myPublicIPv6() {
if (localHostAddresses.size() == 1) { return publicIPv6HostAddresses;
// only one network connection available
return localHostAddresses.iterator().next();
}
// we have more addresses, find an address that is not local
int b0, b1;
for (final InetAddress a: localHostAddresses) {
b0 = 0Xff & a.getAddress()[0];
b1 = 0Xff & a.getAddress()[1];
if (b0 != 10 && // class A reserved
b0 != 127 && // loopback
(b0 != 172 || b1 < 16 || b1 > 31) && // class B reserved
(b0 != 192 || b1 != 168) && // class C reserved
(a.getHostAddress().indexOf(':',0) < 0))
return a;
}
// there is only a local address
// return that one that is returned with InetAddress.getLocalHost()
// if appropriate
try {
final InetAddress localHostAddress = InetAddress.getLocalHost();
if (localHostAddress != null &&
(0Xff & localHostAddress.getAddress()[0]) != 127 &&
localHostAddress.getHostAddress().indexOf(':',0) < 0) return localHostAddress;
} catch (final UnknownHostException e) {
}
// we filter out the loopback address 127.0.0.1 and all addresses without a name
for (final InetAddress a: localHostAddresses) {
if ((0Xff & a.getAddress()[0]) != 127 &&
a.getHostAddress().indexOf(':',0) < 0 &&
a.getHostName() != null &&
!a.getHostName().isEmpty()) return a;
}
// if no address has a name, then take any other than the loopback
for (final InetAddress a: localHostAddresses) {
if ((0Xff & a.getAddress()[0]) != 127 &&
a.getHostAddress().indexOf(':',0) < 0) return a;
}
// if all fails, give back whatever we have
for (final InetAddress a: localHostAddresses) {
if (a.getHostAddress().indexOf(':',0) < 0) return a;
}
// finally, just get any
return localHostAddresses.iterator().next();
} }
/** /**
* generate a list of intranet InetAddresses without the loopback address 127.0.0.1 * generate a list of intranet InetAddresses
* @return list of all intranet addresses * @return list of all intranet addresses
*/ */
public static Set<InetAddress> myIntranetIPs() { public static Set<InetAddress> myIntranetIPs() {
// list all local addresses if (myHostAddresses.size() < 1) try {Thread.sleep(1000);} catch (final InterruptedException e) {}
if (localHostAddresses.size() < 1) try {Thread.sleep(1000);} catch (final InterruptedException e) {} return localHostAddresses;
final Set<InetAddress> list = new HashSet<InetAddress>();
if (localHostAddresses.isEmpty()) return list; // give up
for (final InetAddress a: localHostAddresses) {
if ((0Xff & a.getAddress()[0]) == 127) continue;
list.add(a);
}
return list;
} }
public static boolean isThisHostIP(final String hostName) { public static boolean isThisHostIP(final String hostName) {
@ -982,7 +996,7 @@ public class Domains {
final InetAddress clientAddress = Domains.dnsResolve(hostName); final InetAddress clientAddress = Domains.dnsResolve(hostName);
if (clientAddress == null) return false; if (clientAddress == null) return false;
if (clientAddress.isAnyLocalAddress() || clientAddress.isLoopbackAddress()) return true; if (clientAddress.isAnyLocalAddress() || clientAddress.isLoopbackAddress()) return true;
for (final InetAddress a: localHostAddresses) { for (final InetAddress a: myHostAddresses) {
if (a.equals(clientAddress)) { if (a.equals(clientAddress)) {
isThisHostIP = true; isThisHostIP = true;
break; break;
@ -999,7 +1013,7 @@ public class Domains {
try { try {
if (clientAddress.isAnyLocalAddress() || clientAddress.isLoopbackAddress()) return true; if (clientAddress.isAnyLocalAddress() || clientAddress.isLoopbackAddress()) return true;
for (final InetAddress a: localHostAddresses) { for (final InetAddress a: myHostAddresses) {
if (a.equals(clientAddress)) { if (a.equals(clientAddress)) {
isThisHostIP = true; isThisHostIP = true;
break; break;
@ -1024,7 +1038,9 @@ public class Domains {
* @return true if the host from the string is the localhost * @return true if the host from the string is the localhost
*/ */
public static boolean isLocalhost(final String host) { public static boolean isLocalhost(final String host) {
return (host != null && LOCALHOST_PATTERNS.matcher(host).matches()); return host == null || // filesystems do not have host names
LOCALHOST_PATTERNS.matcher(host).matches() ||
localHostNames.contains(host);
} }
/** /**
@ -1038,7 +1054,9 @@ public class Domains {
*/ */
public static boolean isIntranet(final String host) { public static boolean isIntranet(final String host) {
return (noLocalCheck || // DO NOT REMOVE THIS! it is correct to return true if the check is off return (noLocalCheck || // DO NOT REMOVE THIS! it is correct to return true if the check is off
(host != null && INTRANET_PATTERNS.matcher(host).matches())); host == null || // filesystems do not have host names
INTRANET_PATTERNS.matcher(host).matches()) ||
localHostNames.contains(host);
} }
/** /**
@ -1060,10 +1078,7 @@ public class Domains {
// check local ip addresses // check local ip addresses
if (isIntranet(host)) return true; if (isIntranet(host)) return true;
if (hostaddress != null && ( if (hostaddress != null && (isIntranet(hostaddress.getHostAddress()) || isLocal(hostaddress))) return true;
isIntranet(hostaddress.getHostAddress()) ||
isLocal(hostaddress)
)) return true;
// check if there are other local IP addresses that are not in // check if there are other local IP addresses that are not in
// the standard IP range // the standard IP range

@ -134,6 +134,7 @@ public class RequestHeader extends HeaderFramework {
return false; return false;
} }
final String refererHost = this.refererHost(); final String refererHost = this.refererHost();
return refererHost == null || refererHost.isEmpty() || Domains.isLocalhost(refererHost); if (refererHost == null || refererHost.isEmpty() || Domains.isLocalhost(refererHost)) return true;
return false;
} }
} }

@ -37,6 +37,11 @@ import java.security.cert.X509Certificate;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.Set; import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.FutureTask;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import javax.net.ssl.SSLContext; import javax.net.ssl.SSLContext;
@ -58,6 +63,7 @@ import org.apache.http.HttpHost;
import org.apache.http.HttpResponse; import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope; import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.CredentialsProvider; import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.CookieSpecs; import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig; import org.apache.http.client.config.RequestConfig;
@ -77,7 +83,6 @@ import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.conn.socket.ConnectionSocketFactory; import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory; import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.entity.InputStreamEntity;
import org.apache.http.entity.mime.MultipartEntityBuilder; import org.apache.http.entity.mime.MultipartEntityBuilder;
import org.apache.http.entity.mime.content.ContentBody; import org.apache.http.entity.mime.content.ContentBody;
import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider;
@ -112,7 +117,7 @@ public class HTTPClient {
private long upbytes = 0L; private long upbytes = 0L;
private String host = null; private String host = null;
private final long timeout; private final long timeout;
private static ExecutorService executor = Executors.newFixedThreadPool(200);
public HTTPClient(final ClientIdentification.Agent agent) { public HTTPClient(final ClientIdentification.Agent agent) {
super(); super();
@ -236,23 +241,6 @@ public class HTTPClient {
} }
} }
// public static void setAuth(final String host, final int port, final String user, final String pw) {
// final UsernamePasswordCredentials creds = new UsernamePasswordCredentials(user, pw);
// final AuthScope scope = new AuthScope(host, port);
// credsProvider.setCredentials(scope, creds);
// httpClient.getParams().setParameter(ClientContext.CREDS_PROVIDER, credsProvider);
// }
// /**
// * this method sets a host on which more than the default of 2 router per host are allowed
// *
// * @param the host to be raised in 'route per host'
// */
// public static void setMaxRouteHost(final String host) {
// final HttpHost mHost = new HttpHost(host);
// ((PoolingClientConnectionManager) httpClient.getConnectionManager()).setMaxPerRoute(new HttpRoute(mHost), 50);
// }
/** /**
* This method sets the Header used for the request * This method sets the Header used for the request
* *
@ -454,9 +442,11 @@ public class HTTPClient {
* @param length the contentlength * @param length the contentlength
* @throws IOException * @throws IOException
*/ */
/*
public void POST(final String uri, final InputStream instream, final long length, final boolean concurrent) throws IOException { public void POST(final String uri, final InputStream instream, final long length, final boolean concurrent) throws IOException {
POST(new MultiProtocolURL(uri), instream, length, concurrent); POST(new MultiProtocolURL(uri), instream, length, concurrent);
} }
*/
/** /**
* This method POSTs a page from the server. * This method POSTs a page from the server.
@ -490,10 +480,12 @@ public class HTTPClient {
* @return content bytes * @return content bytes
* @throws IOException * @throws IOException
*/ */
/*
public byte[] POSTbytes(final String uri, final Map<String, ContentBody> parts, final boolean usegzip, final boolean concurrent) throws IOException { public byte[] POSTbytes(final String uri, final Map<String, ContentBody> parts, final boolean usegzip, final boolean concurrent) throws IOException {
final MultiProtocolURL url = new MultiProtocolURL(uri); final MultiProtocolURL url = new MultiProtocolURL(uri);
return POSTbytes(url, url.getHost(), parts, usegzip, concurrent); return POSTbytes(url, url.getHost(), parts, usegzip, concurrent);
} }
*/
/** /**
* send data to the server named by vhost * send data to the server named by vhost
@ -512,8 +504,7 @@ public class HTTPClient {
if (vhost == null) setHost(Domains.LOCALHOST); if (vhost == null) setHost(Domains.LOCALHOST);
final MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create(); final MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create();
for (final Entry<String,ContentBody> part : post.entrySet()) for (final Entry<String,ContentBody> part : post.entrySet()) entityBuilder.addPart(part.getKey(), part.getValue());
entityBuilder.addPart(part.getKey(), part.getValue());
final HttpEntity multipartEntity = entityBuilder.build(); final HttpEntity multipartEntity = entityBuilder.build();
// statistics // statistics
this.upbytes = multipartEntity.getContentLength(); this.upbytes = multipartEntity.getContentLength();
@ -536,6 +527,7 @@ public class HTTPClient {
* @return content bytes * @return content bytes
* @throws IOException * @throws IOException
*/ */
/*
public byte[] POSTbytes(final String uri, final InputStream instream, final long length, final boolean concurrent) throws IOException { public byte[] POSTbytes(final String uri, final InputStream instream, final long length, final boolean concurrent) throws IOException {
final MultiProtocolURL url = new MultiProtocolURL(uri); final MultiProtocolURL url = new MultiProtocolURL(uri);
final HttpPost httpPost = new HttpPost(url.toNormalform(true)); final HttpPost httpPost = new HttpPost(url.toNormalform(true));
@ -549,6 +541,7 @@ public class HTTPClient {
httpPost.setEntity(inputStreamEntity); httpPost.setEntity(inputStreamEntity);
return getContentBytes(httpPost, Integer.MAX_VALUE, concurrent); return getContentBytes(httpPost, Integer.MAX_VALUE, concurrent);
} }
*/
/** /**
* *
@ -682,30 +675,26 @@ public class HTTPClient {
assert !hrequest.expectContinue(); assert !hrequest.expectContinue();
} }
Thread.currentThread().setName("HTTPClient-" + httpUriRequest.getURI().getHost()); Thread.currentThread().setName("HTTPClient-" + httpUriRequest.getURI());
final long time = System.currentTimeMillis(); final long time = System.currentTimeMillis();
try { try {
final CloseableHttpClient client = clientBuilder.build(); final CloseableHttpClient client = clientBuilder.build();
if (concurrent) { if (concurrent) {
final CloseableHttpResponse[] thr = new CloseableHttpResponse[]{null}; FutureTask<CloseableHttpResponse> t = new FutureTask<CloseableHttpResponse>(new Callable<CloseableHttpResponse>() {
final Throwable[] te = new Throwable[]{null};
Thread t = new Thread() {
@Override @Override
public void run() { public CloseableHttpResponse call() throws ClientProtocolException, IOException {
this.setName("HTTPClient.execute(" + httpUriRequest.getURI() + ")"); CloseableHttpResponse response = client.execute(httpUriRequest, context);
try { return response;
thr[0] = client.execute(httpUriRequest, context);
} catch (Throwable e) {
te[0] = e;
} }
} });
}; executor.execute(t);
t.start(); try {
try {t.join(this.timeout);} catch (InterruptedException e) {} this.httpResponse = t.get(this.timeout, TimeUnit.MILLISECONDS);
if (t.isAlive()) try {t.interrupt();} catch (Throwable e) {} } catch (ExecutionException e) {
if (te[0] != null) throw te[0]; throw e.getCause();
if (thr[0] == null) throw new IOException("timout to client after " + this.timeout + "ms"); } catch (Throwable e) {}
this.httpResponse = thr[0]; try {t.cancel(true);} catch (Throwable e) {}
if (this.httpResponse == null) throw new IOException("timout to client after " + this.timeout + "ms");
} else { } else {
this.httpResponse = client.execute(httpUriRequest, context); this.httpResponse = client.execute(httpUriRequest, context);
} }

@ -43,6 +43,7 @@ public class CommonPattern {
public final static Pattern SEMICOLON = Pattern.compile(";"); public final static Pattern SEMICOLON = Pattern.compile(";");
public final static Pattern DOUBLEPOINT = Pattern.compile(":"); public final static Pattern DOUBLEPOINT = Pattern.compile(":");
public final static Pattern SLASH = Pattern.compile("/"); public final static Pattern SLASH = Pattern.compile("/");
public final static Pattern PIPE = Pattern.compile("\\|");
public final static Pattern BACKSLASH = Pattern.compile("\\\\"); public final static Pattern BACKSLASH = Pattern.compile("\\\\");
public final static Pattern QUESTION = Pattern.compile("\\?"); public final static Pattern QUESTION = Pattern.compile("\\?");
public final static Pattern AMP = Pattern.compile("&"); public final static Pattern AMP = Pattern.compile("&");

@ -36,7 +36,6 @@ import java.util.concurrent.BlockingQueue;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import net.yacy.contentcontrol.ContentControlFilterUpdateThread; import net.yacy.contentcontrol.ContentControlFilterUpdateThread;
import net.yacy.cora.document.analysis.Classification.ContentDomain;
import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.document.id.AnchorURL; import net.yacy.cora.document.id.AnchorURL;

@ -82,7 +82,7 @@ public class YacyDomainHandler extends AbstractHandler implements Handler {
newHost = hostPort; newHost = hostPort;
newPort = 80; newPort = 80;
} }
if (newHost.equals(alternativeResolvers.myIP())) return; if (alternativeResolvers.myIPs().contains(newHost)) return;
if (Domains.isLocal(newHost, null)) return; if (Domains.isLocal(newHost, null)) return;
RequestDispatcher dispatcher = request.getRequestDispatcher(path + target); RequestDispatcher dispatcher = request.getRequestDispatcher(path + target);
dispatcher.forward(new DomainRequestWrapper(request, newHost, newPort), response); dispatcher.forward(new DomainRequestWrapper(request, newHost, newPort), response);

@ -37,7 +37,6 @@ import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants; import net.yacy.search.SwitchboardConstants;
import com.google.common.io.Files; import com.google.common.io.Files;
import java.util.concurrent.ExecutionException;
import net.yacy.cora.lod.vocabulary.Tagging; import net.yacy.cora.lod.vocabulary.Tagging;
import net.yacy.cora.protocol.TimeoutRequest; import net.yacy.cora.protocol.TimeoutRequest;
import net.yacy.cora.storage.Configuration.Entry; import net.yacy.cora.storage.Configuration.Entry;

@ -31,10 +31,9 @@ import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedHashSet; import java.util.LinkedHashSet;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.Random; import java.util.Random;
import java.util.Set; import java.util.Set;
import java.util.SortedMap; import java.util.SortedSet;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
@ -55,18 +54,12 @@ import net.yacy.peers.operation.yacyVersion;
*/ */
public class DHTSelection { public class DHTSelection {
public static Set<Seed> selectClusterPeers(final SeedDB seedDB, final SortedMap<byte[], String> peerhashes) { public static Set<Seed> selectClusterPeers(final SeedDB seedDB, final SortedSet<byte[]> peerhashes) {
final Iterator<Map.Entry<byte[], String>> i = peerhashes.entrySet().iterator();
final Set<Seed> l = new HashSet<Seed>(); final Set<Seed> l = new HashSet<Seed>();
Map.Entry<byte[], String> entry;
Seed s; Seed s;
while (i.hasNext()) { for (byte[] hashb: peerhashes) {
entry = i.next(); s = seedDB.get(ASCII.String(hashb)); // should be getConnected; get only during testing time
s = seedDB.get(ASCII.String(entry.getKey())); // should be getConnected; get only during testing time if (s != null) l.add(s);
if (s != null) {
s.setAlternativeAddress(entry.getValue());
l.add(s);
}
} }
return l; return l;
} }

@ -46,6 +46,7 @@ import java.util.Iterator;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore; import java.util.concurrent.Semaphore;
@ -101,7 +102,7 @@ public class Network
// ensure that correct IP is used // ensure that correct IP is used
final String staticIP = sb.getConfig("staticIP", ""); final String staticIP = sb.getConfig("staticIP", "");
if ( staticIP.length() != 0 && Seed.isProperIP(staticIP) == null ) { if (staticIP.length() != 0 && Seed.isProperIP(staticIP)) {
serverCore.useStaticIP = true; serverCore.useStaticIP = true;
sb.peers.mySeed().setIP(staticIP); sb.peers.mySeed().setIP(staticIP);
log.info("staticIP set to " + staticIP); log.info("staticIP set to " + staticIP);
@ -118,9 +119,7 @@ public class Network
} }
public final void publishSeedList() { public final void publishSeedList() {
if ( log.isFine() ) { if (log.isFine()) log.fine("yacyCore.publishSeedList: Triggered Seed Publish");
log.fine("yacyCore.publishSeedList: Triggered Seed Publish");
}
/* /*
if (oldIPStamp.equals((String) seedDB.mySeed.get(yacySeed.IP, "127.0.0.1"))) if (oldIPStamp.equals((String) seedDB.mySeed.get(yacySeed.IP, "127.0.0.1")))
@ -131,14 +130,11 @@ public class Network
yacyCore.log.logDebug("***DEBUG publishSeedList: I can reach myself"); yacyCore.log.logDebug("***DEBUG publishSeedList: I can reach myself");
*/ */
if ( (this.sb.peers.lastSeedUpload_myIP.equals(this.sb.peers.mySeed().getIP())) if ((this.sb.peers.mySeed().getIPs().contains(this.sb.peers.lastSeedUpload_myIP))
&& (this.sb.peers.lastSeedUpload_seedDBSize == this.sb.peers.sizeConnected()) && (this.sb.peers.lastSeedUpload_seedDBSize == this.sb.peers.sizeConnected())
&& (System.currentTimeMillis() - this.sb.peers.lastSeedUpload_timeStamp < 1000 * 60 * 60 * 24) && (System.currentTimeMillis() - this.sb.peers.lastSeedUpload_timeStamp < 1000 * 60 * 60 * 24)
&& (this.sb.peers.mySeed().isPrincipal()) ) { && (this.sb.peers.mySeed().isPrincipal()) ) {
if ( log.isFine() ) { if (log.isFine()) log.fine("yacyCore.publishSeedList: not necessary to publish: oldIP is equal, sizeConnected is equal and I can reach myself under the old IP.");
log
.fine("yacyCore.publishSeedList: not necessary to publish: oldIP is equal, sizeConnected is equal and I can reach myself under the old IP.");
}
return; return;
} }
@ -162,9 +158,7 @@ public class Network
if ( seedUploadMethod.equals("") ) { if ( seedUploadMethod.equals("") ) {
this.sb.setConfig("seedUploadMethod", "none"); this.sb.setConfig("seedUploadMethod", "none");
} }
if ( log.isFine() ) { if (log.isFine()) log.fine("yacyCore.publishSeedList: No uploading method configured");
log.fine("yacyCore.publishSeedList: No uploading method configured");
}
return; return;
} }
} }
@ -231,13 +225,7 @@ public class Network
@Override @Override
public final void run() { public final void run() {
try { try {
this.added = this.added = Protocol.hello(Network.this.sb.peers.mySeed(), Network.this.sb.peers.peerActions, this.seed);
Protocol.hello(
Network.this.sb.peers.mySeed(),
Network.this.sb.peers.peerActions,
this.seed.getClusterAddress(),
this.seed.hash,
this.seed.getName());
if ( this.added < 0 ) { if ( this.added < 0 ) {
// no or wrong response, delete that address // no or wrong response, delete that address
final String cause = "peer ping to peer resulted in error response (added < 0)"; final String cause = "peer ping to peer resulted in error response (added < 0)";
@ -246,7 +234,7 @@ public class Network
+ " peer '" + " peer '"
+ this.seed.getName() + this.seed.getName()
+ "' from " + "' from "
+ this.seed.getPublicAddress() + this.seed.getIPs()
+ ": " + ": "
+ cause); + cause);
Network.this.sb.peers.peerActions.peerDeparture(this.seed, cause); Network.this.sb.peers.peerActions.peerDeparture(this.seed, cause);
@ -258,7 +246,7 @@ public class Network
+ " peer '" + " peer '"
+ this.seed.getName() + this.seed.getName()
+ "' at " + "' at "
+ this.seed.getPublicAddress()); + this.seed.getIPs());
// check if seed's lastSeen has been updated // check if seed's lastSeen has been updated
final Seed newSeed = Network.this.sb.peers.getConnected(this.seed.hash); final Seed newSeed = Network.this.sb.peers.getConnected(this.seed.hash);
if ( newSeed != null ) { if ( newSeed != null ) {
@ -269,7 +257,7 @@ public class Network
+ " peer '" + " peer '"
+ this.seed.getName() + this.seed.getName()
+ "' at " + "' at "
+ this.seed.getPublicAddress() + this.seed.getIPs()
+ " is not online." + " is not online."
+ " Removing Peer from connected"); + " Removing Peer from connected");
} }
@ -284,7 +272,7 @@ public class Network
+ " peer '" + " peer '"
+ this.seed.getName() + this.seed.getName()
+ "' at " + "' at "
+ this.seed.getPublicAddress() + this.seed.getIPs()
+ " with old LastSeen: '" + " with old LastSeen: '"
+ my_SHORT_SECOND_FORMATTER.format(new Date(newSeed + my_SHORT_SECOND_FORMATTER.format(new Date(newSeed
.getLastSeenUTC())) + "'"); .getLastSeenUTC())) + "'");
@ -299,7 +287,7 @@ public class Network
+ " peer '" + " peer '"
+ this.seed.getName() + this.seed.getName()
+ "' at " + "' at "
+ this.seed.getPublicAddress() + this.seed.getIPs()
+ " with old LastSeen: '" + " with old LastSeen: '"
+ my_SHORT_SECOND_FORMATTER.format(new Date(newSeed + my_SHORT_SECOND_FORMATTER.format(new Date(newSeed
.getLastSeenUTC())) .getLastSeenUTC()))
@ -319,7 +307,7 @@ public class Network
+ " peer '" + " peer '"
+ this.seed.getName() + this.seed.getName()
+ "' at " + "' at "
+ this.seed.getPublicAddress() + this.seed.getIPs()
+ " not in connectedDB"); + " not in connectedDB");
} }
} }
@ -364,17 +352,14 @@ public class Network
if ( attempts > PING_INITIAL ) { if ( attempts > PING_INITIAL ) {
attempts = PING_INITIAL; attempts = PING_INITIAL;
} }
final Map<byte[], String> ch = Switchboard.getSwitchboard().clusterhashes; final Set<byte[]> ch = Switchboard.getSwitchboard().clusterhashes;
seeds = DHTSelection.seedsByAge(this.sb.peers, true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection seeds = DHTSelection.seedsByAge(this.sb.peers, true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection
// add also all peers from cluster if this is a public robinson cluster // add also all peers from cluster if this is a public robinson cluster
if ( ch != null ) { if ( ch != null ) {
final Iterator<Map.Entry<byte[], String>> i = ch.entrySet().iterator();
String hash; String hash;
Map.Entry<byte[], String> entry;
Seed seed; Seed seed;
while ( i.hasNext() ) { for (byte[] hashb: ch) {
entry = i.next(); hash = ASCII.String(hashb);
hash = ASCII.String(entry.getKey());
seed = seeds.get(hash); seed = seeds.get(hash);
if (seed == null) { if (seed == null) {
seed = this.sb.peers.get(hash); seed = this.sb.peers.get(hash);
@ -382,7 +367,6 @@ public class Network
continue; continue;
} }
} }
seed.setAlternativeAddress(entry.getValue());
seeds.put(hash, seed); seeds.put(hash, seed);
} }
} }
@ -441,17 +425,14 @@ public class Network
} }
i++; i++;
final String address = seed.getClusterAddress(); final String address = seed.getPublicAddress(seed.getIP());
if ( log.isFine() ) { if ( log.isFine() ) {
log.fine("HELLO #" + i + " to peer '" + seed.get(Seed.NAME, "") + "' at " + address); // debug log.fine("HELLO #" + i + " to peer '" + seed.get(Seed.NAME, "") + "' at " + address); // debug
} }
final String seederror = seed.isProper(false); final String seederror = seed.isProper(false);
if ( (address == null) || (seederror != null) ) { if ( (address == null) || (seederror != null) ) {
// we don't like that address, delete it // we don't like that address, delete it
this.sb.peers.peerActions.peerDeparture(seed, "peer ping to peer resulted in address = " this.sb.peers.peerActions.peerDeparture(seed, "peer ping to peer resulted in address = " + address + "; seederror = " + seederror);
+ address
+ "; seederror = "
+ seederror);
sync.acquire(); sync.acquire();
} else { } else {
// starting a new publisher thread // starting a new publisher thread
@ -567,16 +548,13 @@ public class Network
//if (ip.equals("")) ip = natLib.retrieveIP(DI604use, DI604pw); //if (ip.equals("")) ip = natLib.retrieveIP(DI604use, DI604pw);
// yacyCore.log.logDebug("DEBUG: new IP=" + ip); // yacyCore.log.logDebug("DEBUG: new IP=" + ip);
if ( Seed.isProperIP(ip) == null ) { if (Seed.isProperIP(ip)) {
this.sb.peers.mySeed().setIP(ip); this.sb.peers.mySeed().setIP(ip);
} }
if ( this.sb.peers.mySeed().get(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR).equals(Seed.PEERTYPE_JUNIOR) ) { if ( this.sb.peers.mySeed().get(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR).equals(Seed.PEERTYPE_JUNIOR) ) {
this.sb.peers.mySeed().put(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR); // to start bootstraping, we need to be recognised as PEERTYPE_SENIOR peer this.sb.peers.mySeed().put(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR); // to start bootstraping, we need to be recognised as PEERTYPE_SENIOR peer
} }
log.info("publish: no recipient found, our address is " log.info("publish: no recipient found, our address is " + this.sb.peers.mySeed().getIPs());
+ ((this.sb.peers.mySeed().getPublicAddress() == null) ? "unknown" : this.sb.peers
.mySeed()
.getPublicAddress()));
this.sb.peers.saveMySeed(); this.sb.peers.saveMySeed();
return 0; return 0;
} catch (final InterruptedException e ) { } catch (final InterruptedException e ) {

@ -28,6 +28,7 @@ import java.util.Map;
import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.feed.RSSMessage; import net.yacy.cora.document.feed.RSSMessage;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.storage.ConcurrentARC; import net.yacy.cora.storage.ConcurrentARC;
import net.yacy.kelondro.util.MapTools; import net.yacy.kelondro.util.MapTools;
import net.yacy.peers.operation.yacyVersion; import net.yacy.peers.operation.yacyVersion;
@ -66,7 +67,7 @@ public class PeerActions {
return false; return false;
} }
if ((this.seedDB.mySeedIsDefined()) && (seed.hash.equals(this.seedDB.mySeed().hash))) { if ((this.seedDB.mySeedIsDefined()) && (seed.hash.equals(this.seedDB.mySeed().hash))) {
Network.log.info("connect: SELF reference " + seed.getPublicAddress()); Network.log.info("connect: SELF reference " + seed.getIPs());
return false; return false;
} }
final String peerType = seed.get(Seed.PEERTYPE, Seed.PEERTYPE_VIRGIN); final String peerType = seed.get(Seed.PEERTYPE, Seed.PEERTYPE_VIRGIN);
@ -82,7 +83,7 @@ public class PeerActions {
return false; return false;
} }
final Seed doubleSeed = this.seedDB.lookupByIP(seed.getInetAddress(), seed.getPort(), true, false, false); final Seed doubleSeed = this.seedDB.lookupByIP(Domains.dnsResolve(seed.getIP()), seed.getPort(), true, false, false);
if ((doubleSeed != null) && (doubleSeed.getPort() == seed.getPort()) && (!(doubleSeed.hash.equals(seed.hash)))) { if ((doubleSeed != null) && (doubleSeed.getPort() == seed.getPort()) && (!(doubleSeed.hash.equals(seed.hash)))) {
// a user frauds with his peer different peer hashes // a user frauds with his peer different peer hashes
if (Network.log.isFine()) Network.log.fine("connect: rejecting FRAUD (double hashes " + doubleSeed.hash + "/" + seed.hash + " on same port " + seed.getPort() + ") peer " + seed.getName()); if (Network.log.isFine()) Network.log.fine("connect: rejecting FRAUD (double hashes " + doubleSeed.hash + "/" + seed.hash + " on same port " + seed.getPort() + ") peer " + seed.getName());
@ -107,7 +108,7 @@ public class PeerActions {
} }
if (Math.abs(nowUTC0Time - ctimeUTC0) / 1000 / 60 > 60 * 6 ) { if (Math.abs(nowUTC0Time - ctimeUTC0) / 1000 / 60 > 60 * 6 ) {
// the new connection is out-of-age, we reject the connection // the new connection is out-of-age, we reject the connection
if (Network.log.isFine()) Network.log.fine("connect: rejecting out-dated peer '" + seed.getName() + "' from " + seed.getPublicAddress() + "; nowUTC0=" + nowUTC0Time + ", seedUTC0=" + ctimeUTC0 + ", TimeDiff=" + formatInterval(Math.abs(nowUTC0Time - ctimeUTC0))); if (Network.log.isFine()) Network.log.fine("connect: rejecting out-dated peer '" + seed.getName() + "' from " + seed.getIPs() + "; nowUTC0=" + nowUTC0Time + ", seedUTC0=" + ctimeUTC0 + ", TimeDiff=" + formatInterval(Math.abs(nowUTC0Time - ctimeUTC0)));
return false; return false;
} }
@ -144,13 +145,13 @@ public class PeerActions {
if (!direct) { if (!direct) {
if (ctimeUTC0 < dtimeUTC0) { if (ctimeUTC0 < dtimeUTC0) {
// the disconnection was later, we reject the connection // the disconnection was later, we reject the connection
if (Network.log.isFine()) Network.log.fine("connect: rejecting disconnected peer '" + seed.getName() + "' from " + seed.getPublicAddress()); if (Network.log.isFine()) Network.log.fine("connect: rejecting disconnected peer '" + seed.getName() + "' from " + seed.getIPs());
return false; return false;
} }
} }
// this is a return of a lost peer // this is a return of a lost peer
if (Network.log.isFine()) Network.log.fine("connect: returned KNOWN " + peerType + " peer '" + seed.getName() + "' from " + seed.getPublicAddress()); if (Network.log.isFine()) Network.log.fine("connect: returned KNOWN " + peerType + " peer '" + seed.getName() + "' from " + seed.getIPs());
this.seedDB.addConnected(seed); this.seedDB.addConnected(seed);
return true; return true;
} }
@ -169,10 +170,10 @@ public class PeerActions {
// TODO: update seed name lookup cache // TODO: update seed name lookup cache
}*/ }*/
} catch (final NumberFormatException e) { } catch (final NumberFormatException e) {
if (Network.log.isFine()) Network.log.fine("connect: rejecting wrong peer '" + seed.getName() + "' from " + seed.getPublicAddress() + ". Cause: " + e.getMessage()); if (Network.log.isFine()) Network.log.fine("connect: rejecting wrong peer '" + seed.getName() + "' from " + seed.getIPs() + ". Cause: " + e.getMessage());
return false; return false;
} }
if (Network.log.isFine()) Network.log.fine("connect: updated KNOWN " + ((direct) ? "direct " : "") + peerType + " peer '" + seed.getName() + "' from " + seed.getPublicAddress()); if (Network.log.isFine()) Network.log.fine("connect: updated KNOWN " + ((direct) ? "direct " : "") + peerType + " peer '" + seed.getName() + "' from " + seed.getIPs());
this.seedDB.addConnected(seed); this.seedDB.addConnected(seed);
return true; return true;
} }
@ -181,10 +182,10 @@ public class PeerActions {
if ((this.seedDB.mySeedIsDefined()) && (seed.getIP().equals(this.seedDB.mySeed().getIP()))) { if ((this.seedDB.mySeedIsDefined()) && (seed.getIP().equals(this.seedDB.mySeed().getIP()))) {
// seed from the same IP as the calling client: can be // seed from the same IP as the calling client: can be
// the case if there runs another one over a NAT // the case if there runs another one over a NAT
if (Network.log.isFine()) Network.log.fine("connect: saved NEW seed (myself IP) " + seed.getPublicAddress()); if (Network.log.isFine()) Network.log.fine("connect: saved NEW seed (myself IP) " + seed.getIPs());
} else { } else {
// completely new seed // completely new seed
if (Network.log.isFine()) Network.log.fine("connect: saved NEW " + peerType + " peer '" + seed.getName() + "' from " + seed.getPublicAddress()); if (Network.log.isFine()) Network.log.fine("connect: saved NEW " + peerType + " peer '" + seed.getName() + "' from " + seed.getIPs());
} }
this.seedDB.addConnected(seed); this.seedDB.addConnected(seed);
return true; return true;
@ -204,7 +205,7 @@ public class PeerActions {
public void peerDeparture(final Seed peer, final String cause) { public void peerDeparture(final Seed peer, final String cause) {
if (peer == null) return; if (peer == null) return;
// we do this if we did not get contact with the other peer // we do this if we did not get contact with the other peer
if (Network.log.isFine()) Network.log.fine("connect: no contact to a " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_VIRGIN) + " peer '" + peer.getName() + "' at " + peer.getPublicAddress() + ". Cause: " + cause); if (Network.log.isFine()) Network.log.fine("connect: no contact to a " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_VIRGIN) + " peer '" + peer.getName() + "' at " + peer.getIPs() + ". Cause: " + cause);
synchronized (this.seedDB) { synchronized (this.seedDB) {
if (!this.seedDB.hasDisconnected(ASCII.getBytes(peer.hash))) { this.disconnects++; } if (!this.seedDB.hasDisconnected(ASCII.getBytes(peer.hash))) { this.disconnects++; }
peer.put(Seed.DCT, Long.toString(System.currentTimeMillis())); peer.put(Seed.DCT, Long.toString(System.currentTimeMillis()));

@ -51,6 +51,7 @@ import java.net.InetAddress;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.LinkedHashSet; import java.util.LinkedHashSet;
@ -128,14 +129,7 @@ import org.apache.solr.common.SolrInputDocument;
public final class Protocol { public final class Protocol {
private static byte[] postToFile( @Deprecated
final Seed target,
final String filename,
final Map<String, ContentBody> parts,
final int timeout) throws IOException {
return postToFile(target.getClusterAddress(), target.hash, filename, parts, timeout);
}
private static byte[] postToFile( private static byte[] postToFile(
final SeedDB seedDB, final SeedDB seedDB,
final String targetHash, final String targetHash,
@ -176,9 +170,8 @@ public final class Protocol {
public static int hello( public static int hello(
final Seed mySeed, final Seed mySeed,
final PeerActions peerActions, final PeerActions peerActions,
final String address, final Seed otherSeed) {
final String otherHash, final String address = otherSeed.getPublicAddress(otherSeed.getIP());
final String otherName) {
Map<String, String> result = null; Map<String, String> result = null;
final String salt = crypt.randomSalt(); final String salt = crypt.randomSalt();
@ -198,7 +191,7 @@ public final class Protocol {
content = content =
httpClient.POSTbytes( httpClient.POSTbytes(
new MultiProtocolURL("http://" + address + "/yacy/hello.html"), new MultiProtocolURL("http://" + address + "/yacy/hello.html"),
Seed.b64Hash2hexHash(otherHash) + ".yacyh", Seed.b64Hash2hexHash(otherSeed.hash) + ".yacyh",
parts, parts,
false, true); false, true);
responseTime = System.currentTimeMillis() - start; responseTime = System.currentTimeMillis() - start;
@ -210,12 +203,7 @@ public final class Protocol {
+ "' interrupted."); + "' interrupted.");
return -1; return -1;
} }
Network.log.info("yacyClient.hello thread '" Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "', peer " + address + "; exception: " + e.getMessage());
+ Thread.currentThread().getName()
+ "', peer "
+ address
+ "; exception: "
+ e.getMessage());
// try again (go into loop) // try again (go into loop)
result = null; result = null;
} }
@ -238,35 +226,25 @@ public final class Protocol {
// check consistency with expectation // check consistency with expectation
Seed otherPeer = null; Seed otherPeer = null;
String seed; String seed;
if ( (otherHash != null) && (otherHash.length() > 0) && ((seed = result.get("seed0")) != null) ) { if ( (otherSeed.hash != null) && (otherSeed.hash.length() > 0) && ((seed = result.get("seed0")) != null) ) {
if ( seed.length() > Seed.maxsize ) { if ( seed.length() > Seed.maxsize ) {
Network.log.info("hello/client 0: rejected contacting seed; too large (" Network.log.info("hello/client 0: rejected contacting seed; too large (" + seed.length() + " > " + Seed.maxsize + ")");
+ seed.length()
+ " > "
+ Seed.maxsize
+ ")");
} else { } else {
try { try {
final int p = address.indexOf(':'); // patch the remote peer address to avoid that remote peers spoof the network with wrong addresses
if ( p < 0 ) { final int p = address.lastIndexOf(':');
return -1; if ( p < 0 ) return -1;
}
String h = address.substring(0, p); String h = address.substring(0, p);
if (h.charAt(0) == '[') h = h.substring(1);
if (h.charAt(h.length() - 1) == ']') h = h.substring(0, h.length() - 1);
InetAddress ie = Domains.dnsResolve(h); InetAddress ie = Domains.dnsResolve(h);
final String host = ie == null ? h : ie.getHostAddress(); // hack to prevent NPEs otherPeer = Seed.genRemoteSeed(seed, false, ie.getHostAddress());
otherPeer = Seed.genRemoteSeed(seed, false, host); if ( !otherPeer.hash.equals(otherSeed.hash) ) {
if ( !otherPeer.hash.equals(otherHash) ) { Network.log.info("yacyClient.hello: consistency error: otherPeer.hash = " + otherPeer.hash + ", otherHash = " + otherSeed.hash);
Network.log.info("yacyClient.hello: consistency error: otherPeer.hash = "
+ otherPeer.hash
+ ", otherHash = "
+ otherHash);
return -1; // no success return -1; // no success
} }
} catch (final IOException e ) { } catch (final IOException e ) {
Network.log.info("yacyClient.hello: consistency error: other seed bad:" Network.log.info("yacyClient.hello: consistency error: other seed bad:" + e.getMessage() + ", seed=" + seed);
+ e.getMessage()
+ ", seed="
+ seed);
return -1; // no success return -1; // no success
} }
} }
@ -281,11 +259,15 @@ public final class Protocol {
// set my own seed according to new information // set my own seed according to new information
// we overwrite our own IP number only // we overwrite our own IP number only
if ( serverCore.useStaticIP ) { if ( serverCore.useStaticIP ) {
mySeed.setIP(Switchboard.getSwitchboard().myPublicIP()); mySeed.setIPs(Switchboard.getSwitchboard().myPublicIPs());
} else { } else {
final String myIP = result.get("yourip"); final String myIP = result.get("yourip");
final String properIP = Seed.isProperIP(myIP); // with the IPv6 extension, this may contain several ips, separated by comma ','
if ( properIP == null ) mySeed.setIP(myIP); HashSet<String> h = new HashSet<>();
for (String s: myIP.split(",")) {
if (s.length() > 0 && Seed.isProperIP(s)) h.add(s);
}
if (h.size() > 0) mySeed.setIPs(h);
} }
mySeed.setFlagRootNode( mySeed.setFlagRootNode(
(mytype.equals(Seed.PEERTYPE_SENIOR) || mytype.equals(Seed.PEERTYPE_PRINCIPAL)) && (mytype.equals(Seed.PEERTYPE_SENIOR) || mytype.equals(Seed.PEERTYPE_PRINCIPAL)) &&
@ -303,7 +285,7 @@ public final class Protocol {
accessible.IWasAccessed = false; accessible.IWasAccessed = false;
} }
accessible.lastUpdated = System.currentTimeMillis(); accessible.lastUpdated = System.currentTimeMillis();
Network.amIAccessibleDB.put(otherHash, accessible); Network.amIAccessibleDB.put(otherSeed.hash, accessible);
/* /*
* If we were reported as junior we have to check if your port forwarding channel is broken * If we were reported as junior we have to check if your port forwarding channel is broken
@ -387,52 +369,11 @@ public final class Protocol {
final int connectedAfter = peerActions.sizeConnected(); final int connectedAfter = peerActions.sizeConnected();
// update event tracker // update event tracker
EventTracker.update(EventTracker.EClass.PEERPING, new ProfilingGraph.EventPing( EventTracker.update(EventTracker.EClass.PEERPING, new ProfilingGraph.EventPing(mySeed.getName(), otherSeed.getName(), true, connectedAfter - connectedBefore), false);
mySeed.getName(),
otherName,
true,
connectedAfter - connectedBefore), false);
return count; return count;
} }
/*
private int readSeeds(String prefix) {
String seedStr;
while ( (seedStr = result.get("seed" + i++)) != null ) {
// integrate new seed into own database
// the first seed, "seed0" is the seed of the responding peer
if ( seedStr.length() > Seed.maxsize ) {
Network.log.logInfo("hello/client: rejected contacting seed; too large ("
+ seedStr.length()
+ " > "
+ Seed.maxsize
+ ")");
} else {
try {
if ( i == 1 ) {
final int p = address.indexOf(':');
if ( p < 0 ) {
return -1;
}
InetAddress ia = Domains.dnsResolve(address.substring(0, p));
if (ia == null) continue;
final String host = ia.getHostAddress();
s = Seed.genRemoteSeed(seedStr, false, host);
} else {
s = Seed.genRemoteSeed(seedStr, false, null);
}
if ( peerActions.peerArrival(s, (i == 1)) ) {
count++;
}
} catch (final IOException e ) {
Network.log.logInfo("hello/client: rejected contacting seed; bad ("
+ e.getMessage()
+ ")");
}
}
}
}
*/
public static Seed querySeed(final Seed target, final String seedHash) { public static Seed querySeed(final Seed target, final String seedHash) {
// prepare request // prepare request
final String salt = crypt.randomSalt(); final String salt = crypt.randomSalt();
@ -443,7 +384,7 @@ public final class Protocol {
basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt); basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt);
parts.put("object", UTF8.StringBody("seed")); parts.put("object", UTF8.StringBody("seed"));
parts.put("env", UTF8.StringBody(seedHash)); parts.put("env", UTF8.StringBody(seedHash));
final byte[] content = postToFile(target, "query.html", parts, 10000); final byte[] content = postToFile(target.getPublicAddress(target.getIP()), target.hash, "query.html", parts, 10000);
final Map<String, String> result = FileUtils.table(content); final Map<String, String> result = FileUtils.table(content);
if ( result == null || result.isEmpty() ) { if ( result == null || result.isEmpty() ) {
@ -457,22 +398,24 @@ public final class Protocol {
} }
} }
public static long[] queryRWICount(final Seed target, final String wordHash) { public static long[] queryRWICount(final String targetAddress, final String targetHash, int timeout) {
if (target == null) return new long[] {-1, -1};
// prepare request // prepare request
final String salt = crypt.randomSalt(); final String salt = crypt.randomSalt();
// send request // send request
try { try {
final Map<String, ContentBody> parts = basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt); final Map<String, ContentBody> parts = basicRequestParts(Switchboard.getSwitchboard(), targetHash, salt);
parts.put("object", UTF8.StringBody("rwicount")); parts.put("object", UTF8.StringBody("rwicount"));
parts.put("ttl", UTF8.StringBody("0")); parts.put("ttl", UTF8.StringBody("0"));
parts.put("env", UTF8.StringBody(wordHash)); parts.put("env", UTF8.StringBody(""));
final byte[] content = postToFile(target, "query.html", parts, 6000); ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "posting request to " + targetAddress);
final byte[] content = postToFile(targetAddress, targetHash, "query.html", parts, timeout);
ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received CONTENT from requesting " + targetAddress + (content == null ? "NULL" : (": length = " + content.length)));
final Map<String, String> result = FileUtils.table(content); final Map<String, String> result = FileUtils.table(content);
if (result == null || result.isEmpty()) return new long[] {-1, -1}; if (result == null || result.isEmpty()) return new long[] {-1, -1};
ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESULT from requesting " + targetAddress + " : result = " + result.toString());
final String resp = result.get("response"); final String resp = result.get("response");
ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESPONSE from requesting " + targetAddress + " : response = " + resp);
if (resp == null) return new long[] {-1, -1}; if (resp == null) return new long[] {-1, -1};
String magic = result.get("magic"); String magic = result.get("magic");
if (magic == null) magic = "0"; if (magic == null) magic = "0";
@ -482,6 +425,7 @@ public final class Protocol {
return new long[] {-1, -1}; return new long[] {-1, -1};
} }
} catch (final Exception e ) { } catch (final Exception e ) {
ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received EXCEPTION from requesting " + targetAddress + ": " + e.getMessage());
if (Network.log.isFine()) Network.log.fine("yacyClient.queryRWICount error:" + e.getMessage()); if (Network.log.isFine()) Network.log.fine("yacyClient.queryRWICount error:" + e.getMessage());
return new long[] {-1, -1}; return new long[] {-1, -1};
} }
@ -516,10 +460,7 @@ public final class Protocol {
parts.put("time", UTF8.StringBody(Long.toString(maxTime))); parts.put("time", UTF8.StringBody(Long.toString(maxTime)));
// final byte[] result = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/urls.xml"), (int) maxTime, target.getHexHash() + ".yacyh", parts); // final byte[] result = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/urls.xml"), (int) maxTime, target.getHexHash() + ".yacyh", parts);
final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, (int) maxTime); final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, (int) maxTime);
final byte[] result = final byte[] result = httpClient.POSTbytes(new MultiProtocolURL("http://" + target.getPublicAddress(target.getIP()) + "/yacy/urls.xml"), target.getHexHash() + ".yacyh", parts, false, true);
httpClient.POSTbytes(new MultiProtocolURL("http://"
+ target.getClusterAddress()
+ "/yacy/urls.xml"), target.getHexHash() + ".yacyh", parts, false, true);
final RSSReader reader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result); final RSSReader reader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result);
if ( reader == null ) { if ( reader == null ) {
Network.log.warn("yacyClient.queryRemoteCrawlURLs failed asking peer '" Network.log.warn("yacyClient.queryRemoteCrawlURLs failed asking peer '"
@ -586,8 +527,8 @@ public final class Protocol {
final long timestamp = System.currentTimeMillis(); final long timestamp = System.currentTimeMillis();
event.addExpectedRemoteReferences(count); event.addExpectedRemoteReferences(count);
SearchResult result; SearchResult result;
String clusteraddress = target.getClusterAddress(); String clusteraddress = target.getPublicAddress(target.getIP());
if (clusteraddress.equals(event.peers.mySeed().getClusterAddress())) clusteraddress = "localhost:" + event.peers.mySeed().getPort(); if (target.clash(event.peers.mySeed().getIPs())) clusteraddress = "localhost:" + event.peers.mySeed().getPort();
try { try {
result = result =
new SearchResult( new SearchResult(
@ -680,7 +621,7 @@ public final class Protocol {
maxDistance, maxDistance,
partitions, partitions,
target.getHexHash() + ".yacyh", target.getHexHash() + ".yacyh",
target.getClusterAddress(), target.getPublicAddress(),
null null
); );
} catch (final IOException e ) { } catch (final IOException e ) {
@ -1033,7 +974,7 @@ public final class Protocol {
Network.log.info("SEARCH skip (solr), remote Solr interface not accessible, peer=" + target.getName()); Network.log.info("SEARCH skip (solr), remote Solr interface not accessible, peer=" + target.getName());
return -1; return -1;
} }
final String address = myseed ? "localhost:" + target.getPort() : target.getPublicAddress(); final String address = myseed ? "localhost:" + target.getPort() : target.getPublicAddress(target.getIP());
final int solrtimeout = Switchboard.getSwitchboard().getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 6000); final int solrtimeout = Switchboard.getSwitchboard().getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 6000);
Thread remoteRequest = new Thread() { Thread remoteRequest = new Thread() {
@Override @Override
@ -1285,7 +1226,7 @@ public final class Protocol {
final String salt = crypt.randomSalt(); final String salt = crypt.randomSalt();
// determining target address // determining target address
final String address = target.getClusterAddress(); final String address = target.getPublicAddress(target.getIP());
if ( address == null ) { if ( address == null ) {
return null; return null;
} }
@ -1435,7 +1376,7 @@ public final class Protocol {
final ReferenceContainerCache<WordReference> indexes, final ReferenceContainerCache<WordReference> indexes,
boolean gzipBody, boolean gzipBody,
final int timeout) { final int timeout) {
final String address = targetSeed.getPublicAddress(); final String address = targetSeed.getPublicAddress(targetSeed.getIP());
if ( address == null ) { if ( address == null ) {
Network.log.warn("no address for transferRWI"); Network.log.warn("no address for transferRWI");
return null; return null;
@ -1507,7 +1448,7 @@ public final class Protocol {
boolean gzipBody, boolean gzipBody,
final int timeout) { final int timeout) {
// this post a message to the remote message board // this post a message to the remote message board
final String address = targetSeed.getPublicAddress(); final String address = targetSeed.getPublicAddress(targetSeed.getIP());
if ( address == null ) { if ( address == null ) {
return null; return null;
} }
@ -1577,7 +1518,7 @@ public final class Protocol {
// this post a message to the remote message board // this post a message to the remote message board
final String salt = crypt.randomSalt(); final String salt = crypt.randomSalt();
String address = targetSeed.getClusterAddress(); String address = targetSeed.getPublicAddress(targetSeed.getIP());
if ( address == null ) { if ( address == null ) {
address = "localhost:8090"; address = "localhost:8090";
} }
@ -1619,7 +1560,7 @@ public final class Protocol {
final Map<String, ContentBody> parts = final Map<String, ContentBody> parts =
basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt); basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt);
parts.put("object", UTF8.StringBody("host")); parts.put("object", UTF8.StringBody("host"));
final byte[] content = postToFile(target, "idx.json", parts, 30000); final byte[] content = postToFile(target.getPublicAddress(target.getIP()), target.hash, "idx.json", parts, 30000);
if ( content == null || content.length == 0 ) { if ( content == null || content.length == 0 ) {
Network.log.warn("yacyClient.loadIDXHosts error: empty result"); Network.log.warn("yacyClient.loadIDXHosts error: empty result");
return null; return null;
@ -1702,64 +1643,45 @@ public final class Protocol {
return false; return false;
} }
private static final LinkedHashMap<String, ContentBody> basicRequestParts( /**
final Switchboard sb, * put in all the essentials for routing and network authentication
final String targetHash, * @param sb
final String salt) { * @param targetHash
// put in all the essentials for routing and network authentication * @param salt
// generate a session key * @return
final LinkedHashMap<String, ContentBody> parts = */
basicRequestParts( private static final LinkedHashMap<String, ContentBody> basicRequestParts(final Switchboard sb, final String targetHash, final String salt) {
sb.peers.mySeed().hash,
targetHash,
Switchboard.getSwitchboard().getConfig(
SwitchboardConstants.NETWORK_NAME,
Seed.DFLT_NETWORK_UNIT));
parts.put("key", UTF8.StringBody(salt));
// authentication essentials
final String authenticationControl = sb.getConfig("network.unit.protocol.control", "uncontrolled");
final String authenticationMethod =
sb.getConfig("network.unit.protocol.request.authentication.method", "");
if ( (authenticationControl.equals("controlled")) && (authenticationMethod.length() > 0) ) {
if ( authenticationMethod.equals("salted-magic-sim") ) {
// generate an authentication essential using the salt, the iam-hash and the network magic
final String magic =
sb.getConfig("network.unit.protocol.request.authentication.essentials", "");
final String md5 = Digest.encodeMD5Hex(salt + sb.peers.mySeed().hash + magic);
parts.put("magicmd5", UTF8.StringBody(md5));
}
}
return parts;
}
private static final LinkedHashMap<String, ContentBody> basicRequestParts(
final String myHash,
final String targetHash,
final String networkName) {
// put in all the essentials for routing and network authentication
// generate a session key
final LinkedHashMap<String, ContentBody> parts = new LinkedHashMap<String, ContentBody>(); final LinkedHashMap<String, ContentBody> parts = new LinkedHashMap<String, ContentBody>();
// just standard identification essentials // just standard identification essentials
if ( myHash != null ) { if ( sb.peers.mySeed().hash != null ) {
parts.put("iam", UTF8.StringBody(myHash)); parts.put("iam", UTF8.StringBody(sb.peers.mySeed().hash));
if ( targetHash != null ) { if ( targetHash != null ) parts.put("youare", UTF8.StringBody(targetHash));
parts.put("youare", UTF8.StringBody(targetHash));
}
// time information for synchronization // time information for synchronization
// use our own formatter to prevent concurrency locks with other processes // use our own formatter to prevent concurrency locks with other processes
final GenericFormatter my_SHORT_SECOND_FORMATTER = final GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second);
new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second);
parts.put("mytime", UTF8.StringBody(my_SHORT_SECOND_FORMATTER.format())); parts.put("mytime", UTF8.StringBody(my_SHORT_SECOND_FORMATTER.format()));
parts.put("myUTC", UTF8.StringBody(Long.toString(System.currentTimeMillis()))); parts.put("myUTC", UTF8.StringBody(Long.toString(System.currentTimeMillis())));
// network identification // network identification
parts.put(SwitchboardConstants.NETWORK_NAME, UTF8.StringBody(networkName)); parts.put(SwitchboardConstants.NETWORK_NAME, UTF8.StringBody(Switchboard.getSwitchboard().getConfig(
SwitchboardConstants.NETWORK_NAME,
Seed.DFLT_NETWORK_UNIT)));
} }
parts.put("key", UTF8.StringBody(salt));
// authentication essentials
final String authenticationControl = sb.getConfig("network.unit.protocol.control", "uncontrolled");
final String authenticationMethod = sb.getConfig("network.unit.protocol.request.authentication.method", "");
if ((authenticationControl.equals("controlled")) && (authenticationMethod.length() > 0) ) {
if (authenticationMethod.equals("salted-magic-sim") ) {
// generate an authentication essential using the salt, the iam-hash and the network magic
final String magic = sb.getConfig("network.unit.protocol.request.authentication.essentials", "");
final String md5 = Digest.encodeMD5Hex(salt + sb.peers.mySeed().hash + magic);
parts.put("magicmd5", UTF8.StringBody(md5));
}
}
return parts; return parts;
} }

@ -30,7 +30,7 @@ import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.Random; import java.util.Random;
import java.util.Set; import java.util.Set;
import java.util.SortedMap; import java.util.SortedSet;
import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrQuery;
@ -144,7 +144,7 @@ public class RemoteSearch extends Thread {
final int start, final int count, final int start, final int count,
final long time, final long time,
final Blacklist blacklist, final Blacklist blacklist,
final SortedMap<byte[], String> clusterselection) { final SortedSet<byte[]> clusterselection) {
// check own peer status // check own peer status
//if (wordIndex.seedDB.mySeed() == null || wordIndex.seedDB.mySeed().getPublicAddress() == null) { return null; } //if (wordIndex.seedDB.mySeed() == null || wordIndex.seedDB.mySeed().getPublicAddress() == null) { return null; }
@ -277,14 +277,13 @@ public class RemoteSearch extends Thread {
final Blacklist blacklist) { final Blacklist blacklist) {
// check own peer status // check own peer status
if (event.peers.mySeed() == null || event.peers.mySeed().getPublicAddress() == null) { return null; } if (event.peers.mySeed() == null || event.peers.mySeed().getIPs().size() == 0) { return null; }
assert urlhashes != null; assert urlhashes != null;
assert urlhashes.length() > 0; assert urlhashes.length() > 0;
// prepare seed targets and threads // prepare seed targets and threads
final Seed targetPeer = event.peers.getConnected(targethash); final Seed targetPeer = event.peers.getConnected(targethash);
if (targetPeer == null || targetPeer.hash == null) return null; if (targetPeer == null || targetPeer.hash == null) return null;
if (event.preselectedPeerHashes != null) targetPeer.setAlternativeAddress(event.preselectedPeerHashes.get(ASCII.getBytes(targetPeer.hash)));
Thread secondary = new Thread() { Thread secondary = new Thread() {
@Override @Override
public void run() { public void run() {
@ -332,12 +331,8 @@ public class RemoteSearch extends Thread {
assert solrQuery != null; assert solrQuery != null;
// check own peer status // check own peer status
if (event.peers.mySeed() == null || event.peers.mySeed().getPublicAddress() == null) { return null; } if (event.peers.mySeed() == null || event.peers.mySeed().getIPs().size() == 0) { return null; }
// prepare seed targets and threads // prepare threads
if (targetPeer != null && targetPeer.hash != null && event.preselectedPeerHashes != null) {
if (!targetPeer.getFlagSolrAvailable()) return null; // solr interface not avail.
targetPeer.setAlternativeAddress(event.preselectedPeerHashes.get(ASCII.getBytes(targetPeer.hash)));
}
Thread solr = new Thread() { Thread solr = new Thread() {
@Override @Override
public void run() { public void run() {

@ -44,13 +44,16 @@ import java.io.File;
import java.io.FileReader; import java.io.FileReader;
import java.io.FileWriter; import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.net.Inet6Address;
import java.net.InetAddress; import java.net.InetAddress;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.text.ParseException; import java.text.ParseException;
import java.util.Comparator; import java.util.Comparator;
import java.util.Date; import java.util.Date;
import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.Random; import java.util.Random;
import java.util.Set; import java.util.Set;
import java.util.TreeMap; import java.util.TreeMap;
@ -132,6 +135,7 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
private static final String IPTYPE = "IPType"; private static final String IPTYPE = "IPType";
private static final String FLAGS = "Flags"; private static final String FLAGS = "Flags";
public static final String FLAGSZERO = " "; public static final String FLAGSZERO = " ";
/** the applications version */ /** the applications version */
public static final String VERSION = "Version"; public static final String VERSION = "Version";
@ -142,31 +146,52 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
/** the name of the peer (user-set) */ /** the name of the peer (user-set) */
public static final String NAME = "Name"; public static final String NAME = "Name";
public static final String HASH = "Hash"; public static final String HASH = "Hash";
/** Birthday - first startup */ /** Birthday - first startup */
private static final String BDATE = "BDate"; private static final String BDATE = "BDate";
/** UTC-Offset */ /** UTC-Offset */
public static final String UTC = "UTC"; public static final String UTC = "UTC";
private static final String PEERTAGS = "Tags"; private static final String PEERTAGS = "Tags";
/** the speed of indexing (pages/minute) of the peer */ /** the speed of indexing (pages/minute) of the peer */
public static final String ISPEED = "ISpeed"; public static final String ISPEED = "ISpeed";
/** the speed of retrieval (queries/minute) of the peer */ /** the speed of retrieval (queries/minute) of the peer */
public static final String RSPEED = "RSpeed"; public static final String RSPEED = "RSpeed";
/** the number of minutes that the peer is up in minutes/day (moving average MA30) */ /** the number of minutes that the peer is up in minutes/day (moving average MA30) */
public static final String UPTIME = "Uptime"; public static final String UPTIME = "Uptime";
/** the number of links that the peer has stored (LURL's) */ /** the number of links that the peer has stored (LURL's) */
public static final String LCOUNT = "LCount"; public static final String LCOUNT = "LCount";
/** the number of links that the peer has noticed, but not loaded (NURL's) */ /** the number of links that the peer has noticed, but not loaded (NURL's) */
public static final String NCOUNT = "NCount"; public static final String NCOUNT = "NCount";
/** the number of links that the peer provides for remote crawls (ZURL's) */ /** the number of links that the peer provides for remote crawls (ZURL's) */
public static final String RCOUNT = "RCount"; public static final String RCOUNT = "RCount";
/** the number of different words the peer has indexed */ /** the number of different words the peer has indexed */
public static final String ICOUNT = "ICount"; public static final String ICOUNT = "ICount";
/** the number of seeds that the peer has stored */ /** the number of seeds that the peer has stored */
public static final String SCOUNT = "SCount"; public static final String SCOUNT = "SCount";
/** the number of clients that the peer connects (connects/hour as double) */ /** the number of clients that the peer connects (connects/hour as double) */
public static final String CCOUNT = "CCount"; public static final String CCOUNT = "CCount";
/** the public IP of this peer (old field, will be used to carry the IPv4) */
public static final String IP = "IP"; public static final String IP = "IP";
/** more public IPs of this peer, containing only IPv6 addresses. This list of of IPv6 addresses is separated with a vertical bar/pipe '|'.
* This list may have zero entries if the host does not have a IPv6 address. It may have more than one IPv6 address if the
* Host detects more than one public IPv6 locally but did not get a feedback from other peers if any of these addresses are
* reachable. The list may contain only one address, if a 'hello' with backping to another peer was successful and the other peer
* peer confirmed one of the IPv6 IPs which is then the only entry in this field.
*/
public static final String IP6 = "IP6";
public static final String PORT = "Port"; public static final String PORT = "Port";
public static final String SEEDLISTURL = "seedURL"; public static final String SEEDLISTURL = "seedURL";
public static final String NEWS = "news"; // news attachment public static final String NEWS = "news"; // news attachment
@ -192,7 +217,6 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
public final String hash; public final String hash;
/** a set of identity founding values, eg. IP, name of the peer, YaCy-version, ... */ /** a set of identity founding values, eg. IP, name of the peer, YaCy-version, ... */
private final ConcurrentMap<String, String> dna; private final ConcurrentMap<String, String> dna;
private String alternativeIP = null;
private long birthdate; // keep this value in ram since it is often used and may cause lockings in concurrent situations. private long birthdate; // keep this value in ram since it is often used and may cause lockings in concurrent situations.
public Seed(final String theHash, final ConcurrentMap<String, String> theDna) { public Seed(final String theHash, final ConcurrentMap<String, String> theDna) {
@ -297,34 +321,85 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
} }
/** /**
* used when doing routing within a cluster; this can assign a ip and a port that is used instead the * try to get the public IP<br>
* address stored in the seed DNA *
* @return the IP or localhost IP (127.0.0.1)
*/ */
public void setAlternativeAddress(final String ipport) { @Deprecated
if ( ipport == null ) { public final String getIP() {
return; final String ipx = this.dna.get(Seed.IP); // may contain both, IPv4 or IPv6
final String ip6 = this.dna.get(Seed.IP6);
Set<String> ip6s = MapTools.string2set(ip6, "|");
if (ip6s == null || ip6s.size() == 0) {
if (ipx != null && !ipx.isEmpty()) return chopZoneID(ipx);
} }
final int p = ipport.lastIndexOf(':'); if (ip6s != null && ip6s.size() == 1) {
if ( p < 0 ) { // We prefer IPv6
this.alternativeIP = ipport; for (String s: ip6s) if (s.length() > 0) return chopZoneID(s);
} else { if (ipx != null && !ipx.isEmpty()) return chopZoneID(ipx);
this.alternativeIP = ipport.substring(0, p);
} }
if (this.alternativeIP.charAt(0) == '[' && this.alternativeIP.charAt(this.alternativeIP.length() - 1) == ']') { // if we have more than one IPv6, then chances are high that one of them do not work.
// IPv6 patch // in that case we prefer the IPv4
this.alternativeIP = this.alternativeIP.substring(1, this.alternativeIP.length() - 1); if (ipx != null && !ipx.isEmpty()) return chopZoneID(ipx);
} if (ip6s != null) for (String s: ip6s) if (s.length() > 0) return chopZoneID(s);
// in case that we don't have any address using the dna (i.e. a fresh peer), then use all locally known addresses
for (InetAddress i: Domains.myPublicIPv4()) return chopZoneID(i.getHostAddress());
for (InetAddress i: Domains.myPublicIPv6()) return chopZoneID(i.getHostAddress());
// final chance
return Domains.LOCALHOST;
} }
/** /**
* try to get the public IP<br> * Get all my public IPs. If there was a static IP assignment, only one, that IP is returned.
* * If no feedback from other peers exist, then all locally determined IPs are returned.
* @return the IP or localhost IP (127.0.0.1) * If a feedback from other peers exist, then return at most two IPs:
* the latest IPv4 and the latest IPv6 which was returned during a hello process from a remote peer
* @return a set of IPs which are supposed to be my own public IPs
*/ */
public final String getIP() { public final Set<String> getIPs() {
final String ip = this.dna.get(Seed.IP); Set<String> h = new LinkedHashSet<>();
return (ip == null || ip.isEmpty()) ? Domains.LOCALHOST : ip; // not public (but leave as is for now 2014-08-24) final String ipx = this.dna.get(Seed.IP); // may contain both, IPv4 or IPv6
final String ip6 = this.dna.get(Seed.IP6);
Set<String> ip6s = MapTools.string2set(ip6, "|");
if (ip6s == null || ip6s.size() == 0) {
if (ipx != null && !ipx.isEmpty()) h.add(chopZoneID(ipx));
} else if (ip6s != null && ip6s.size() == 1) {
// We add IPv6 first because then those addresses appear first
// in the LinkedHashSet and are preferred by methods using only the first one.
for (String s: ip6s) if (s.length() > 0) h.add(chopZoneID(s));
if (ipx != null && !ipx.isEmpty()) h.add(chopZoneID(ipx));
} else {
// if we have more than one IPv6, then chances are high that one of them do not work.
// in that case we prefer the IPv4
if (ipx != null && !ipx.isEmpty()) h.add(chopZoneID(ipx));
if (ip6s != null) for (String s: ip6s) if (s.length() > 0) h.add(chopZoneID(s));
}
// in case that we don't have any address using the dna (i.e. a fresh peer), then use all locally known addresses
if (h.size() == 0) {
for (InetAddress i: Domains.myPublicIPv4()) h.add(chopZoneID(i.getHostAddress()));
for (InetAddress i: Domains.myPublicIPv6()) h.add(chopZoneID(i.getHostAddress()));
h.add(Domains.LOCALHOST);
}
return h;
}
private String chopZoneID(String ip) {
int i = ip.indexOf('%');
return i < 0 ? ip : ip.substring(0, i);
}
public boolean clash(Set<String> ips) {
Set<String> myIPs = getIPs();
for (String s: ips) {
if (myIPs.contains(s) && isProperIP(s)) return true;
}
return false;
} }
/** /**
@ -424,8 +499,54 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
} }
} }
/**
* set the Peer ip.
* This sets the IP and IP6 field according to the current fill state of that fields:
* - if no field has a content, then IP is filled with the given ip, even if that ip is of type IPv6
* - if IP is already set then check if this is equivalent with the given ip. If both are equal, nothing is done.
* If they are not equal, the IP and IPv6 field is set according to the type if the given ip: if the given ip
* is of type IPv4, then IP is set with ip, otherwise IP6 is set with the ip.
* ATTENTION: if the given IP is IPv6, then after the call that IP is the only one assigned to the peer!
* @param ip
*/
public final void setIP(final String ip) { public final void setIP(final String ip) {
if (!isProperIP(ip)) return;
String oldIP = this.dna.get(Seed.IP);
String oldIP6 = this.dna.get(Seed.IP6);
if ((oldIP == null || oldIP.length() == 0) && (oldIP6 == null || oldIP6.length() == 0)) {
this.dna.put(Seed.IP, ip); this.dna.put(Seed.IP, ip);
} else {
if (oldIP == null || !oldIP.equals(ip)) {
if (oldIP == null || oldIP.length() == 0 || ip.indexOf(':') == 0) this.dna.put(Seed.IP, ip); else this.dna.put(Seed.IP6, ip);
}
}
}
/**
* Set several local IPs which are good to access this peer.
* This OVERWRITES ALL ips stored before!
* @param ips list of IPs
*/
public final void setIPs(final Set<String> ips) {
// we must sort IPv4 and IPv6 here
Set<String> ipv6 = new HashSet<>();
Set<String> ipv4 = new HashSet<>();
for (String ip: ips) if (isProperIP(ip)) {
if (ip.indexOf(':') >= 0) ipv6.add(ip); else ipv4.add(ip);
}
if (ipv4.size() == 0) {
if (ipv6.size() == 1) {
// if the only IP we have is IPv6, then put this into IP field
this.dna.put(Seed.IP, ipv6.iterator().next());
this.dna.put(Seed.IP6, "");
} else if (ipv6.size() > 1) {
this.dna.put(Seed.IP, "");
this.dna.put(Seed.IP6, MapTools.set2string(ipv6, "|", false));
}
} else {
this.dna.put(Seed.IP, ipv4.iterator().next());
this.dna.put(Seed.IP6, MapTools.set2string(ipv6, "|", false));
}
} }
public final void setPort(final String port) { public final void setPort(final String port) {
@ -575,78 +696,65 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
} }
/** /**
* deprecated, use getIPs() instead
* @return the public address of the peer as IP:port string or <code>null</code> if no valid values for * @return the public address of the peer as IP:port string or <code>null</code> if no valid values for
* either the IP or the port could be retrieved from this yacySeed object * either the IP or the port could be retrieved from this yacySeed object
*/ */
@Deprecated
public final String getPublicAddress() { public final String getPublicAddress() {
String ip = getIP(); return getPublicAddress(getIP());
if (ip == null) ip = Domains.LOCALHOST; // that should not happen }
int p = ip.lastIndexOf(':');
if (p > 0 && (ip.indexOf(':') == p || "]:".equals(ip.substring(p - 1, p + 1)))) return ip; // includes already the port
final String port = this.dna.get(Seed.PORT); /**
* generate a public address using a given ip. This combines the ip with the port and encloses the ip
* with square brackets if the ip is of typeIPv6
* @param ip
* @return an address string which can be used as host:port part of an url
*/
public final String getPublicAddress(final InetAddress ip) {
// we do not use getPublicAddress(String ip) here to be able to check IPv6 with instanceof Inet6Address which is faster than indexOf(':')
if (ip == null) throw new RuntimeException("ip == NULL"); // that should not happen
final String port = this.dna.get(Seed.PORT); // we do not use getPort() here to avoid String->Integer->toString() conversion
if ( port == null || port.length() < 2 || port.length() > 5 ) { if ( port == null || port.length() < 2 || port.length() > 5 ) {
return null; throw new RuntimeException("port not wellformed: " + port); // that should not happen
} }
final StringBuilder sb = new StringBuilder();
final StringBuilder sb = new StringBuilder(ip.length() + port.length() + 3); if (ip instanceof Inet6Address) {
if (ip.indexOf(':') >= 0) { sb.append('[').append(ip.getHostAddress()).append(']');
// IPv6 Address!, see: http://en.wikipedia.org/wiki/IPv6_address#Literal_IPv6_addresses_in_network_resource_identifiers
if (!ip.startsWith("[")) sb.append('[');
sb.append(ip);
if (!ip.endsWith("]")) sb.append(']');
sb.append(':');
sb.append(port);
} else { } else {
sb.append(ip); sb.append(ip.getHostAddress());
}
sb.append(':'); sb.append(':');
sb.append(port); sb.append(port);
}
return sb.toString(); return sb.toString();
} }
/** /**
* If this seed is part of a cluster, the peer has probably the {@linkplain #alternativeIP} object set to * generate a public address using a given ip. This combines the ip with the port and encloses the ip
* a local IP. If this is present and the public IP of this peer is identical to the public IP of the own * with square brackets if the ip is of typeIPv6
* seed, construct an address using this IP; otherwise return the public address * @param ip
* * @return an address string which can be used as host:port part of an url
* @see #getPublicAddress()
* @return the alternative IP:port if present, else the public address
*/ */
public final String getClusterAddress() { public final String getPublicAddress(final String ip) {
if ( this.alternativeIP == null ) { if (ip == null) throw new RuntimeException("ip == NULL"); // that should not happen
return getPublicAddress(); final String port = this.dna.get(Seed.PORT); // we do not use getPort() here to avoid String->Integer->toString() conversion
}
final String port = this.dna.get(Seed.PORT);
if ( port == null || port.length() < 2 || port.length() > 5 ) { if ( port == null || port.length() < 2 || port.length() > 5 ) {
return null; throw new RuntimeException("port not wellformed: " + port); // that should not happen
} }
final StringBuilder sb = new StringBuilder(ip.length() + port.length() + 3);
final StringBuilder sb = new StringBuilder(this.alternativeIP.length() + port.length() + 3); if (ip.indexOf(':') >= 0) {
if (this.alternativeIP.indexOf(':') >= 0) { if (!ip.startsWith("[")) sb.append('[');
// IPv6 Address!, see: http://en.wikipedia.org/wiki/IPv6_address#Literal_IPv6_addresses_in_network_resource_identifiers sb.append(ip);
sb.append('['); if (!ip.endsWith("]")) sb.append(']');
sb.append(this.alternativeIP);
sb.append(']');
sb.append(':');
sb.append(port);
} else { } else {
sb.append(this.alternativeIP); sb.append(ip);
}
sb.append(':'); sb.append(':');
sb.append(port); sb.append(port);
}
return sb.toString(); return sb.toString();
} }
/**
* @return the IP address of the peer represented by this yacySeed object as {@link InetAddress}
*/
public final InetAddress getInetAddress() {
return Domains.dnsResolve(getIP());
}
/** @return the port number of this seed or <code>-1</code> if not present */ /** @return the port number of this seed or <code>-1</code> if not present */
public final int getPort() { public final int getPort() {
final String port = this.dna.get(Seed.PORT); final String port = this.dna.get(Seed.PORT);
@ -662,12 +770,8 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
// because java thinks it must apply the UTC offset to the current time, // because java thinks it must apply the UTC offset to the current time,
// to create a string that looks like our current time, it adds the local UTC offset to the // to create a string that looks like our current time, it adds the local UTC offset to the
// time. To create a corrected UTC Date string, we first subtract the local UTC offset. // time. To create a corrected UTC Date string, we first subtract the local UTC offset.
final GenericFormatter my_SHORT_SECOND_FORMATTER = final GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); // use our own formatter to prevent concurrency locks with other processes
new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); // use our own formatter to prevent concurrency locks with other processes final String ls = my_SHORT_SECOND_FORMATTER.format(new Date(System.currentTimeMillis() /*- DateFormatter.UTCDiff()*/));
final String ls =
my_SHORT_SECOND_FORMATTER
.format(new Date(System.currentTimeMillis() /*- DateFormatter.UTCDiff()*/));
//System.out.println("SETTING LAST-SEEN of " + this.getName() + " to " + ls);
this.dna.put(Seed.LASTSEEN, ls); this.dna.put(Seed.LASTSEEN, ls);
} }
@ -1101,10 +1205,8 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
// check IP // check IP
if ( !checkOwnIP ) { if ( !checkOwnIP ) {
// checking of IP is omitted if we read the own seed file // checking of IP is omitted if we read the own seed file
final String ipCheck = isProperIP(getIP()); final String ip = getIP();
if ( ipCheck != null ) { if (!isProperIP(ip)) return "not a proper IP " + ip;
return ipCheck;
}
} }
// seedURL // seedURL
@ -1134,23 +1236,19 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
return null; return null;
} }
public static final String isProperIP(final String ipString) { /**
// returns null if ipString is proper, a string with the cause otherwise * check if the given string containing an IP is proper. This checks also if the IP is within the given
if ( ipString == null ) { * range of the network definition
return ipString + " -> IP is null"; * @param ipString
} * @return true iff the IP is proper
if ( ipString.length() < 8 ) { */
return ipString + " -> IP is too short: "; public static final boolean isProperIP(final String ipString) {
} if (ipString == null) return false;
if ( Switchboard.getSwitchboard().isAllIPMode() ) { if (ipString.length() < 3) return false;
return null; if (Switchboard.getSwitchboard().isAllIPMode()) return true; // accept everyting
}
final boolean islocal = Domains.isLocal(ipString, null); final boolean islocal = Domains.isLocal(ipString, null);
//if (islocal && Switchboard.getSwitchboard().isGlobalMode()) return ipString + " - local IP for global mode rejected"; //if (islocal && Switchboard.getSwitchboard().isGlobalMode()) return ipString + " - local IP for global mode rejected";
if ( !islocal && Switchboard.getSwitchboard().isIntranetMode() ) { return islocal == Switchboard.getSwitchboard().isIntranetMode();
return ipString + " - global IP for intranet mode rejected";
}
return null;
} }
@Override @Override

@ -35,7 +35,8 @@ import java.util.Collection;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.TreeMap; import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
@ -208,11 +209,7 @@ public final class SeedDB implements AlternativeDomainNames {
System.exit(-1); System.exit(-1);
} }
} }
if (Switchboard.getSwitchboard().isIntranetMode()) { this.mySeed.setIPs(Switchboard.getSwitchboard().myPublicIPs());
this.mySeed.setIP(Domains.myPublicLocalIP().getHostAddress()); // in intranet mode host address is best choice (to become senior peer)
} else {
this.mySeed.setIP("");// we delete the old information to see what we have now
}
this.mySeed.put(Seed.PEERTYPE, Seed.PEERTYPE_VIRGIN); // markup startup condition this.mySeed.put(Seed.PEERTYPE, Seed.PEERTYPE_VIRGIN); // markup startup condition
} }
@ -229,10 +226,6 @@ public final class SeedDB implements AlternativeDomainNames {
if (this.mySeed == null) { if (this.mySeed == null) {
if (sizeConnected() == 0) try {Thread.sleep(5000);} catch (final InterruptedException e) {} // wait for init if (sizeConnected() == 0) try {Thread.sleep(5000);} catch (final InterruptedException e) {} // wait for init
initMySeed(); initMySeed();
// check if my seed has an IP assigned
if (myIP() == null || myIP().isEmpty()) {
this.mySeed.setIP(Domains.myPublicLocalIP().getHostAddress());
}
} }
return this.mySeed; return this.mySeed;
} }
@ -246,11 +239,16 @@ public final class SeedDB implements AlternativeDomainNames {
return mySeed().getName() + ".yacy"; return mySeed().getName() + ".yacy";
} }
@Override @Deprecated
public String myIP() { public String myIP() {
return mySeed().getIP(); return mySeed().getIP();
} }
@Override
public Set<String> myIPs() {
return mySeed().getIPs();
}
@Override @Override
public int myPort() { public int myPort() {
return mySeed().getPort(); return mySeed().getPort();
@ -349,7 +347,7 @@ public final class SeedDB implements AlternativeDomainNames {
return new seedEnum(up, field, this.seedPotentialDB); return new seedEnum(up, field, this.seedPotentialDB);
} }
public TreeMap<byte[], String> /* peer-b64-hashes/ipport */ clusterHashes(final String clusterdefinition) { public TreeSet<byte[]> /* peer-b64-hashes/ipport */ clusterHashes(final String clusterdefinition) {
// collects seeds according to cluster definition string, which consists of // collects seeds according to cluster definition string, which consists of
// comma-separated .yacy or .yacyh-domains // comma-separated .yacy or .yacyh-domains
// the domain may be extended by an alternative address specification of the form // the domain may be extended by an alternative address specification of the form
@ -359,18 +357,16 @@ public final class SeedDB implements AlternativeDomainNames {
// address ::= (<peername>'.yacy'|<peerhexhash>'.yacyh'){'='<ip>{':'<port}} // address ::= (<peername>'.yacy'|<peerhexhash>'.yacyh'){'='<ip>{':'<port}}
// clusterdef ::= {address}{','address}* // clusterdef ::= {address}{','address}*
final String[] addresses = (clusterdefinition.isEmpty()) ? new String[0] : clusterdefinition.split(","); final String[] addresses = (clusterdefinition.isEmpty()) ? new String[0] : clusterdefinition.split(",");
final TreeMap<byte[], String> clustermap = new TreeMap<byte[], String>(Base64Order.enhancedCoder); final TreeSet<byte[]> clustermap = new TreeSet<>(Base64Order.enhancedCoder);
Seed seed; Seed seed;
String hash, yacydom, ipport; String hash, yacydom;
int p; int p;
for (final String addresse : addresses) { for (final String addresse : addresses) {
p = addresse.indexOf('='); p = addresse.indexOf('=');
if (p >= 0) { if (p >= 0) {
yacydom = addresse.substring(0, p); yacydom = addresse.substring(0, p);
ipport = addresse.substring(p + 1);
} else { } else {
yacydom = addresse; yacydom = addresse;
ipport = null;
} }
if (yacydom.endsWith(".yacyh")) { if (yacydom.endsWith(".yacyh")) {
// find a peer with its hexhash // find a peer with its hexhash
@ -379,7 +375,7 @@ public final class SeedDB implements AlternativeDomainNames {
if (seed == null) { if (seed == null) {
Network.log.warn("cluster peer '" + yacydom + "' was not found."); Network.log.warn("cluster peer '" + yacydom + "' was not found.");
} else { } else {
clustermap.put(ASCII.getBytes(hash), ipport); clustermap.add(ASCII.getBytes(hash));
} }
} else if (yacydom.endsWith(".yacy")) { } else if (yacydom.endsWith(".yacy")) {
// find a peer with its name // find a peer with its name
@ -387,7 +383,7 @@ public final class SeedDB implements AlternativeDomainNames {
if (seed == null) { if (seed == null) {
Network.log.warn("cluster peer '" + yacydom + "' was not found."); Network.log.warn("cluster peer '" + yacydom + "' was not found.");
} else { } else {
clustermap.put(ASCII.getBytes(seed.hash), ipport); clustermap.add(ASCII.getBytes(seed.hash));
} }
} else { } else {
Network.log.warn("cluster peer '" + addresse + "' has wrong syntax. the name must end with .yacy or .yacyh"); Network.log.warn("cluster peer '" + addresse + "' has wrong syntax. the name must end with .yacy or .yacyh");
@ -732,9 +728,7 @@ public final class SeedDB implements AlternativeDomainNames {
} }
// check local seed // check local seed
if (this.mySeed == null) return null; if (this.mySeed == null || !this.mySeed.getIPs().contains(ipString)) return null;
String s = this.mySeed.getIP();
if (s == null || !ipString.equals(s)) return null;
int p = this.mySeed.getPort(); int p = this.mySeed.getPort();
if (port > 0 && p != port) return null; if (port > 0 && p != port) return null;
//System.out.println("*** found lookupByIP as my seed: " + peerIP.toString() + " -> " + this.mySeed.getName()); //System.out.println("*** found lookupByIP as my seed: " + peerIP.toString() + " -> " + this.mySeed.getName());
@ -915,7 +909,7 @@ public final class SeedDB implements AlternativeDomainNames {
seed = this.mySeed; seed = this.mySeed;
else return null; else return null;
} }
return seed.getPublicAddress() + ((subdom == null) ? "" : ("/" + subdom)); return seed.getPublicAddress(seed.getIP()) + ((subdom == null) ? "" : ("/" + subdom));
} else if (host.endsWith(".yacy")) { } else if (host.endsWith(".yacy")) {
// identify subdomain // identify subdomain
p = host.indexOf('.'); p = host.indexOf('.');
@ -932,7 +926,7 @@ public final class SeedDB implements AlternativeDomainNames {
// take local ip instead of external // take local ip instead of external
return Switchboard.getSwitchboard().myPublicIP() + ":" + Switchboard.getSwitchboard().getConfig("port", "8090") + ((subdom == null) ? "" : ("/" + subdom)); return Switchboard.getSwitchboard().myPublicIP() + ":" + Switchboard.getSwitchboard().getConfig("port", "8090") + ((subdom == null) ? "" : ("/" + subdom));
} }
return seed.getPublicAddress() + ((subdom == null) ? "" : ("/" + subdom)); return seed.getPublicAddress(seed.getIP()) + ((subdom == null) ? "" : ("/" + subdom));
} else { } else {
return null; return null;
} }
@ -942,11 +936,11 @@ public final class SeedDB implements AlternativeDomainNames {
// find target address // find target address
String address; String address;
if (targetHash.equals(mySeed().hash)) { if (targetHash.equals(mySeed().hash)) {
address = mySeed().getClusterAddress(); address = mySeed().getPublicAddress(mySeed().getIP());
} else { } else {
final Seed targetSeed = getConnected(targetHash); final Seed targetSeed = getConnected(targetHash);
if (targetSeed == null) { return null; } if (targetSeed == null) { return null; }
address = targetSeed.getClusterAddress(); address = targetSeed.getPublicAddress(targetSeed.getIP());
} }
if (address == null) address = "localhost" + (this.mySeed.getPort() > 0 ? ":" + this.mySeed.getPort() : ""); if (address == null) address = "localhost" + (this.mySeed.getPort() > 0 ? ":" + this.mySeed.getPort() : "");
return address; return address;

@ -268,8 +268,7 @@ public class Transmission {
// get possibly newer target Info // get possibly newer target Info
final Seed newTarget = Transmission.this.seeds.get(this.dhtTarget.hash); final Seed newTarget = Transmission.this.seeds.get(this.dhtTarget.hash);
if (newTarget != null) { if (newTarget != null) {
final String oldAddress = this.dhtTarget.getPublicAddress(); if (this.dhtTarget.clash(newTarget.getIPs())) {
if ((oldAddress != null) && (oldAddress.equals(newTarget.getPublicAddress()))) {
newTarget.setFlagAcceptRemoteIndex(false); newTarget.setFlagAcceptRemoteIndex(false);
Transmission.this.seeds.update(newTarget.hash, newTarget); Transmission.this.seeds.update(newTarget.hash, newTarget);
} else { } else {

@ -64,7 +64,6 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Properties; import java.util.Properties;
import java.util.Set; import java.util.Set;
import java.util.SortedMap;
import java.util.SortedSet; import java.util.SortedSet;
import java.util.TreeMap; import java.util.TreeMap;
import java.util.TreeSet; import java.util.TreeSet;
@ -265,7 +264,7 @@ public final class Switchboard extends serverSwitch {
public int searchQueriesRobinsonFromLocal = 0; // absolute counter of all local queries submitted on this peer from a local or autheticated used public int searchQueriesRobinsonFromLocal = 0; // absolute counter of all local queries submitted on this peer from a local or autheticated used
public int searchQueriesRobinsonFromRemote = 0; // absolute counter of all local queries submitted on this peer from a remote IP without authentication public int searchQueriesRobinsonFromRemote = 0; // absolute counter of all local queries submitted on this peer from a remote IP without authentication
public float searchQueriesGlobal = 0f; // partial counter of remote queries (1/number-of-requested-peers) public float searchQueriesGlobal = 0f; // partial counter of remote queries (1/number-of-requested-peers)
public SortedMap<byte[], String> clusterhashes; // map of peerhash(String)/alternative-local-address as ip:port or only ip (String) or null if address in seed should be used public SortedSet<byte[]> clusterhashes; // a set of cluster hashes
public List<Pattern> networkWhitelist, networkBlacklist; public List<Pattern> networkWhitelist, networkBlacklist;
public FilterEngine domainList; public FilterEngine domainList;
private Dispatcher dhtDispatcher; private Dispatcher dhtDispatcher;
@ -1592,7 +1591,7 @@ public final class Switchboard extends serverSwitch {
getConfig(SwitchboardConstants.CLUSTER_MODE, SwitchboardConstants.CLUSTER_MODE_PUBLIC_PEER); getConfig(SwitchboardConstants.CLUSTER_MODE, SwitchboardConstants.CLUSTER_MODE_PUBLIC_PEER);
if ( clustermode.equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER) ) { if ( clustermode.equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER) ) {
// check if we got the request from a peer in the public cluster // check if we got the request from a peer in the public cluster
return this.clusterhashes.containsKey(ASCII.getBytes(peer)); return this.clusterhashes.contains(ASCII.getBytes(peer));
} }
return false; return false;
} }
@ -1610,7 +1609,7 @@ public final class Switchboard extends serverSwitch {
getConfig(SwitchboardConstants.CLUSTER_MODE, SwitchboardConstants.CLUSTER_MODE_PUBLIC_PEER); getConfig(SwitchboardConstants.CLUSTER_MODE, SwitchboardConstants.CLUSTER_MODE_PUBLIC_PEER);
if ( clustermode.equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER) ) { if ( clustermode.equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER) ) {
// check if we got the request from a peer in the public cluster // check if we got the request from a peer in the public cluster
return this.clusterhashes.containsKey(ASCII.getBytes(seed.hash)); return this.clusterhashes.contains(ASCII.getBytes(seed.hash));
} }
return false; return false;
} }
@ -2888,9 +2887,6 @@ public final class Switchboard extends serverSwitch {
if ( (processCase == EventOrigin.GLOBAL_CRAWLING) && (queueEntry.initiator() != null) ) { if ( (processCase == EventOrigin.GLOBAL_CRAWLING) && (queueEntry.initiator() != null) ) {
final Seed initiatorPeer = this.peers.get(ASCII.String(queueEntry.initiator())); final Seed initiatorPeer = this.peers.get(ASCII.String(queueEntry.initiator()));
if ( initiatorPeer != null ) { if ( initiatorPeer != null ) {
if ( this.clusterhashes != null ) {
initiatorPeer.setAlternativeAddress(this.clusterhashes.get(queueEntry.initiator()));
}
// start a thread for receipt sending to avoid a blocking here // start a thread for receipt sending to avoid a blocking here
SolrDocument sd = this.index.fulltext().getDefaultConfiguration().toSolrDocument(newEntry); SolrDocument sd = this.index.fulltext().getDefaultConfiguration().toSolrDocument(newEntry);
new Thread(new receiptSending(initiatorPeer, new URIMetadataNode(sd)), "sending receipt to " + ASCII.String(queueEntry.initiator())).start(); new Thread(new receiptSending(initiatorPeer, new URIMetadataNode(sd)), "sending receipt to " + ASCII.String(queueEntry.initiator())).start();
@ -3712,6 +3708,11 @@ public final class Switchboard extends serverSwitch {
mySeed.setFlagAcceptRemoteCrawl(getConfigBool("crawlResponse", true)); mySeed.setFlagAcceptRemoteCrawl(getConfigBool("crawlResponse", true));
mySeed.setFlagAcceptRemoteIndex(getConfigBool("allowReceiveIndex", true)); mySeed.setFlagAcceptRemoteIndex(getConfigBool("allowReceiveIndex", true));
mySeed.setFlagSSLAvailable(this.getHttpServer() != null && this.getHttpServer().withSSL() && getConfigBool("server.https", false)); mySeed.setFlagSSLAvailable(this.getHttpServer() != null && this.getHttpServer().withSSL() && getConfigBool("server.https", false));
// set local ips
String staticIP = this.getConfig("staticIP", "");
if (staticIP.length() > 0) mySeed.setIP(staticIP);
mySeed.setIPs(Switchboard.getSwitchboard().myPublicIPs());
} }
public void loadSeedLists() { public void loadSeedLists() {

@ -37,6 +37,7 @@ import java.util.LinkedHashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.SortedMap; import java.util.SortedMap;
import java.util.SortedSet;
import java.util.TreeMap; import java.util.TreeMap;
import java.util.concurrent.BlockingQueue; import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
@ -131,7 +132,7 @@ public final class SearchEvent {
public final List<RemoteSearch> primarySearchThreadsL; public final List<RemoteSearch> primarySearchThreadsL;
public final List<Thread> nodeSearchThreads; public final List<Thread> nodeSearchThreads;
public Thread[] secondarySearchThreads; public Thread[] secondarySearchThreads;
public final SortedMap<byte[], String> preselectedPeerHashes; public final SortedSet<byte[]> preselectedPeerHashes;
private final SortedMap<byte[], Integer> IACount; private final SortedMap<byte[], Integer> IACount;
private final SortedMap<byte[], String> IAResults; private final SortedMap<byte[], String> IAResults;
private final SortedMap<byte[], HeuristicResult> heuristics; private final SortedMap<byte[], HeuristicResult> heuristics;
@ -198,7 +199,7 @@ public final class SearchEvent {
final QueryParams query, final QueryParams query,
final SeedDB peers, final SeedDB peers,
final WorkTables workTables, final WorkTables workTables,
final SortedMap<byte[], String> preselectedPeerHashes, final SortedSet<byte[]> preselectedPeerHashes,
final boolean generateAbstracts, final boolean generateAbstracts,
final LoaderDispatcher loader, final LoaderDispatcher loader,
final int remote_maxcount, final int remote_maxcount,

@ -29,7 +29,7 @@ package net.yacy.search.query;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.Map; import java.util.Map;
import java.util.SortedMap; import java.util.SortedSet;
import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.ConcurrentLog;
import net.yacy.data.WorkTables; import net.yacy.data.WorkTables;
@ -135,7 +135,7 @@ public class SearchEventCache {
final QueryParams query, final QueryParams query,
final SeedDB peers, final SeedDB peers,
final WorkTables workTables, final WorkTables workTables,
final SortedMap<byte[], String> preselectedPeerHashes, final SortedSet<byte[]> preselectedPeerHashes,
final boolean generateAbstracts, final boolean generateAbstracts,
final LoaderDispatcher loader, final LoaderDispatcher loader,
final int remote_maxcount, final int remote_maxcount,

@ -26,7 +26,6 @@ package net.yacy.search.snippet;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator; import java.util.Comparator;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
@ -48,7 +47,6 @@ import net.yacy.crawler.retrieval.Request;
import net.yacy.crawler.retrieval.Response; import net.yacy.crawler.retrieval.Response;
import net.yacy.document.Document; import net.yacy.document.Document;
import net.yacy.document.Parser; import net.yacy.document.Parser;
import net.yacy.document.SentenceReader;
import net.yacy.document.SnippetExtractor; import net.yacy.document.SnippetExtractor;
import net.yacy.document.WordTokenizer; import net.yacy.document.WordTokenizer;
import net.yacy.document.parser.html.CharacterCoding; import net.yacy.document.parser.html.CharacterCoding;

@ -24,6 +24,8 @@
package net.yacy.server.http; package net.yacy.server.http;
import java.util.Set;
public interface AlternativeDomainNames { public interface AlternativeDomainNames {
/** /**
@ -42,10 +44,10 @@ public interface AlternativeDomainNames {
public String myAlternativeAddress(); public String myAlternativeAddress();
/** /**
* return the IP as string of my server address * return a set of IPs of this server
* @return IP as string of this server * @return IP as set of strings of this server
*/ */
public String myIP(); public Set<String> myIPs();
/** /**
* return the port of my server address * return the port of my server address

@ -140,7 +140,9 @@ public class serverAccessTracker {
track.add(new Track(System.currentTimeMillis(), accessPath)); track.add(new Track(System.currentTimeMillis(), accessPath));
clearTooOldAccess(track); clearTooOldAccess(track);
} }
if (Domains.isLocalhost(host)) lastLocalhostAccess = System.currentTimeMillis(); if (Domains.isLocalhost(host)) lastLocalhostAccess = System.currentTimeMillis(); else {
System.out.println("******** Access not from localhost: " + host);
}
} }
public static Collection<Track> accessTrack(final String host) { public static Collection<Track> accessTrack(final String host) {

@ -31,10 +31,13 @@ import java.io.IOException;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.io.Reader; import java.io.Reader;
import java.net.InetAddress; import java.net.InetAddress;
import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.Map; import java.util.Map;
import java.util.NavigableMap; import java.util.NavigableMap;
import java.util.Random; import java.util.Random;
import java.util.Set;
import java.util.TreeMap; import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
@ -50,6 +53,7 @@ import net.yacy.http.YaCyHttpServer;
import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.workflow.BusyThread; import net.yacy.kelondro.workflow.BusyThread;
import net.yacy.kelondro.workflow.WorkflowThread; import net.yacy.kelondro.workflow.WorkflowThread;
import net.yacy.peers.Seed;
import net.yacy.search.SwitchboardConstants; import net.yacy.search.SwitchboardConstants;
public class serverSwitch public class serverSwitch
@ -139,8 +143,11 @@ public class serverSwitch
/** /**
* get my public IP, either set statically or figure out dynamic * get my public IP, either set statically or figure out dynamic
* @return * This method is deprecated because there may be more than one public IPs of this peer,
* i.e. one IPv4 and one IPv6. Please use myPublicIPs() instead
* @return the public IP of this peer, if known
*/ */
@Deprecated
public String myPublicIP() { public String myPublicIP() {
// if a static IP was configured, we have to return it here ... // if a static IP was configured, we have to return it here ...
final String staticIP = getConfig("staticIP", ""); final String staticIP = getConfig("staticIP", "");
@ -152,6 +159,31 @@ public class serverSwitch
return null; return null;
} }
/**
* Get all my public IPs. If there was a static IP assignment, only one, that IP is returned.
* @return a set of IPs which are supposed to be my own public IPs
*/
public Set<String> myPublicIPs() {
// if a static IP was configured, we have to return it here ...
final String staticIP = getConfig("staticIP", "");
if ( staticIP.length() > 0 ) {
HashSet<String> h = new HashSet<>();
h.add(staticIP);
return h;
}
Set<String> h = new LinkedHashSet<>();
for (InetAddress i: Domains.myPublicIPv6()) {
String s = i.getHostAddress();
if (Seed.isProperIP(s)) h.add(s);
}
for (InetAddress i: Domains.myPublicIPv4()) {
String s = i.getHostAddress();
if (Seed.isProperIP(s)) h.add(s);
}
return h;
}
// a logger for this switchboard // a logger for this switchboard
public void setLog(final ConcurrentLog log) { public void setLog(final ConcurrentLog log) {
this.log = log; this.log = log;

@ -530,7 +530,7 @@ public final class yacy {
p.load(fis); p.load(fis);
fis.close(); fis.close();
// Test for server access restriction (is implemented using Jetty IPaccessHandler which does not support IPv6 // Test for server access restriction (is implemented using Jetty IPaccessHandler which does not support IPv6
// try to disavle IPv6 // try to disable IPv6
String teststr = p.getProperty("serverClient", "*"); String teststr = p.getProperty("serverClient", "*");
if (!teststr.equals("*")) { if (!teststr.equals("*")) {
// testing on Win-8 showed this property has to be set befor Switchboard starts // testing on Win-8 showed this property has to be set befor Switchboard starts
@ -569,7 +569,6 @@ public final class yacy {
} }
} }
/** /**
* Main-method which is started by java. Checks for special arguments or * Main-method which is started by java. Checks for special arguments or
* starts up the application. * starts up the application.
@ -601,7 +600,6 @@ public final class yacy {
if (OS.isWindows) headless = false; if (OS.isWindows) headless = false;
if (args.length >= 1 && args[0].toLowerCase().equals("-gui")) headless = false; if (args.length >= 1 && args[0].toLowerCase().equals("-gui")) headless = false;
System.setProperty("java.awt.headless", headless ? "true" : "false"); System.setProperty("java.awt.headless", headless ? "true" : "false");
// System.setProperty("java.net.preferIPv4Stack", "true"); // DO NOT PREFER IPv6, i.e. freifunk uses ipv6 only and host resolving does not work
String s = ""; for (final String a: args) s += a + " "; String s = ""; for (final String a: args) s += a + " ";
yacyRelease.startParameter = s.trim(); yacyRelease.startParameter = s.trim();

@ -6,7 +6,7 @@ PIDFILE="yacy.pid"
OS="`uname`" OS="`uname`"
#get javastart args #get javastart args
JAVA_ARGS="-server -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Dfile.encoding=UTF-8"; JAVA_ARGS="-server -Djava.awt.headless=true -Dfile.encoding=UTF-8";
#JAVA_ARGS="-verbose:gc -XX:+PrintGCTimeStamps -XX:+PrintGCDetails $JAVA_ARGS"; #JAVA_ARGS="-verbose:gc -XX:+PrintGCTimeStamps -XX:+PrintGCDetails $JAVA_ARGS";
#check if OS is Sun Solaris or one of the OpenSolaris distributions and use different version of id if necessary #check if OS is Sun Solaris or one of the OpenSolaris distributions and use different version of id if necessary

@ -17,7 +17,7 @@ if exist DATA\SETTINGS\httpProxy.conf GoTo :RENAMEINDEX
if exist DATA\SETTINGS\yacy.conf GoTo :GETSTARTOPTS if exist DATA\SETTINGS\yacy.conf GoTo :GETSTARTOPTS
:STARTJAVA :STARTJAVA
set javacmd=%javacmd% -XX:-UseGCOverheadLimit -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Dfile.encoding=UTF-8 set javacmd=%javacmd% -XX:-UseGCOverheadLimit -Djava.awt.headless=true -Dfile.encoding=UTF-8
Rem Starting YaCy Rem Starting YaCy
Echo Generated classpath:%CLASSPATH% Echo Generated classpath:%CLASSPATH%
Echo JRE Parameters:%javacmd% Echo JRE Parameters:%javacmd%

Loading…
Cancel
Save