From e48189c710c0bb4f5a981e7b6fcf628e40b57c3a Mon Sep 17 00:00:00 2001 From: orbiter Date: Sun, 29 Apr 2007 22:05:34 +0000 Subject: [PATCH] enhanced cluster routing - cluster definitions can now contain an addition for local ip addresses - cluster-cluster communication uses the local ip address instead the global address, if one is given git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3624 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/Blog.java | 2 +- htroot/Bookmarks.java | 2 +- htroot/ConfigNetwork_p.java | 3 +- htroot/ConfigRobotsTxt_p.java | 2 +- htroot/CrawlURLFetch_p.java | 6 +- htroot/DetailedSearch.java | 2 +- htroot/Messages_p.java | 2 +- htroot/Network.java | 12 ++-- htroot/News.java | 6 +- htroot/Status.java | 4 +- htroot/Surftips.java | 4 +- htroot/htdocsdefault/dir.java | 6 +- htroot/www/welcome.java | 2 +- htroot/yacy/transfer.java | 4 +- htroot/yacysearch.java | 2 +- source/de/anomic/data/wiki/wikiParser.java | 2 +- source/de/anomic/data/wikiCode.java | 4 +- .../de/anomic/plasma/plasmaDHTTransfer.java | 2 +- .../plasma/plasmaRankingDistribution.java | 2 +- .../de/anomic/plasma/plasmaSearchEvent.java | 5 +- .../de/anomic/plasma/plasmaSwitchboard.java | 13 +++-- source/de/anomic/yacy/yacyClient.java | 25 ++++---- source/de/anomic/yacy/yacyCore.java | 36 +++++++----- source/de/anomic/yacy/yacyDHTAction.java | 9 ++- source/de/anomic/yacy/yacyPeerActions.java | 18 +++--- source/de/anomic/yacy/yacySearch.java | 20 ++++--- source/de/anomic/yacy/yacySeed.java | 29 +++++++++- source/de/anomic/yacy/yacySeedDB.java | 58 ++++++++++++------- 28 files changed, 169 insertions(+), 113 deletions(-) diff --git a/htroot/Blog.java b/htroot/Blog.java index cef677cb3..f62c3b078 100644 --- a/htroot/Blog.java +++ b/htroot/Blog.java @@ -84,7 +84,7 @@ public class Blog { boolean hasRights = switchboard.verifyAuthentication(header, true); final boolean xml = ((String)header.get(httpHeader.CONNECTION_PROP_PATH)).endsWith(".xml"); - final String address = yacyCore.seedDB.mySeed.getAddress(); + final String address = yacyCore.seedDB.mySeed.getPublicAddress(); if(hasRights) { prop.put("mode_admin",1); diff --git a/htroot/Bookmarks.java b/htroot/Bookmarks.java index 0f750bce4..0c6187917 100644 --- a/htroot/Bookmarks.java +++ b/htroot/Bookmarks.java @@ -87,7 +87,7 @@ public class Bookmarks { prop.put("user",username); // set peer address - final String address = yacyCore.seedDB.mySeed.getAddress(); + final String address = yacyCore.seedDB.mySeed.getPublicAddress(); prop.put("address", address); //defaultvalues diff --git a/htroot/ConfigNetwork_p.java b/htroot/ConfigNetwork_p.java index f1d47c34d..a2b13f642 100644 --- a/htroot/ConfigNetwork_p.java +++ b/htroot/ConfigNetwork_p.java @@ -177,7 +177,8 @@ public class ConfigNetwork_p { String[] s = input.split(","); input = ""; for (int i = 0; i < s.length; i++) { - if ((s[i].endsWith(".yacyh")) || (s[i].endsWith(".yacy"))) input += "," + s[i]; + if ((s[i].endsWith(".yacyh")) || (s[i].endsWith(".yacy")) || + (s[i].indexOf(".yacyh=") > 0) || (s[i].indexOf(".yacy=") > 0)) input += "," + s[i]; } if (input.length() == 0) return input; else return input.substring(1); } diff --git a/htroot/ConfigRobotsTxt_p.java b/htroot/ConfigRobotsTxt_p.java index 159e68816..5728c9e08 100644 --- a/htroot/ConfigRobotsTxt_p.java +++ b/htroot/ConfigRobotsTxt_p.java @@ -62,7 +62,7 @@ public class ConfigRobotsTxt_p { final servletProperties prop = new servletProperties(); httpdRobotsTxtConfig rbc = ((plasmaSwitchboard)env).robotstxtConfig; - prop.put("clientname", yacyCore.seedDB.mySeed.getAddress()); + prop.put("clientname", yacyCore.seedDB.mySeed.getPublicAddress()); if (post != null) { if (post.containsKey("save")) { diff --git a/htroot/CrawlURLFetch_p.java b/htroot/CrawlURLFetch_p.java index 03340d1d2..c5d510f1c 100644 --- a/htroot/CrawlURLFetch_p.java +++ b/htroot/CrawlURLFetch_p.java @@ -161,7 +161,7 @@ public class CrawlURLFetch_p { ys = yacyCore.seedDB.get(post.get("peerhash", null)); if (ys != null) { if ((url = URLFetcher.getListServletURL( - ys.getAddress(), + ys.getPublicAddress(), URLFetcher.MODE_LIST, count, yacyCore.seedDB.mySeed.hash)) == null) { @@ -311,7 +311,7 @@ public class CrawlURLFetch_p { private static int getURLs2Fetch(yacySeed seed, httpRemoteProxyConfig theRemoteProxyConfig) { try { String answer = new String(httpc.wget( - URLFetcher.getListServletURL(seed.getAddress(), URLFetcher.MODE_COUNT, 0, null), + URLFetcher.getListServletURL(seed.getPublicAddress(), URLFetcher.MODE_COUNT, 0, null), seed.getIP(), 5000, null, null, @@ -469,7 +469,7 @@ public class CrawlURLFetch_p { } if (ys == null) return null; - return getListServletURL(ys.getAddress(), MODE_LIST, this.count, yacyCore.seedDB.mySeed.hash); + return getListServletURL(ys.getPublicAddress(), MODE_LIST, this.count, yacyCore.seedDB.mySeed.hash); } private int stackURLs(String[] urls) throws InterruptedException { diff --git a/htroot/DetailedSearch.java b/htroot/DetailedSearch.java index 59c510c3d..9656af759 100644 --- a/htroot/DetailedSearch.java +++ b/htroot/DetailedSearch.java @@ -230,7 +230,7 @@ public class DetailedSearch { final long searchtime = 1000 * Long.parseLong(post.get("localTime", "10")); final boolean yacyonline = ((yacyCore.seedDB != null) && (yacyCore.seedDB.mySeed != null) && - (yacyCore.seedDB.mySeed.getAddress() != null)); + (yacyCore.seedDB.mySeed.getPublicAddress() != null)); String urlmask = ""; if (post.containsKey("urlmask") && post.get("urlmask").equals("no")) { diff --git a/htroot/Messages_p.java b/htroot/Messages_p.java index 8420da73a..b482ec44f 100644 --- a/htroot/Messages_p.java +++ b/htroot/Messages_p.java @@ -76,7 +76,7 @@ public class Messages_p { serverObjects prop = new serverObjects(); // set peer address / name - final String peerAddress = yacyCore.seedDB.mySeed.getAddress(); + final String peerAddress = yacyCore.seedDB.mySeed.getPublicAddress(); final String peerName = yacyCore.seedDB.mySeed.getName(); prop.put("peerAddress", peerAddress); prop.put("peerName", peerName); diff --git a/htroot/Network.java b/htroot/Network.java index ab5ccca41..420e6e9db 100644 --- a/htroot/Network.java +++ b/htroot/Network.java @@ -202,19 +202,19 @@ public class Network { yacySeed peer = new yacySeed((String) post.get("peerHash"),map); yacyCore.peerActions.updateMySeed(); - final int added = yacyClient.publishMySeed(peer.getAddress(), peer.hash); + final int added = yacyClient.publishMySeed(peer.getPublicAddress(), peer.hash); if (added <= 0) { prop.put("table_comment",1); - prop.put("table_comment_status","publish: disconnected peer '" + peer.getName() + "/" + post.get("peerHash") + "' from " + peer.getAddress()); + prop.put("table_comment_status","publish: disconnected peer '" + peer.getName() + "/" + post.get("peerHash") + "' from " + peer.getPublicAddress()); } else { peer = yacyCore.seedDB.getConnected(peer.hash); if (peer == null) { prop.put("table_comment",1); - prop.put("table_comment_status","publish: disconnected peer '" + peer.getName() + "/" + post.get("peerHash") + "' from " + peer.getAddress()); + prop.put("table_comment_status","publish: disconnected peer '" + peer.getName() + "/" + post.get("peerHash") + "' from " + peer.getPublicAddress()); } else { prop.put("table_comment",2); - prop.put("table_comment_status","publish: handshaked " + peer.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + peer.getName() + "' at " + peer.getAddress()); + prop.put("table_comment_status","publish: handshaked " + peer.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + peer.getName() + "' at " + peer.getPublicAddress()); prop.put("table_comment_details",peer.toString()); } } @@ -352,7 +352,7 @@ public class Network { } else { prop.put(STR_TABLE_LIST + conCount + "_updatedWiki", 1); prop.put(STR_TABLE_LIST + conCount + "_updatedWiki_page", (String) wikiMap.get("page")); - prop.put(STR_TABLE_LIST + conCount + "_updatedWiki_address", seed.getAddress()); + prop.put(STR_TABLE_LIST + conCount + "_updatedWiki_address", seed.getPublicAddress()); } if ((blogMap = (Map) updatedBlog.get(seed.hash)) == null) { prop.put(STR_TABLE_LIST + conCount + "_updatedBlog", 0); @@ -360,7 +360,7 @@ public class Network { prop.put(STR_TABLE_LIST + conCount + "_updatedBlog", 1); prop.put(STR_TABLE_LIST + conCount + "_updatedBlog_page", (String) blogMap.get("page")); prop.put(STR_TABLE_LIST + conCount + "_updatedBlog_subject", (String) blogMap.get("subject")); - prop.put(STR_TABLE_LIST + conCount + "_updatedBlog_address", seed.getAddress()); + prop.put(STR_TABLE_LIST + conCount + "_updatedBlog_address", seed.getPublicAddress()); } PPM = seed.getPPM(); QPM = seed.getQPM(); diff --git a/htroot/News.java b/htroot/News.java index a4565f33d..d9486e300 100644 --- a/htroot/News.java +++ b/htroot/News.java @@ -184,11 +184,11 @@ public class News { title = record.attribute("title", ""); description = record.attribute("url", ""); } else if (category.equals(yacyNewsPool.CATEGORY_WIKI_UPDATE)) { - link = (seed==null)?"":"http://" + seed.getAddress() + "/Wiki.html?page=" + record.attribute("page", ""); + link = (seed==null)?"":"http://" + seed.getPublicAddress() + "/Wiki.html?page=" + record.attribute("page", ""); title = record.attribute("author", "Anonymous") + ": " + record.attribute("page", ""); description = "Wiki Update: " + record.attribute("description", ""); } else if (category.equals(yacyNewsPool.CATEGORY_BLOG_ADD)) { - link = (seed==null)?"":"http://" + seed.getAddress() + "/Blog.html?page=" + record.attribute("page", ""); + link = (seed==null)?"":"http://" + seed.getPublicAddress() + "/Blog.html?page=" + record.attribute("page", ""); title = record.attribute("author", "Anonymous") + ": " + record.attribute("page", ""); description = "Blog Entry: " + record.attribute("subject", ""); } else { @@ -205,7 +205,7 @@ public class News { } // adding the peer address - prop.put("address",yacyCore.seedDB.mySeed.getAddress()); + prop.put("address",yacyCore.seedDB.mySeed.getPublicAddress()); // return rewrite properties return prop; diff --git a/htroot/Status.java b/htroot/Status.java index b67807fb1..9c8bed11e 100644 --- a/htroot/Status.java +++ b/htroot/Status.java @@ -224,13 +224,13 @@ public class Status { prop.put("peerStatistics_principalConnects", yacyCore.peerActions.principalConnects); prop.put("peerStatistics_disconnects", yacyCore.peerActions.disconnects); prop.put("peerStatistics_connects", yacyCore.seedDB.mySeed.get(yacySeed.CCOUNT, "0")); - if (yacyCore.seedDB.mySeed.getAddress() == null) { + if (yacyCore.seedDB.mySeed.getPublicAddress() == null) { thisHash = yacyCore.seedDB.mySeed.hash; prop.put("peerAddress", 1); // not assigned + instructions } else { thisHash = yacyCore.seedDB.mySeed.hash; prop.put("peerAddress", 2); // Address - prop.put("peerAddress_address", yacyCore.seedDB.mySeed.getAddress()); + prop.put("peerAddress_address", yacyCore.seedDB.mySeed.getPublicAddress()); prop.put("peerAddress_peername", env.getConfig("peerName", "").toLowerCase()); } } diff --git a/htroot/Surftips.java b/htroot/Surftips.java index 550ce4739..98370d7a0 100644 --- a/htroot/Surftips.java +++ b/htroot/Surftips.java @@ -273,7 +273,7 @@ public class Surftips { yacySeed seed = yacyCore.seedDB.getConnected(record.originator()); if (seed == null) seed = yacyCore.seedDB.getDisconnected(record.originator()); if (seed != null) { - url = "http://" + seed.getAddress() + "/Wiki.html?page=" + record.attribute("page", ""); + url = "http://" + seed.getPublicAddress() + "/Wiki.html?page=" + record.attribute("page", ""); entry = rowdef.newEntry(new byte[][]{ url.getBytes(), (record.attribute("author", "Anonymous") + ": " + record.attribute("page", "")).getBytes("UTF-8"), @@ -288,7 +288,7 @@ public class Surftips { yacySeed seed = yacyCore.seedDB.getConnected(record.originator()); if (seed == null) seed = yacyCore.seedDB.getDisconnected(record.originator()); if (seed != null) { - url = "http://" + seed.getAddress() + "/Blog.html?page=" + record.attribute("page", ""); + url = "http://" + seed.getPublicAddress() + "/Blog.html?page=" + record.attribute("page", ""); entry = rowdef.newEntry(new byte[][]{ url.getBytes(), (record.attribute("author", "Anonymous") + ": " + record.attribute("page", "")).getBytes("UTF-8"), diff --git a/htroot/htdocsdefault/dir.java b/htroot/htdocsdefault/dir.java index 962b96d2b..c0528e2af 100644 --- a/htroot/htdocsdefault/dir.java +++ b/htroot/htdocsdefault/dir.java @@ -105,7 +105,7 @@ public class dir { // general settings prop.put("peername", env.getConfig("peerName", "")); prop.put("peerdomain", env.getConfig("peerName", "").toLowerCase()); - prop.put("peeraddress", yacyCore.seedDB.mySeed.getAddress()); + prop.put("peeraddress", yacyCore.seedDB.mySeed.getPublicAddress()); prop.put("hostname", serverCore.publicIP()); try{ prop.put("hostip", InetAddress.getByName(serverCore.publicIP()).getHostAddress()); @@ -288,7 +288,7 @@ public class dir { if (f.isDirectory()) { // the entry is a directory prop.put("dirlist_" + fileIdx + "_dir" , 1); - prop.putSafeXML("dirlist_" + fileIdx + "_dir_URL","http://" + yacyCore.seedDB.mySeed.getAddress() + path + fileName + "/"); + prop.putSafeXML("dirlist_" + fileIdx + "_dir_URL","http://" + yacyCore.seedDB.mySeed.getPublicAddress() + path + fileName + "/"); } else { // determine if we should display the description string or a preview image boolean showImage = /* (description.length() == 0) && */ (fileName.endsWith(".jpg") || fileName.endsWith(".gif") || fileName.endsWith(".png")); @@ -300,7 +300,7 @@ public class dir { prop.put("dirlist_" + fileIdx + "_dir_sizeBytes" , Long.toString(f.length())); // the unique url prop.putSafeXML("dirlist_" + fileIdx + "_dir_yacyhURL",yacyhURL(yacyCore.seedDB.mySeed, fileName, md5s)); - prop.putSafeXML("dirlist_" + fileIdx + "_dir_URL","http://" + yacyCore.seedDB.mySeed.getAddress() + path + fileName); + prop.putSafeXML("dirlist_" + fileIdx + "_dir_URL","http://" + yacyCore.seedDB.mySeed.getPublicAddress() + path + fileName); // the md5 sum of the file prop.put("dirlist_" + fileIdx + "_dir_md5s",md5s); // description mode: 0...image preview, 1...description text diff --git a/htroot/www/welcome.java b/htroot/www/welcome.java index 2e8ee5546..5706ac2ee 100644 --- a/htroot/www/welcome.java +++ b/htroot/www/welcome.java @@ -68,7 +68,7 @@ public class welcome { prop.put("peername", env.getConfig("peerName", "")); prop.put("peerdomain", env.getConfig("peerName", "").toLowerCase()); - prop.put("peeraddress", yacyCore.seedDB.mySeed.getAddress()); + prop.put("peeraddress", yacyCore.seedDB.mySeed.getPublicAddress()); prop.put("hostname", serverCore.publicIP()); try{ prop.put("hostip", InetAddress.getByName(serverCore.publicIP()).getHostAddress()); diff --git a/htroot/yacy/transfer.java b/htroot/yacy/transfer.java index 1d21c8708..a1558d3ea 100644 --- a/htroot/yacy/transfer.java +++ b/htroot/yacy/transfer.java @@ -91,7 +91,7 @@ public final class transfer { // reject unknown peers: this does not appear fair, but anonymous senders are dangerous // reject paths that contain '..' because they are dangerous if (otherseed == null) sb.getLog().logFine("RankingTransmission: rejected unknown peer '" + otherpeer + "', current IP " + header.get("CLIENTIP", "unknown")); - if (filename.indexOf("..") >= 0) sb.getLog().logFine("RankingTransmission: rejected wrong path '" + filename + "' from peer " + otherseed.getName() + "/" + otherseed.getAddress()+ ", current IP " + header.get("CLIENTIP", "unknown")); + if (filename.indexOf("..") >= 0) sb.getLog().logFine("RankingTransmission: rejected wrong path '" + filename + "' from peer " + otherseed.getName() + "/" + otherseed.getPublicAddress()+ ", current IP " + header.get("CLIENTIP", "unknown")); return prop; } @@ -106,7 +106,7 @@ public final class transfer { String access = kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(otherpeer + ":" + filename)) + ":" + kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw("" + System.currentTimeMillis())); prop.putASIS("response", "ok"); prop.putASIS("process_access", access); - prop.putASIS("process_address", yacyCore.seedDB.mySeed.getAddress()); + prop.putASIS("process_address", yacyCore.seedDB.mySeed.getPublicAddress()); prop.putASIS("process_protocol", "http"); prop.putASIS("process_path", ""); // currently empty; the store process will find a path prop.putASIS("process_maxsize", "-1"); // if response is too big we return the size of the file diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index 8f7b1607c..943dea596 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -258,7 +258,7 @@ public class yacysearch { } // prepare search properties - final boolean yacyonline = ((yacyCore.seedDB != null) && (yacyCore.seedDB.mySeed != null) && (yacyCore.seedDB.mySeed.getAddress() != null)); + final boolean yacyonline = ((yacyCore.seedDB != null) && (yacyCore.seedDB.mySeed != null) && (yacyCore.seedDB.mySeed.getPublicAddress() != null)); final boolean samesearch = env.getConfig("last-search", "").equals(querystring + contentdomString); final boolean globalsearch = (global) && (yacyonline) && (!samesearch); diff --git a/source/de/anomic/data/wiki/wikiParser.java b/source/de/anomic/data/wiki/wikiParser.java index a9aa43c39..3165b07bb 100644 --- a/source/de/anomic/data/wiki/wikiParser.java +++ b/source/de/anomic/data/wiki/wikiParser.java @@ -68,7 +68,7 @@ public class wikiParser { tokens = new Token[] { new SimpleToken('=', '=', new String[][] { null, { "h2" }, { "h3" }, { "h4" } }, true), new SimpleToken('\'', '\'', new String[][] { null, { "i" }, { "b" }, null, { "b", "i" } }, false), - new LinkToken(yacyCore.seedDB.mySeed.getAddress(), "Wiki.html?page=", sb), + new LinkToken(yacyCore.seedDB.mySeed.getPublicAddress(), "Wiki.html?page=", sb), new ListToken('*', "ul"), new ListToken('#', "ol"), new ListToken(':', "blockquote", null), diff --git a/source/de/anomic/data/wikiCode.java b/source/de/anomic/data/wikiCode.java index 3a82cfb10..7e16600eb 100644 --- a/source/de/anomic/data/wikiCode.java +++ b/source/de/anomic/data/wikiCode.java @@ -750,7 +750,7 @@ public class wikiCode { // or an image DATA/HTDOCS/grafics/kaskelix.jpg with [[Image:grafics/kaskelix.jpg]] // you are free to use other sub-paths of DATA/HTDOCS if (kl.indexOf("://")<1) { - kl = "http://" + yacyCore.seedDB.mySeed.getAddress().trim() + "/" + kl; + kl = "http://" + yacyCore.seedDB.mySeed.getPublicAddress().trim() + "/" + kl; } result = result.substring(0, p0) + "" + result.substring(p1 + 2); @@ -791,7 +791,7 @@ public class wikiCode { // or a file DATA/HTDOCS/www/page.html with [www/page.html] // you are free to use other sub-paths of DATA/HTDOCS if (kl.indexOf("://")<1) { - kl = "http://" + yacyCore.seedDB.mySeed.getAddress().trim() + "/" + kl; + kl = "http://" + yacyCore.seedDB.mySeed.getPublicAddress().trim() + "/" + kl; } result = result.substring(0, p0) + "" + kv + "" + result.substring(p1 + 1); } diff --git a/source/de/anomic/plasma/plasmaDHTTransfer.java b/source/de/anomic/plasma/plasmaDHTTransfer.java index 46d41b3de..8696b3018 100644 --- a/source/de/anomic/plasma/plasmaDHTTransfer.java +++ b/source/de/anomic/plasma/plasmaDHTTransfer.java @@ -236,7 +236,7 @@ public class plasmaDHTTransfer extends Thread { return; // doing a peer ping to the remote seed - int added = yacyClient.publishMySeed(this.seed.getAddress(), this.seed.hash); + int added = yacyClient.publishMySeed(this.seed.getPublicAddress(), this.seed.hash); if (added < 0) { // inc. retry counter retryCount++; diff --git a/source/de/anomic/plasma/plasmaRankingDistribution.java b/source/de/anomic/plasma/plasmaRankingDistribution.java index 5f6d78bcb..085a675e8 100644 --- a/source/de/anomic/plasma/plasmaRankingDistribution.java +++ b/source/de/anomic/plasma/plasmaRankingDistribution.java @@ -175,7 +175,7 @@ public final class plasmaRankingDistribution { target = yacyCore.seedDB.anySeedVersion(yacyVersion.YACY_ACCEPTS_RANKING_TRANSMISSION); if (target == null) continue; - String targetaddress = target.getAddress(); + String targetaddress = target.getPublicAddress(); if (transferRankingAddress(crfile, targetaddress)) return true; } return false; diff --git a/source/de/anomic/plasma/plasmaSearchEvent.java b/source/de/anomic/plasma/plasmaSearchEvent.java index d96a9e620..8ce9b75d2 100644 --- a/source/de/anomic/plasma/plasmaSearchEvent.java +++ b/source/de/anomic/plasma/plasmaSearchEvent.java @@ -48,7 +48,6 @@ import java.util.Iterator; import java.util.Map; import java.util.Set; import java.util.TreeMap; -import java.util.TreeSet; import de.anomic.index.indexContainer; import de.anomic.index.indexRWIEntry; @@ -79,7 +78,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable { private yacySearch[] primarySearchThreads, secondarySearchThreads; private long searchtime; private int searchcount; - private TreeSet preselectedPeerHashes; + private TreeMap preselectedPeerHashes; public plasmaSearchEvent(plasmaSearchQuery query, plasmaSearchRankingProfile ranking, @@ -90,7 +89,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable { plasmaWordIndex wordIndex, plasmaCrawlLURL urlStore, plasmaSnippetCache snippetCache, - TreeSet preselectedPeerHashes) { + TreeMap preselectedPeerHashes) { this.log = log; this.wordIndex = wordIndex; this.query = query; diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 85c7f7d72..9260e7480 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -118,6 +118,7 @@ import java.util.Hashtable; import java.util.Iterator; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.TreeSet; import de.anomic.data.blogBoard; @@ -251,7 +252,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser public double lastrequestedQueries = 0d; public int totalPPM = 0; public double totalQPM = 0d; - public TreeSet clusterhashes; + public TreeMap clusterhashes; // map of peerhash(String)/alternative-local-address as ip:port or only ip (String) or null if address in seed should be used /* * Remote Proxy configuration @@ -864,6 +865,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser public plasmaSwitchboard(String rootPath, String initPath, String configPath) { super(rootPath, initPath, configPath); + sb=this; // set loglevel and log setLog(new serverLog("PLASMA")); @@ -1276,7 +1278,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // init robinson cluster this.clusterhashes = yacyCore.seedDB.clusterHashes(getConfig("cluster.peers.yacydomain", "")); - sb=this; log.logConfig("Finished Switchboard Initialization"); } @@ -1355,7 +1356,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser return network.indexOf(peer) >= 0; } else if (clustermode.equals("publiccluster")) { // check if we got the request from a peer in the public cluster - return this.clusterhashes.contains(peer); + return this.clusterhashes.containsKey(peer); } else { return false; } @@ -1370,10 +1371,10 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser if (clustermode.equals("privatecluster")) { // check if we got the request from a peer in the private cluster String network = getConfig("cluster.peers.ipport", ""); - return network.indexOf(seed.getAddress()) >= 0; + return network.indexOf(seed.getPublicAddress()) >= 0; } else if (clustermode.equals("publiccluster")) { // check if we got the request from a peer in the public cluster - return this.clusterhashes.contains(seed.hash); + return this.clusterhashes.containsKey(seed.hash); } else { return false; } @@ -2830,7 +2831,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser hash = yacySeed.hexHash2b64Hash(host.substring(p + 1, host.length() - 6)); seed = yacyCore.seedDB.getConnected(hash); filename = comp.url().getFile(); - if ((seed == null) || ((address = seed.getAddress()) == null)) { + if ((seed == null) || ((address = seed.getPublicAddress()) == null)) { // seed is not known from here wordIndex.removeWordReferences(plasmaCondenser.getWords(("yacyshare " + filename.replace('?', ' ') + " " + comp.title()).getBytes(), "UTF-8").keySet(), urlentry.hash()); wordIndex.loadedURL.remove(urlentry.hash()); // clean up diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index 82a57a591..c60f629b8 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -255,7 +255,7 @@ public final class yacyClient { // sending request final HashMap result = nxTools.table( httpc.wget( - new URL("http://" + target.getAddress() + + new URL("http://" + target.getClusterAddress() + "/yacy/query.html" + "?iam=" + yacyCore.seedDB.mySeed.hash + "&youare=" + target.hash + @@ -291,7 +291,7 @@ public final class yacyClient { // sending request final HashMap result = nxTools.table( httpc.wget( - new URL("http://" + target.getAddress() + + new URL("http://" + target.getClusterAddress() + "/yacy/query.html" + "?iam=" + yacyCore.seedDB.mySeed.hash + "&youare=" + target.hash + @@ -327,7 +327,7 @@ public final class yacyClient { // building url final String querystr = - "http://" + target.getAddress() + + "http://" + target.getClusterAddress() + "/yacy/query.html" + "?iam=" + yacyCore.seedDB.mySeed.hash + "&youare=" + target.hash + @@ -410,7 +410,7 @@ public final class yacyClient { (yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy()); // building url - final String url = "http://" + ((targetPeer.getIP().equals(yacyCore.seedDB.mySeed.getIP())) ? "localhost:" + targetPeer.getPort() : targetPeer.getAddress()) + "/yacy/search.html"; + final String url = "http://" + targetPeer.getClusterAddress() + "/yacy/search.html"; // adding all needed parameters /* @@ -669,17 +669,16 @@ public final class yacyClient { return null; } } - public static String targetAddress(String targetHash) { // find target address String address; if (targetHash.equals(yacyCore.seedDB.mySeed.hash)) { - address = yacyCore.seedDB.mySeed.getAddress(); + address = yacyCore.seedDB.mySeed.getClusterAddress(); } else { final yacySeed targetSeed = yacyCore.seedDB.getConnected(targetHash); if (targetSeed == null) { return null; } - address = targetSeed.getAddress(); + address = targetSeed.getClusterAddress(); } if (address == null) address = "localhost:8080"; return address; @@ -827,7 +826,7 @@ public final class yacyClient { post.put("ttl", "0"); // determining target address - final String address = targetSeed.getAddress(); + final String address = targetSeed.getClusterAddress(); if (address == null) { return null; } // sending request @@ -892,7 +891,7 @@ public final class yacyClient { final String key = crypt.randomSalt(); // determining target address - String address = targetSeed.getAddress(); + String address = targetSeed.getClusterAddress(); if (address == null) { return null; } // sending request @@ -1015,7 +1014,7 @@ public final class yacyClient { } private static HashMap transferRWI(yacySeed targetSeed, indexContainer[] indexes, boolean gzipBody, int timeout) { - final String address = targetSeed.getAddress(); + final String address = targetSeed.getPublicAddress(); if (address == null) { return null; } // should we use the proxy? @@ -1091,7 +1090,7 @@ public final class yacyClient { private static HashMap transferURL(yacySeed targetSeed, indexURLEntry[] urls, boolean gzipBody, int timeout) { // this post a message to the remote message board - final String address = targetSeed.getAddress(); + final String address = targetSeed.getPublicAddress(); if (address == null) { return null; } // should we use the proxy? @@ -1163,7 +1162,7 @@ public final class yacyClient { final serverObjects post = new serverObjects(2); post.put("iam", yacyCore.seedDB.mySeed.hash); post.put("youare", targetSeed.hash); - String address = targetSeed.getAddress(); + String address = targetSeed.getClusterAddress(); if (address == null) { address = "localhost:8080"; } try { return nxTools.table( @@ -1200,7 +1199,7 @@ public final class yacyClient { final HashMap result = nxTools.table( httpc.wget( - new URL("http://" + target.getAddress() + "/yacy/search.html" + + new URL("http://" + target.getPublicAddress() + "/yacy/search.html" + "?myseed=" + yacyCore.seedDB.mySeed.genSeedStr(null) + "&youare=" + target.hash + "&key=" + "&myseed=" + yacyCore.seedDB.mySeed.genSeedStr(null) + diff --git a/source/de/anomic/yacy/yacyCore.java b/source/de/anomic/yacy/yacyCore.java index c66c1d353..9a375eeec 100644 --- a/source/de/anomic/yacy/yacyCore.java +++ b/source/de/anomic/yacy/yacyCore.java @@ -358,9 +358,9 @@ public class yacyCore { return true; } log.logInfo("re-connect own seed"); - final String oldAddress = seedDB.mySeed.getAddress(); + final String oldAddress = seedDB.mySeed.getPublicAddress(); /*final int newSeeds =*/ publishMySeed(true); - return (oldAddress != null && oldAddress.equals(seedDB.mySeed.getAddress())); + return (oldAddress != null && oldAddress.equals(seedDB.mySeed.getPublicAddress())); } protected class publishThread extends Thread { @@ -384,15 +384,15 @@ public class yacyCore { public void run() { try { - this.added = yacyClient.publishMySeed(seed.getAddress(), seed.hash); + this.added = yacyClient.publishMySeed(seed.getPublicAddress(), seed.hash); if (this.added < 0) { // no or wrong response, delete that address - log.logInfo("publish: disconnected " + this.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' from " + this.seed.getAddress()); + log.logInfo("publish: disconnected " + this.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' from " + this.seed.getPublicAddress()); peerActions.peerDeparture(this.seed); } else { // success! we have published our peer to a senior peer // update latest news from the other peer - log.logInfo("publish: handshaked " + this.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getAddress()); + log.logInfo("publish: handshaked " + this.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getPublicAddress()); } } catch (Exception e) { log.logSevere("publishThread: error with target seed " + seed.toString() + ": " + e.getMessage(), e); @@ -433,12 +433,17 @@ public class yacyCore { seeds = seedDB.seedsByAge(true, attempts + 10); // best for fast connection // add also all peers from cluster if this is a public robinson cluster if (plasmaSwitchboard.getSwitchboard().clusterhashes != null) { - Iterator i = plasmaSwitchboard.getSwitchboard().clusterhashes.iterator(); + Iterator i = plasmaSwitchboard.getSwitchboard().clusterhashes.entrySet().iterator(); String hash; + Map.Entry entry; + yacySeed seed; while (i.hasNext()) { - hash = (String) i.next(); + entry = (Map.Entry) i.next(); + hash = (String) entry.getKey(); if (seeds.containsKey(hash)) continue; - seeds.put(hash, seedDB.get(hash)); + seed = seedDB.get(hash); + seed.setAlternativeAddress((String) entry.getValue()); + seeds.put(hash, seed); } } } else { @@ -499,7 +504,7 @@ public class yacyCore { seed = (yacySeed) seedList.remove(0); if (seed == null) continue; - final String address = seed.getAddress(); + final String address = seed.getPublicAddress(); log.logFine("HELLO #" + i + " to peer '" + seed.get(yacySeed.NAME, "") + "' at " + address); // debug if ((address == null) || (seed.isProper() != null)) { // we don't like that address, delete it @@ -542,20 +547,21 @@ public class yacyCore { while ((newSeeds < 0) && (contactedSeedCount < peerPingInitial) && (!seedList.isEmpty())) { seed = (yacySeed) seedList.remove(0); if (seed != null) { - final String address = seed.getAddress(); + String address = seed.getPublicAddress(); log.logFine("HELLO x" + contactedSeedCount + " to peer '" + seed.get(yacySeed.NAME, "") + "' at " + address); // debug if ((address == null) || (seed.isProper() != null)) { peerActions.peerDeparture(seed); } else { + if (seed.alternativeIP != null) address = seed.alternativeIP + ":" + seed.getPort(); contactedSeedCount++; //new publishThread(yacyCore.publishThreadGroup,seeds[i],sync,syncList)).start(); try { - newSeeds = yacyClient.publishMySeed(seed.getAddress(), seed.hash); + newSeeds = yacyClient.publishMySeed(address, seed.hash); if (newSeeds < 0) { - log.logInfo("publish: disconnected " + seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + seed.getName() + "' from " + seed.getAddress()); + log.logInfo("publish: disconnected " + seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + seed.getName() + "' from " + seed.getPublicAddress()); peerActions.peerDeparture(seed); } else { - log.logInfo("publish: handshaked " + seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + seed.getName() + "' at " + seed.getAddress()); + log.logInfo("publish: handshaked " + seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + seed.getName() + "' at " + seed.getPublicAddress()); } } catch (Exception e) { log.logSevere("publishMySeed: error with target seed " + seed.toString() + ": " + e.getMessage(), e); @@ -632,7 +638,7 @@ public class yacyCore { if (seedDB.mySeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR).equals(yacySeed.PEERTYPE_JUNIOR)) // ??????????????? seedDB.mySeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); // to start bootstraping, we need to be recognised as PEERTYPE_SENIOR peer log.logInfo("publish: no recipient found, our address is " + - ((seedDB.mySeed.getAddress() == null) ? "unknown" : seedDB.mySeed.getAddress())); + ((seedDB.mySeed.getPublicAddress() == null) ? "unknown" : seedDB.mySeed.getPublicAddress())); peerActions.saveMySeed(); return 0; } catch (InterruptedException e) { @@ -782,7 +788,7 @@ public class yacyCore { String logt; // be shure that we have something to say - if (seedDB.mySeed.getAddress() == null) { + if (seedDB.mySeed.getPublicAddress() == null) { final String errorMsg = "We have no valid IP address until now"; log.logWarning("SaveSeedList: " + errorMsg); return errorMsg; diff --git a/source/de/anomic/yacy/yacyDHTAction.java b/source/de/anomic/yacy/yacyDHTAction.java index 4fc684e3e..87b71f14d 100644 --- a/source/de/anomic/yacy/yacyDHTAction.java +++ b/source/de/anomic/yacy/yacyDHTAction.java @@ -46,6 +46,7 @@ package de.anomic.yacy; import java.util.ArrayList; import java.util.Enumeration; import java.util.Hashtable; +import java.util.TreeMap; import java.util.TreeSet; import de.anomic.kelondro.kelondroBase64Order; @@ -217,10 +218,12 @@ public class yacyDHTAction implements yacyPeerAction { return seed; } - public synchronized yacySeed getPublicClusterCrawlSeed(String urlHash, TreeSet clusterhashes) { - kelondroCloneableIterator i = new kelondroRotateIterator(new kelondroCloneableSetIterator(clusterhashes, urlHash), null); + public synchronized yacySeed getPublicClusterCrawlSeed(String urlHash, TreeMap clusterhashes) { + kelondroCloneableIterator i = new kelondroRotateIterator(new kelondroCloneableSetIterator((TreeSet) clusterhashes.keySet(), urlHash), null); if (i.hasNext()) { - return seedDB.getConnected((String) i.next()); + yacySeed seed = seedDB.getConnected((String) i.next()); + seed.setAlternativeAddress((String) clusterhashes.get(seed.hash)); + return seed; } return null; } diff --git a/source/de/anomic/yacy/yacyPeerActions.java b/source/de/anomic/yacy/yacyPeerActions.java index 4ccbca7a1..68e292d0d 100644 --- a/source/de/anomic/yacy/yacyPeerActions.java +++ b/source/de/anomic/yacy/yacyPeerActions.java @@ -276,7 +276,7 @@ public class yacyPeerActions { return false; } if ((this.seedDB.mySeed != null) && (seed.hash.equals(this.seedDB.mySeed.hash))) { - yacyCore.log.logInfo("connect: SELF reference " + seed.getAddress()); + yacyCore.log.logInfo("connect: SELF reference " + seed.getPublicAddress()); return false; } final String peerType = seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN); @@ -312,7 +312,7 @@ public class yacyPeerActions { } if (Math.abs(nowUTC0Time - ctimeUTC0) > 60 * 60 * 24 * 1000) { // the new connection is out-of-age, we reject the connection - yacyCore.log.logFine("connect: rejecting out-dated peer '" + seed.getName() + "' from " + seed.getAddress() + "; nowUTC0=" + nowUTC0Time + ", seedUTC0=" + ctimeUTC0 + ", TimeDiff=" + serverDate.intervalToString(Math.abs(nowUTC0Time - ctimeUTC0))); + yacyCore.log.logFine("connect: rejecting out-dated peer '" + seed.getName() + "' from " + seed.getPublicAddress() + "; nowUTC0=" + nowUTC0Time + ", seedUTC0=" + ctimeUTC0 + ", TimeDiff=" + serverDate.intervalToString(Math.abs(nowUTC0Time - ctimeUTC0))); return false; } @@ -349,13 +349,13 @@ public class yacyPeerActions { if (!direct) { if (ctimeUTC0 < dtimeUTC0) { // the disconnection was later, we reject the connection - yacyCore.log.logFine("connect: rejecting disconnected peer '" + seed.getName() + "' from " + seed.getAddress()); + yacyCore.log.logFine("connect: rejecting disconnected peer '" + seed.getName() + "' from " + seed.getPublicAddress()); return false; } } // this is a return of a lost peer - yacyCore.log.logFine("connect: returned KNOWN " + peerType + " peer '" + seed.getName() + "' from " + seed.getAddress()); + yacyCore.log.logFine("connect: returned KNOWN " + peerType + " peer '" + seed.getName() + "' from " + seed.getPublicAddress()); this.seedDB.addConnected(seed); return true; } else { @@ -374,10 +374,10 @@ public class yacyPeerActions { // TODO: update seed name lookup cache } } catch (NumberFormatException e) { - yacyCore.log.logFine("connect: rejecting wrong peer '" + seed.getName() + "' from " + seed.getAddress() + ". Cause: " + e.getMessage()); + yacyCore.log.logFine("connect: rejecting wrong peer '" + seed.getName() + "' from " + seed.getPublicAddress() + ". Cause: " + e.getMessage()); return false; } - yacyCore.log.logFine("connect: updated KNOWN " + ((direct) ? "direct " : "") + peerType + " peer '" + seed.getName() + "' from " + seed.getAddress()); + yacyCore.log.logFine("connect: updated KNOWN " + ((direct) ? "direct " : "") + peerType + " peer '" + seed.getName() + "' from " + seed.getPublicAddress()); seedDB.addConnected(seed); return true; } else { @@ -385,10 +385,10 @@ public class yacyPeerActions { if (seed.get(yacySeed.IP, "127.0.0.1").equals(this.seedDB.mySeed.get(yacySeed.IP, "127.0.0.1"))) { // seed from the same IP as the calling client: can be // the case if there runs another one over a NAT - yacyCore.log.logFine("connect: saved NEW seed (myself IP) " + seed.getAddress()); + yacyCore.log.logFine("connect: saved NEW seed (myself IP) " + seed.getPublicAddress()); } else { // completely new seed - yacyCore.log.logFine("connect: saved NEW " + peerType + " peer '" + seed.getName() + "' from " + seed.getAddress()); + yacyCore.log.logFine("connect: saved NEW " + peerType + " peer '" + seed.getName() + "' from " + seed.getPublicAddress()); } if (peerType.equals(yacySeed.PEERTYPE_SENIOR)) this.seniorConnects++; // update statistics @@ -402,7 +402,7 @@ public class yacyPeerActions { private final void disconnectPeer(yacySeed seed) { // we do this if we did not get contact with the other peer - yacyCore.log.logFine("connect: no contact to a " + seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN) + " peer '" + seed.getName() + "' at " + seed.getAddress()); + yacyCore.log.logFine("connect: no contact to a " + seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN) + " peer '" + seed.getName() + "' at " + seed.getPublicAddress()); synchronized (seedDB) { if (!seedDB.hasDisconnected(seed.hash)) { disconnects++; } seed.put("dct", Long.toString(System.currentTimeMillis())); diff --git a/source/de/anomic/yacy/yacySearch.java b/source/de/anomic/yacy/yacySearch.java index 3771fd520..e8f975371 100644 --- a/source/de/anomic/yacy/yacySearch.java +++ b/source/de/anomic/yacy/yacySearch.java @@ -50,7 +50,6 @@ import java.util.Iterator; import java.util.Map; import java.util.Set; import java.util.TreeMap; -import java.util.TreeSet; import de.anomic.index.indexContainer; import de.anomic.kelondro.kelondroBitfield; @@ -147,13 +146,18 @@ public class yacySearch extends Thread { return targetPeer; } - private static yacySeed[] selectClusterPeers(TreeSet peerhashes) { - Iterator i = peerhashes.iterator(); + private static yacySeed[] selectClusterPeers(TreeMap peerhashes) { + Iterator i = peerhashes.entrySet().iterator(); ArrayList l = new ArrayList(); + Map.Entry entry; yacySeed s; while (i.hasNext()) { - s = yacyCore.seedDB.get((String) i.next()); // should be getConnected; get only during testing time - if (s != null) l.add(s); + entry = (Map.Entry) i.next(); + s = yacyCore.seedDB.get((String) entry.getKey()); // should be getConnected; get only during testing time + if (s != null) { + s.setAlternativeAddress((String) entry.getValue()); + l.add(s); + } } yacySeed[] result = new yacySeed[l.size()]; for (int j = 0; j < l.size(); j++) { @@ -243,9 +247,9 @@ public class yacySearch extends Thread { indexContainer containerCache, Map abstractCache, int targets, plasmaURLPattern blacklist, plasmaSnippetCache snippetCache, plasmaSearchTimingProfile timingProfile, plasmaSearchRankingProfile rankingProfile, - kelondroBitfield constraint, TreeSet clusterselection) { + kelondroBitfield constraint, TreeMap clusterselection) { // check own peer status - if (yacyCore.seedDB.mySeed == null || yacyCore.seedDB.mySeed.getAddress() == null) { return null; } + if (yacyCore.seedDB.mySeed == null || yacyCore.seedDB.mySeed.getPublicAddress() == null) { return null; } // prepare seed targets and threads final yacySeed[] targetPeers = (clusterselection == null) ? selectDHTPeers(plasmaSearchQuery.hashes2Set(wordhashes), targets) : selectClusterPeers(clusterselection); @@ -269,7 +273,7 @@ public class yacySearch extends Thread { plasmaSearchTimingProfile timingProfile, plasmaSearchRankingProfile rankingProfile, kelondroBitfield constraint) { // check own peer status - if (yacyCore.seedDB.mySeed == null || yacyCore.seedDB.mySeed.getAddress() == null) { return null; } + if (yacyCore.seedDB.mySeed == null || yacyCore.seedDB.mySeed.getPublicAddress() == null) { return null; } // prepare seed targets and threads final yacySeed targetPeer = yacyCore.seedDB.getConnected(targethash); diff --git a/source/de/anomic/yacy/yacySeed.java b/source/de/anomic/yacy/yacySeed.java index e4d824227..5fe9caa0a 100644 --- a/source/de/anomic/yacy/yacySeed.java +++ b/source/de/anomic/yacy/yacySeed.java @@ -174,6 +174,7 @@ public class yacySeed { private final Map dna; public int available; public int selectscore = -1; // only for debugging + public String alternativeIP = null; public yacySeed(String theHash, Map theDna) { // create a seed with a pre-defined hash map @@ -245,6 +246,16 @@ public class yacySeed { public static boolean isDefaultPeerName(String name) { return name != null && name.length() > 10 && name.charAt(0) <= '9' && name.charAt(name.length() - 1) <= '9' && name.indexOf("dpn") > 0; } + + /** + * used when doing routing within a cluster; this can assign a ip and a port + * that is used instead the address stored in the seed DNA + */ + public void setAlternativeAddress(String ipport) { + if (ipport == null) return; + int p = ipport.indexOf(':'); + if (p < 0) this.alternativeIP = ipport; else this.alternativeIP = ipport.substring(0, p); + } /** * try to get the IP
@@ -376,7 +387,7 @@ public class yacySeed { } } - public final String getAddress() { + public final String getPublicAddress() { // returns an ip:port string String ip = (String) this.dna.get(yacySeed.IP); if (ip == null) { return null; } @@ -390,6 +401,22 @@ public class yacySeed { return ip + ":" + port; } + + public final String getClusterAddress() { + // if this seed is part of a cluster, the peer has probably the + // alternativeIP object set to a local ip + // if this is present and the public ip of this peer is identical to the public ip of the own seed, + // construct an address using this ip; othervise return the public address + if ((this.alternativeIP == null) || + (yacyCore.seedDB == null) || + (!(this.getIP().equals(yacyCore.seedDB.mySeed.getIP())))) return getPublicAddress(); + + final String port = (String) this.dna.get(yacySeed.PORT); + if (port == null) { return null; } + if (port.length() < 2) { return null; } + + return this.alternativeIP + ":" + port; + } public final InetAddress getInetAddress() { // returns the ip address diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index 9faf38bf8..968df698d 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -58,6 +58,7 @@ import java.util.HashMap; import java.util.Hashtable; import java.util.Iterator; import java.util.Map; +import java.util.TreeMap; import java.util.TreeSet; import de.anomic.http.httpHeader; @@ -248,36 +249,51 @@ public final class yacySeedDB { return new seedEnum(up, field, seedPotentialDB); } - public TreeSet /* peer-b64-hashes */ clusterHashes(String clusterdefinition) { + public TreeMap /* peer-b64-hashes/ipport */ clusterHashes(String clusterdefinition) { // collects seeds according to cluster definition string, which consists of // comma-separated .yacy or .yacyh-domains - String[] cluster = clusterdefinition.split(","); - TreeSet clusterset = new TreeSet(kelondroBase64Order.enhancedCoder); + // the domain may be extended by an alternative address specification of the form + // or :. The port must be identical to the port specified in the peer seed, + // therefore it is optional. The address specification is separated by a '='; the complete + // address has therefore the form + // address ::= ('.yacy'|'.yacyh'){'='{':'= 0) { + yacydom = addresses[i].substring(0, p); + ipport = addresses[i].substring(p + 1); + } else { + yacydom = addresses[i]; + ipport = null; + } + if (yacydom.endsWith(".yacyh")) { // find a peer with its hexhash - hash = yacySeed.hexHash2b64Hash(cluster[i].substring(0, cluster[i].length() - 6)); + hash = yacySeed.hexHash2b64Hash(yacydom.substring(0, yacydom.length() - 6)); seed = get(hash); if (seed == null) { - yacyCore.log.logWarning("cluster peer '" + cluster[i] + "' was not found."); + yacyCore.log.logWarning("cluster peer '" + yacydom + "' was not found."); } else { - clusterset.add(hash); + clustermap.put(hash, ipport); } - } else if (cluster[i].endsWith(".yacy")) { + } else if (yacydom.endsWith(".yacy")) { // find a peer with its name - seed = lookupByName(cluster[i].substring(0, cluster[i].length() - 5)); + seed = lookupByName(yacydom.substring(0, yacydom.length() - 5)); if (seed == null) { - yacyCore.log.logWarning("cluster peer '" + cluster[i] + "' was not found."); + yacyCore.log.logWarning("cluster peer '" + yacydom + "' was not found."); } else { - clusterset.add(seed.hash); + clustermap.put(seed.hash, ipport); } } else { - yacyCore.log.logWarning("cluster peer '" + cluster[i] + "' has wrong syntax. the name must end with .yacy or .yacyh"); + yacyCore.log.logWarning("cluster peer '" + addresses[i] + "' has wrong syntax. the name must end with .yacy or .yacyh"); } } - return clusterset; + return clustermap; } public Iterator /*of yacySeed*/ seedsInCluster(String firstHash, TreeSet clusterhashes) { @@ -594,7 +610,7 @@ public final class yacySeedDB { try { seed = (yacySeed) e.nextElement(); if (seed != null) { - addressStr = seed.getAddress(); + addressStr = seed.getPublicAddress(); if (addressStr == null) { serverLog.logWarning("YACY","lookupByIP: address of seed " + seed.getName() + " is null."); continue; @@ -618,7 +634,7 @@ public final class yacySeedDB { try { seed = (yacySeed) e.nextElement(); if (seed != null) { - addressStr = seed.getAddress(); + addressStr = seed.getPublicAddress(); if ((pos = addressStr.indexOf(":"))!= -1) { addressStr = addressStr.substring(0,pos); } @@ -637,7 +653,7 @@ public final class yacySeedDB { while (e.hasMoreElements()) { try { seed = (yacySeed) e.nextElement(); - if ((seed != null) && ((addressStr = seed.getAddress()) != null)) { + if ((seed != null) && ((addressStr = seed.getPublicAddress()) != null)) { if ((pos = addressStr.indexOf(":"))!= -1) { addressStr = addressStr.substring(0,pos); } @@ -651,7 +667,7 @@ public final class yacySeedDB { try { // check local seed - addressStr = mySeed.getAddress(); + addressStr = mySeed.getPublicAddress(); if ((pos = addressStr.indexOf(":"))!= -1) { addressStr = addressStr.substring(0,pos); } @@ -866,7 +882,7 @@ public final class yacySeedDB { seed = mySeed; else return null; } - return seed.getAddress() + ((subdom == null) ? "" : ("/" + subdom)); + return seed.getPublicAddress() + ((subdom == null) ? "" : ("/" + subdom)); } else if (host.endsWith(".yacy")) { // identify subdomain p = host.indexOf("."); @@ -882,7 +898,7 @@ public final class yacySeedDB { // take local ip instead of external return serverCore.publicIP() + ":" + serverCore.getPortNr(sb.getConfig("port", "8080")) + ((subdom == null) ? "" : ("/" + subdom)); } - return seed.getAddress() + ((subdom == null) ? "" : ("/" + subdom)); + return seed.getPublicAddress() + ((subdom == null) ? "" : ("/" + subdom)); } else { return null; }