enhanced cluster routing

- cluster definitions can now contain an addition for local ip addresses
- cluster-cluster communication uses the local ip address instead the global address, if one is given

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3624 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 18 years ago
parent 97d4ab2053
commit e48189c710

@ -84,7 +84,7 @@ public class Blog {
boolean hasRights = switchboard.verifyAuthentication(header, true);
final boolean xml = ((String)header.get(httpHeader.CONNECTION_PROP_PATH)).endsWith(".xml");
final String address = yacyCore.seedDB.mySeed.getAddress();
final String address = yacyCore.seedDB.mySeed.getPublicAddress();
if(hasRights) {
prop.put("mode_admin",1);

@ -87,7 +87,7 @@ public class Bookmarks {
prop.put("user",username);
// set peer address
final String address = yacyCore.seedDB.mySeed.getAddress();
final String address = yacyCore.seedDB.mySeed.getPublicAddress();
prop.put("address", address);
//defaultvalues

@ -177,7 +177,8 @@ public class ConfigNetwork_p {
String[] s = input.split(",");
input = "";
for (int i = 0; i < s.length; i++) {
if ((s[i].endsWith(".yacyh")) || (s[i].endsWith(".yacy"))) input += "," + s[i];
if ((s[i].endsWith(".yacyh")) || (s[i].endsWith(".yacy")) ||
(s[i].indexOf(".yacyh=") > 0) || (s[i].indexOf(".yacy=") > 0)) input += "," + s[i];
}
if (input.length() == 0) return input; else return input.substring(1);
}

@ -62,7 +62,7 @@ public class ConfigRobotsTxt_p {
final servletProperties prop = new servletProperties();
httpdRobotsTxtConfig rbc = ((plasmaSwitchboard)env).robotstxtConfig;
prop.put("clientname", yacyCore.seedDB.mySeed.getAddress());
prop.put("clientname", yacyCore.seedDB.mySeed.getPublicAddress());
if (post != null) {
if (post.containsKey("save")) {

@ -161,7 +161,7 @@ public class CrawlURLFetch_p {
ys = yacyCore.seedDB.get(post.get("peerhash", null));
if (ys != null) {
if ((url = URLFetcher.getListServletURL(
ys.getAddress(),
ys.getPublicAddress(),
URLFetcher.MODE_LIST,
count,
yacyCore.seedDB.mySeed.hash)) == null) {
@ -311,7 +311,7 @@ public class CrawlURLFetch_p {
private static int getURLs2Fetch(yacySeed seed, httpRemoteProxyConfig theRemoteProxyConfig) {
try {
String answer = new String(httpc.wget(
URLFetcher.getListServletURL(seed.getAddress(), URLFetcher.MODE_COUNT, 0, null),
URLFetcher.getListServletURL(seed.getPublicAddress(), URLFetcher.MODE_COUNT, 0, null),
seed.getIP(),
5000,
null, null,
@ -469,7 +469,7 @@ public class CrawlURLFetch_p {
}
if (ys == null) return null;
return getListServletURL(ys.getAddress(), MODE_LIST, this.count, yacyCore.seedDB.mySeed.hash);
return getListServletURL(ys.getPublicAddress(), MODE_LIST, this.count, yacyCore.seedDB.mySeed.hash);
}
private int stackURLs(String[] urls) throws InterruptedException {

@ -230,7 +230,7 @@ public class DetailedSearch {
final long searchtime = 1000 * Long.parseLong(post.get("localTime", "10"));
final boolean yacyonline = ((yacyCore.seedDB != null) &&
(yacyCore.seedDB.mySeed != null) &&
(yacyCore.seedDB.mySeed.getAddress() != null));
(yacyCore.seedDB.mySeed.getPublicAddress() != null));
String urlmask = "";
if (post.containsKey("urlmask") && post.get("urlmask").equals("no")) {

@ -76,7 +76,7 @@ public class Messages_p {
serverObjects prop = new serverObjects();
// set peer address / name
final String peerAddress = yacyCore.seedDB.mySeed.getAddress();
final String peerAddress = yacyCore.seedDB.mySeed.getPublicAddress();
final String peerName = yacyCore.seedDB.mySeed.getName();
prop.put("peerAddress", peerAddress);
prop.put("peerName", peerName);

@ -202,19 +202,19 @@ public class Network {
yacySeed peer = new yacySeed((String) post.get("peerHash"),map);
yacyCore.peerActions.updateMySeed();
final int added = yacyClient.publishMySeed(peer.getAddress(), peer.hash);
final int added = yacyClient.publishMySeed(peer.getPublicAddress(), peer.hash);
if (added <= 0) {
prop.put("table_comment",1);
prop.put("table_comment_status","publish: disconnected peer '" + peer.getName() + "/" + post.get("peerHash") + "' from " + peer.getAddress());
prop.put("table_comment_status","publish: disconnected peer '" + peer.getName() + "/" + post.get("peerHash") + "' from " + peer.getPublicAddress());
} else {
peer = yacyCore.seedDB.getConnected(peer.hash);
if (peer == null) {
prop.put("table_comment",1);
prop.put("table_comment_status","publish: disconnected peer '" + peer.getName() + "/" + post.get("peerHash") + "' from " + peer.getAddress());
prop.put("table_comment_status","publish: disconnected peer '" + peer.getName() + "/" + post.get("peerHash") + "' from " + peer.getPublicAddress());
} else {
prop.put("table_comment",2);
prop.put("table_comment_status","publish: handshaked " + peer.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + peer.getName() + "' at " + peer.getAddress());
prop.put("table_comment_status","publish: handshaked " + peer.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + peer.getName() + "' at " + peer.getPublicAddress());
prop.put("table_comment_details",peer.toString());
}
}
@ -352,7 +352,7 @@ public class Network {
} else {
prop.put(STR_TABLE_LIST + conCount + "_updatedWiki", 1);
prop.put(STR_TABLE_LIST + conCount + "_updatedWiki_page", (String) wikiMap.get("page"));
prop.put(STR_TABLE_LIST + conCount + "_updatedWiki_address", seed.getAddress());
prop.put(STR_TABLE_LIST + conCount + "_updatedWiki_address", seed.getPublicAddress());
}
if ((blogMap = (Map) updatedBlog.get(seed.hash)) == null) {
prop.put(STR_TABLE_LIST + conCount + "_updatedBlog", 0);
@ -360,7 +360,7 @@ public class Network {
prop.put(STR_TABLE_LIST + conCount + "_updatedBlog", 1);
prop.put(STR_TABLE_LIST + conCount + "_updatedBlog_page", (String) blogMap.get("page"));
prop.put(STR_TABLE_LIST + conCount + "_updatedBlog_subject", (String) blogMap.get("subject"));
prop.put(STR_TABLE_LIST + conCount + "_updatedBlog_address", seed.getAddress());
prop.put(STR_TABLE_LIST + conCount + "_updatedBlog_address", seed.getPublicAddress());
}
PPM = seed.getPPM();
QPM = seed.getQPM();

@ -184,11 +184,11 @@ public class News {
title = record.attribute("title", "");
description = record.attribute("url", "");
} else if (category.equals(yacyNewsPool.CATEGORY_WIKI_UPDATE)) {
link = (seed==null)?"":"http://" + seed.getAddress() + "/Wiki.html?page=" + record.attribute("page", "");
link = (seed==null)?"":"http://" + seed.getPublicAddress() + "/Wiki.html?page=" + record.attribute("page", "");
title = record.attribute("author", "Anonymous") + ": " + record.attribute("page", "");
description = "Wiki Update: " + record.attribute("description", "");
} else if (category.equals(yacyNewsPool.CATEGORY_BLOG_ADD)) {
link = (seed==null)?"":"http://" + seed.getAddress() + "/Blog.html?page=" + record.attribute("page", "");
link = (seed==null)?"":"http://" + seed.getPublicAddress() + "/Blog.html?page=" + record.attribute("page", "");
title = record.attribute("author", "Anonymous") + ": " + record.attribute("page", "");
description = "Blog Entry: " + record.attribute("subject", "");
} else {
@ -205,7 +205,7 @@ public class News {
}
// adding the peer address
prop.put("address",yacyCore.seedDB.mySeed.getAddress());
prop.put("address",yacyCore.seedDB.mySeed.getPublicAddress());
// return rewrite properties
return prop;

@ -224,13 +224,13 @@ public class Status {
prop.put("peerStatistics_principalConnects", yacyCore.peerActions.principalConnects);
prop.put("peerStatistics_disconnects", yacyCore.peerActions.disconnects);
prop.put("peerStatistics_connects", yacyCore.seedDB.mySeed.get(yacySeed.CCOUNT, "0"));
if (yacyCore.seedDB.mySeed.getAddress() == null) {
if (yacyCore.seedDB.mySeed.getPublicAddress() == null) {
thisHash = yacyCore.seedDB.mySeed.hash;
prop.put("peerAddress", 1); // not assigned + instructions
} else {
thisHash = yacyCore.seedDB.mySeed.hash;
prop.put("peerAddress", 2); // Address
prop.put("peerAddress_address", yacyCore.seedDB.mySeed.getAddress());
prop.put("peerAddress_address", yacyCore.seedDB.mySeed.getPublicAddress());
prop.put("peerAddress_peername", env.getConfig("peerName", "<nameless>").toLowerCase());
}
}

@ -273,7 +273,7 @@ public class Surftips {
yacySeed seed = yacyCore.seedDB.getConnected(record.originator());
if (seed == null) seed = yacyCore.seedDB.getDisconnected(record.originator());
if (seed != null) {
url = "http://" + seed.getAddress() + "/Wiki.html?page=" + record.attribute("page", "");
url = "http://" + seed.getPublicAddress() + "/Wiki.html?page=" + record.attribute("page", "");
entry = rowdef.newEntry(new byte[][]{
url.getBytes(),
(record.attribute("author", "Anonymous") + ": " + record.attribute("page", "")).getBytes("UTF-8"),
@ -288,7 +288,7 @@ public class Surftips {
yacySeed seed = yacyCore.seedDB.getConnected(record.originator());
if (seed == null) seed = yacyCore.seedDB.getDisconnected(record.originator());
if (seed != null) {
url = "http://" + seed.getAddress() + "/Blog.html?page=" + record.attribute("page", "");
url = "http://" + seed.getPublicAddress() + "/Blog.html?page=" + record.attribute("page", "");
entry = rowdef.newEntry(new byte[][]{
url.getBytes(),
(record.attribute("author", "Anonymous") + ": " + record.attribute("page", "")).getBytes("UTF-8"),

@ -105,7 +105,7 @@ public class dir {
// general settings
prop.put("peername", env.getConfig("peerName", "<nameless>"));
prop.put("peerdomain", env.getConfig("peerName", "<nameless>").toLowerCase());
prop.put("peeraddress", yacyCore.seedDB.mySeed.getAddress());
prop.put("peeraddress", yacyCore.seedDB.mySeed.getPublicAddress());
prop.put("hostname", serverCore.publicIP());
try{
prop.put("hostip", InetAddress.getByName(serverCore.publicIP()).getHostAddress());
@ -288,7 +288,7 @@ public class dir {
if (f.isDirectory()) {
// the entry is a directory
prop.put("dirlist_" + fileIdx + "_dir" , 1);
prop.putSafeXML("dirlist_" + fileIdx + "_dir_URL","http://" + yacyCore.seedDB.mySeed.getAddress() + path + fileName + "/");
prop.putSafeXML("dirlist_" + fileIdx + "_dir_URL","http://" + yacyCore.seedDB.mySeed.getPublicAddress() + path + fileName + "/");
} else {
// determine if we should display the description string or a preview image
boolean showImage = /* (description.length() == 0) && */ (fileName.endsWith(".jpg") || fileName.endsWith(".gif") || fileName.endsWith(".png"));
@ -300,7 +300,7 @@ public class dir {
prop.put("dirlist_" + fileIdx + "_dir_sizeBytes" , Long.toString(f.length()));
// the unique url
prop.putSafeXML("dirlist_" + fileIdx + "_dir_yacyhURL",yacyhURL(yacyCore.seedDB.mySeed, fileName, md5s));
prop.putSafeXML("dirlist_" + fileIdx + "_dir_URL","http://" + yacyCore.seedDB.mySeed.getAddress() + path + fileName);
prop.putSafeXML("dirlist_" + fileIdx + "_dir_URL","http://" + yacyCore.seedDB.mySeed.getPublicAddress() + path + fileName);
// the md5 sum of the file
prop.put("dirlist_" + fileIdx + "_dir_md5s",md5s);
// description mode: 0...image preview, 1...description text

@ -68,7 +68,7 @@ public class welcome {
prop.put("peername", env.getConfig("peerName", "<nameless>"));
prop.put("peerdomain", env.getConfig("peerName", "<nameless>").toLowerCase());
prop.put("peeraddress", yacyCore.seedDB.mySeed.getAddress());
prop.put("peeraddress", yacyCore.seedDB.mySeed.getPublicAddress());
prop.put("hostname", serverCore.publicIP());
try{
prop.put("hostip", InetAddress.getByName(serverCore.publicIP()).getHostAddress());

@ -91,7 +91,7 @@ public final class transfer {
// reject unknown peers: this does not appear fair, but anonymous senders are dangerous
// reject paths that contain '..' because they are dangerous
if (otherseed == null) sb.getLog().logFine("RankingTransmission: rejected unknown peer '" + otherpeer + "', current IP " + header.get("CLIENTIP", "unknown"));
if (filename.indexOf("..") >= 0) sb.getLog().logFine("RankingTransmission: rejected wrong path '" + filename + "' from peer " + otherseed.getName() + "/" + otherseed.getAddress()+ ", current IP " + header.get("CLIENTIP", "unknown"));
if (filename.indexOf("..") >= 0) sb.getLog().logFine("RankingTransmission: rejected wrong path '" + filename + "' from peer " + otherseed.getName() + "/" + otherseed.getPublicAddress()+ ", current IP " + header.get("CLIENTIP", "unknown"));
return prop;
}
@ -106,7 +106,7 @@ public final class transfer {
String access = kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(otherpeer + ":" + filename)) + ":" + kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw("" + System.currentTimeMillis()));
prop.putASIS("response", "ok");
prop.putASIS("process_access", access);
prop.putASIS("process_address", yacyCore.seedDB.mySeed.getAddress());
prop.putASIS("process_address", yacyCore.seedDB.mySeed.getPublicAddress());
prop.putASIS("process_protocol", "http");
prop.putASIS("process_path", ""); // currently empty; the store process will find a path
prop.putASIS("process_maxsize", "-1"); // if response is too big we return the size of the file

@ -258,7 +258,7 @@ public class yacysearch {
}
// prepare search properties
final boolean yacyonline = ((yacyCore.seedDB != null) && (yacyCore.seedDB.mySeed != null) && (yacyCore.seedDB.mySeed.getAddress() != null));
final boolean yacyonline = ((yacyCore.seedDB != null) && (yacyCore.seedDB.mySeed != null) && (yacyCore.seedDB.mySeed.getPublicAddress() != null));
final boolean samesearch = env.getConfig("last-search", "").equals(querystring + contentdomString);
final boolean globalsearch = (global) && (yacyonline) && (!samesearch);

@ -68,7 +68,7 @@ public class wikiParser {
tokens = new Token[] {
new SimpleToken('=', '=', new String[][] { null, { "h2" }, { "h3" }, { "h4" } }, true),
new SimpleToken('\'', '\'', new String[][] { null, { "i" }, { "b" }, null, { "b", "i" } }, false),
new LinkToken(yacyCore.seedDB.mySeed.getAddress(), "Wiki.html?page=", sb),
new LinkToken(yacyCore.seedDB.mySeed.getPublicAddress(), "Wiki.html?page=", sb),
new ListToken('*', "ul"),
new ListToken('#', "ol"),
new ListToken(':', "blockquote", null),

@ -750,7 +750,7 @@ public class wikiCode {
// or an image DATA/HTDOCS/grafics/kaskelix.jpg with [[Image:grafics/kaskelix.jpg]]
// you are free to use other sub-paths of DATA/HTDOCS
if (kl.indexOf("://")<1) {
kl = "http://" + yacyCore.seedDB.mySeed.getAddress().trim() + "/" + kl;
kl = "http://" + yacyCore.seedDB.mySeed.getPublicAddress().trim() + "/" + kl;
}
result = result.substring(0, p0) + "<img src=\"" + kl + "\"" + align + alt + ">" + result.substring(p1 + 2);
@ -791,7 +791,7 @@ public class wikiCode {
// or a file DATA/HTDOCS/www/page.html with [www/page.html]
// you are free to use other sub-paths of DATA/HTDOCS
if (kl.indexOf("://")<1) {
kl = "http://" + yacyCore.seedDB.mySeed.getAddress().trim() + "/" + kl;
kl = "http://" + yacyCore.seedDB.mySeed.getPublicAddress().trim() + "/" + kl;
}
result = result.substring(0, p0) + "<a class=\"extern\" href=\"" + kl + "\">" + kv + "</a>" + result.substring(p1 + 1);
}

@ -236,7 +236,7 @@ public class plasmaDHTTransfer extends Thread {
return;
// doing a peer ping to the remote seed
int added = yacyClient.publishMySeed(this.seed.getAddress(), this.seed.hash);
int added = yacyClient.publishMySeed(this.seed.getPublicAddress(), this.seed.hash);
if (added < 0) {
// inc. retry counter
retryCount++;

@ -175,7 +175,7 @@ public final class plasmaRankingDistribution {
target = yacyCore.seedDB.anySeedVersion(yacyVersion.YACY_ACCEPTS_RANKING_TRANSMISSION);
if (target == null) continue;
String targetaddress = target.getAddress();
String targetaddress = target.getPublicAddress();
if (transferRankingAddress(crfile, targetaddress)) return true;
}
return false;

@ -48,7 +48,6 @@ import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import de.anomic.index.indexContainer;
import de.anomic.index.indexRWIEntry;
@ -79,7 +78,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
private yacySearch[] primarySearchThreads, secondarySearchThreads;
private long searchtime;
private int searchcount;
private TreeSet preselectedPeerHashes;
private TreeMap preselectedPeerHashes;
public plasmaSearchEvent(plasmaSearchQuery query,
plasmaSearchRankingProfile ranking,
@ -90,7 +89,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
plasmaWordIndex wordIndex,
plasmaCrawlLURL urlStore,
plasmaSnippetCache snippetCache,
TreeSet preselectedPeerHashes) {
TreeMap preselectedPeerHashes) {
this.log = log;
this.wordIndex = wordIndex;
this.query = query;

@ -118,6 +118,7 @@ import java.util.Hashtable;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import de.anomic.data.blogBoard;
@ -251,7 +252,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
public double lastrequestedQueries = 0d;
public int totalPPM = 0;
public double totalQPM = 0d;
public TreeSet clusterhashes;
public TreeMap clusterhashes; // map of peerhash(String)/alternative-local-address as ip:port or only ip (String) or null if address in seed should be used
/*
* Remote Proxy configuration
@ -864,6 +865,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
public plasmaSwitchboard(String rootPath, String initPath, String configPath) {
super(rootPath, initPath, configPath);
sb=this;
// set loglevel and log
setLog(new serverLog("PLASMA"));
@ -1276,7 +1278,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// init robinson cluster
this.clusterhashes = yacyCore.seedDB.clusterHashes(getConfig("cluster.peers.yacydomain", ""));
sb=this;
log.logConfig("Finished Switchboard Initialization");
}
@ -1355,7 +1356,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
return network.indexOf(peer) >= 0;
} else if (clustermode.equals("publiccluster")) {
// check if we got the request from a peer in the public cluster
return this.clusterhashes.contains(peer);
return this.clusterhashes.containsKey(peer);
} else {
return false;
}
@ -1370,10 +1371,10 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
if (clustermode.equals("privatecluster")) {
// check if we got the request from a peer in the private cluster
String network = getConfig("cluster.peers.ipport", "");
return network.indexOf(seed.getAddress()) >= 0;
return network.indexOf(seed.getPublicAddress()) >= 0;
} else if (clustermode.equals("publiccluster")) {
// check if we got the request from a peer in the public cluster
return this.clusterhashes.contains(seed.hash);
return this.clusterhashes.containsKey(seed.hash);
} else {
return false;
}
@ -2830,7 +2831,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
hash = yacySeed.hexHash2b64Hash(host.substring(p + 1, host.length() - 6));
seed = yacyCore.seedDB.getConnected(hash);
filename = comp.url().getFile();
if ((seed == null) || ((address = seed.getAddress()) == null)) {
if ((seed == null) || ((address = seed.getPublicAddress()) == null)) {
// seed is not known from here
wordIndex.removeWordReferences(plasmaCondenser.getWords(("yacyshare " + filename.replace('?', ' ') + " " + comp.title()).getBytes(), "UTF-8").keySet(), urlentry.hash());
wordIndex.loadedURL.remove(urlentry.hash()); // clean up

@ -255,7 +255,7 @@ public final class yacyClient {
// sending request
final HashMap result = nxTools.table(
httpc.wget(
new URL("http://" + target.getAddress() +
new URL("http://" + target.getClusterAddress() +
"/yacy/query.html" +
"?iam=" + yacyCore.seedDB.mySeed.hash +
"&youare=" + target.hash +
@ -291,7 +291,7 @@ public final class yacyClient {
// sending request
final HashMap result = nxTools.table(
httpc.wget(
new URL("http://" + target.getAddress() +
new URL("http://" + target.getClusterAddress() +
"/yacy/query.html" +
"?iam=" + yacyCore.seedDB.mySeed.hash +
"&youare=" + target.hash +
@ -327,7 +327,7 @@ public final class yacyClient {
// building url
final String querystr =
"http://" + target.getAddress() +
"http://" + target.getClusterAddress() +
"/yacy/query.html" +
"?iam=" + yacyCore.seedDB.mySeed.hash +
"&youare=" + target.hash +
@ -410,7 +410,7 @@ public final class yacyClient {
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
// building url
final String url = "http://" + ((targetPeer.getIP().equals(yacyCore.seedDB.mySeed.getIP())) ? "localhost:" + targetPeer.getPort() : targetPeer.getAddress()) + "/yacy/search.html";
final String url = "http://" + targetPeer.getClusterAddress() + "/yacy/search.html";
// adding all needed parameters
/*
@ -669,17 +669,16 @@ public final class yacyClient {
return null;
}
}
public static String targetAddress(String targetHash) {
// find target address
String address;
if (targetHash.equals(yacyCore.seedDB.mySeed.hash)) {
address = yacyCore.seedDB.mySeed.getAddress();
address = yacyCore.seedDB.mySeed.getClusterAddress();
} else {
final yacySeed targetSeed = yacyCore.seedDB.getConnected(targetHash);
if (targetSeed == null) { return null; }
address = targetSeed.getAddress();
address = targetSeed.getClusterAddress();
}
if (address == null) address = "localhost:8080";
return address;
@ -827,7 +826,7 @@ public final class yacyClient {
post.put("ttl", "0");
// determining target address
final String address = targetSeed.getAddress();
final String address = targetSeed.getClusterAddress();
if (address == null) { return null; }
// sending request
@ -892,7 +891,7 @@ public final class yacyClient {
final String key = crypt.randomSalt();
// determining target address
String address = targetSeed.getAddress();
String address = targetSeed.getClusterAddress();
if (address == null) { return null; }
// sending request
@ -1015,7 +1014,7 @@ public final class yacyClient {
}
private static HashMap transferRWI(yacySeed targetSeed, indexContainer[] indexes, boolean gzipBody, int timeout) {
final String address = targetSeed.getAddress();
final String address = targetSeed.getPublicAddress();
if (address == null) { return null; }
// should we use the proxy?
@ -1091,7 +1090,7 @@ public final class yacyClient {
private static HashMap transferURL(yacySeed targetSeed, indexURLEntry[] urls, boolean gzipBody, int timeout) {
// this post a message to the remote message board
final String address = targetSeed.getAddress();
final String address = targetSeed.getPublicAddress();
if (address == null) { return null; }
// should we use the proxy?
@ -1163,7 +1162,7 @@ public final class yacyClient {
final serverObjects post = new serverObjects(2);
post.put("iam", yacyCore.seedDB.mySeed.hash);
post.put("youare", targetSeed.hash);
String address = targetSeed.getAddress();
String address = targetSeed.getClusterAddress();
if (address == null) { address = "localhost:8080"; }
try {
return nxTools.table(
@ -1200,7 +1199,7 @@ public final class yacyClient {
final HashMap result = nxTools.table(
httpc.wget(
new URL("http://" + target.getAddress() + "/yacy/search.html" +
new URL("http://" + target.getPublicAddress() + "/yacy/search.html" +
"?myseed=" + yacyCore.seedDB.mySeed.genSeedStr(null) +
"&youare=" + target.hash + "&key=" +
"&myseed=" + yacyCore.seedDB.mySeed.genSeedStr(null) +

@ -358,9 +358,9 @@ public class yacyCore {
return true;
}
log.logInfo("re-connect own seed");
final String oldAddress = seedDB.mySeed.getAddress();
final String oldAddress = seedDB.mySeed.getPublicAddress();
/*final int newSeeds =*/ publishMySeed(true);
return (oldAddress != null && oldAddress.equals(seedDB.mySeed.getAddress()));
return (oldAddress != null && oldAddress.equals(seedDB.mySeed.getPublicAddress()));
}
protected class publishThread extends Thread {
@ -384,15 +384,15 @@ public class yacyCore {
public void run() {
try {
this.added = yacyClient.publishMySeed(seed.getAddress(), seed.hash);
this.added = yacyClient.publishMySeed(seed.getPublicAddress(), seed.hash);
if (this.added < 0) {
// no or wrong response, delete that address
log.logInfo("publish: disconnected " + this.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' from " + this.seed.getAddress());
log.logInfo("publish: disconnected " + this.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' from " + this.seed.getPublicAddress());
peerActions.peerDeparture(this.seed);
} else {
// success! we have published our peer to a senior peer
// update latest news from the other peer
log.logInfo("publish: handshaked " + this.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getAddress());
log.logInfo("publish: handshaked " + this.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getPublicAddress());
}
} catch (Exception e) {
log.logSevere("publishThread: error with target seed " + seed.toString() + ": " + e.getMessage(), e);
@ -433,12 +433,17 @@ public class yacyCore {
seeds = seedDB.seedsByAge(true, attempts + 10); // best for fast connection
// add also all peers from cluster if this is a public robinson cluster
if (plasmaSwitchboard.getSwitchboard().clusterhashes != null) {
Iterator i = plasmaSwitchboard.getSwitchboard().clusterhashes.iterator();
Iterator i = plasmaSwitchboard.getSwitchboard().clusterhashes.entrySet().iterator();
String hash;
Map.Entry entry;
yacySeed seed;
while (i.hasNext()) {
hash = (String) i.next();
entry = (Map.Entry) i.next();
hash = (String) entry.getKey();
if (seeds.containsKey(hash)) continue;
seeds.put(hash, seedDB.get(hash));
seed = seedDB.get(hash);
seed.setAlternativeAddress((String) entry.getValue());
seeds.put(hash, seed);
}
}
} else {
@ -499,7 +504,7 @@ public class yacyCore {
seed = (yacySeed) seedList.remove(0);
if (seed == null) continue;
final String address = seed.getAddress();
final String address = seed.getPublicAddress();
log.logFine("HELLO #" + i + " to peer '" + seed.get(yacySeed.NAME, "") + "' at " + address); // debug
if ((address == null) || (seed.isProper() != null)) {
// we don't like that address, delete it
@ -542,20 +547,21 @@ public class yacyCore {
while ((newSeeds < 0) && (contactedSeedCount < peerPingInitial) && (!seedList.isEmpty())) {
seed = (yacySeed) seedList.remove(0);
if (seed != null) {
final String address = seed.getAddress();
String address = seed.getPublicAddress();
log.logFine("HELLO x" + contactedSeedCount + " to peer '" + seed.get(yacySeed.NAME, "") + "' at " + address); // debug
if ((address == null) || (seed.isProper() != null)) {
peerActions.peerDeparture(seed);
} else {
if (seed.alternativeIP != null) address = seed.alternativeIP + ":" + seed.getPort();
contactedSeedCount++;
//new publishThread(yacyCore.publishThreadGroup,seeds[i],sync,syncList)).start();
try {
newSeeds = yacyClient.publishMySeed(seed.getAddress(), seed.hash);
newSeeds = yacyClient.publishMySeed(address, seed.hash);
if (newSeeds < 0) {
log.logInfo("publish: disconnected " + seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + seed.getName() + "' from " + seed.getAddress());
log.logInfo("publish: disconnected " + seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + seed.getName() + "' from " + seed.getPublicAddress());
peerActions.peerDeparture(seed);
} else {
log.logInfo("publish: handshaked " + seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + seed.getName() + "' at " + seed.getAddress());
log.logInfo("publish: handshaked " + seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + seed.getName() + "' at " + seed.getPublicAddress());
}
} catch (Exception e) {
log.logSevere("publishMySeed: error with target seed " + seed.toString() + ": " + e.getMessage(), e);
@ -632,7 +638,7 @@ public class yacyCore {
if (seedDB.mySeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR).equals(yacySeed.PEERTYPE_JUNIOR)) // ???????????????
seedDB.mySeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); // to start bootstraping, we need to be recognised as PEERTYPE_SENIOR peer
log.logInfo("publish: no recipient found, our address is " +
((seedDB.mySeed.getAddress() == null) ? "unknown" : seedDB.mySeed.getAddress()));
((seedDB.mySeed.getPublicAddress() == null) ? "unknown" : seedDB.mySeed.getPublicAddress()));
peerActions.saveMySeed();
return 0;
} catch (InterruptedException e) {
@ -782,7 +788,7 @@ public class yacyCore {
String logt;
// be shure that we have something to say
if (seedDB.mySeed.getAddress() == null) {
if (seedDB.mySeed.getPublicAddress() == null) {
final String errorMsg = "We have no valid IP address until now";
log.logWarning("SaveSeedList: " + errorMsg);
return errorMsg;

@ -46,6 +46,7 @@ package de.anomic.yacy;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.Hashtable;
import java.util.TreeMap;
import java.util.TreeSet;
import de.anomic.kelondro.kelondroBase64Order;
@ -217,10 +218,12 @@ public class yacyDHTAction implements yacyPeerAction {
return seed;
}
public synchronized yacySeed getPublicClusterCrawlSeed(String urlHash, TreeSet clusterhashes) {
kelondroCloneableIterator i = new kelondroRotateIterator(new kelondroCloneableSetIterator(clusterhashes, urlHash), null);
public synchronized yacySeed getPublicClusterCrawlSeed(String urlHash, TreeMap clusterhashes) {
kelondroCloneableIterator i = new kelondroRotateIterator(new kelondroCloneableSetIterator((TreeSet) clusterhashes.keySet(), urlHash), null);
if (i.hasNext()) {
return seedDB.getConnected((String) i.next());
yacySeed seed = seedDB.getConnected((String) i.next());
seed.setAlternativeAddress((String) clusterhashes.get(seed.hash));
return seed;
}
return null;
}

@ -276,7 +276,7 @@ public class yacyPeerActions {
return false;
}
if ((this.seedDB.mySeed != null) && (seed.hash.equals(this.seedDB.mySeed.hash))) {
yacyCore.log.logInfo("connect: SELF reference " + seed.getAddress());
yacyCore.log.logInfo("connect: SELF reference " + seed.getPublicAddress());
return false;
}
final String peerType = seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN);
@ -312,7 +312,7 @@ public class yacyPeerActions {
}
if (Math.abs(nowUTC0Time - ctimeUTC0) > 60 * 60 * 24 * 1000) {
// the new connection is out-of-age, we reject the connection
yacyCore.log.logFine("connect: rejecting out-dated peer '" + seed.getName() + "' from " + seed.getAddress() + "; nowUTC0=" + nowUTC0Time + ", seedUTC0=" + ctimeUTC0 + ", TimeDiff=" + serverDate.intervalToString(Math.abs(nowUTC0Time - ctimeUTC0)));
yacyCore.log.logFine("connect: rejecting out-dated peer '" + seed.getName() + "' from " + seed.getPublicAddress() + "; nowUTC0=" + nowUTC0Time + ", seedUTC0=" + ctimeUTC0 + ", TimeDiff=" + serverDate.intervalToString(Math.abs(nowUTC0Time - ctimeUTC0)));
return false;
}
@ -349,13 +349,13 @@ public class yacyPeerActions {
if (!direct) {
if (ctimeUTC0 < dtimeUTC0) {
// the disconnection was later, we reject the connection
yacyCore.log.logFine("connect: rejecting disconnected peer '" + seed.getName() + "' from " + seed.getAddress());
yacyCore.log.logFine("connect: rejecting disconnected peer '" + seed.getName() + "' from " + seed.getPublicAddress());
return false;
}
}
// this is a return of a lost peer
yacyCore.log.logFine("connect: returned KNOWN " + peerType + " peer '" + seed.getName() + "' from " + seed.getAddress());
yacyCore.log.logFine("connect: returned KNOWN " + peerType + " peer '" + seed.getName() + "' from " + seed.getPublicAddress());
this.seedDB.addConnected(seed);
return true;
} else {
@ -374,10 +374,10 @@ public class yacyPeerActions {
// TODO: update seed name lookup cache
}
} catch (NumberFormatException e) {
yacyCore.log.logFine("connect: rejecting wrong peer '" + seed.getName() + "' from " + seed.getAddress() + ". Cause: " + e.getMessage());
yacyCore.log.logFine("connect: rejecting wrong peer '" + seed.getName() + "' from " + seed.getPublicAddress() + ". Cause: " + e.getMessage());
return false;
}
yacyCore.log.logFine("connect: updated KNOWN " + ((direct) ? "direct " : "") + peerType + " peer '" + seed.getName() + "' from " + seed.getAddress());
yacyCore.log.logFine("connect: updated KNOWN " + ((direct) ? "direct " : "") + peerType + " peer '" + seed.getName() + "' from " + seed.getPublicAddress());
seedDB.addConnected(seed);
return true;
} else {
@ -385,10 +385,10 @@ public class yacyPeerActions {
if (seed.get(yacySeed.IP, "127.0.0.1").equals(this.seedDB.mySeed.get(yacySeed.IP, "127.0.0.1"))) {
// seed from the same IP as the calling client: can be
// the case if there runs another one over a NAT
yacyCore.log.logFine("connect: saved NEW seed (myself IP) " + seed.getAddress());
yacyCore.log.logFine("connect: saved NEW seed (myself IP) " + seed.getPublicAddress());
} else {
// completely new seed
yacyCore.log.logFine("connect: saved NEW " + peerType + " peer '" + seed.getName() + "' from " + seed.getAddress());
yacyCore.log.logFine("connect: saved NEW " + peerType + " peer '" + seed.getName() + "' from " + seed.getPublicAddress());
}
if (peerType.equals(yacySeed.PEERTYPE_SENIOR))
this.seniorConnects++; // update statistics
@ -402,7 +402,7 @@ public class yacyPeerActions {
private final void disconnectPeer(yacySeed seed) {
// we do this if we did not get contact with the other peer
yacyCore.log.logFine("connect: no contact to a " + seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN) + " peer '" + seed.getName() + "' at " + seed.getAddress());
yacyCore.log.logFine("connect: no contact to a " + seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN) + " peer '" + seed.getName() + "' at " + seed.getPublicAddress());
synchronized (seedDB) {
if (!seedDB.hasDisconnected(seed.hash)) { disconnects++; }
seed.put("dct", Long.toString(System.currentTimeMillis()));

@ -50,7 +50,6 @@ import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import de.anomic.index.indexContainer;
import de.anomic.kelondro.kelondroBitfield;
@ -147,13 +146,18 @@ public class yacySearch extends Thread {
return targetPeer;
}
private static yacySeed[] selectClusterPeers(TreeSet peerhashes) {
Iterator i = peerhashes.iterator();
private static yacySeed[] selectClusterPeers(TreeMap peerhashes) {
Iterator i = peerhashes.entrySet().iterator();
ArrayList l = new ArrayList();
Map.Entry entry;
yacySeed s;
while (i.hasNext()) {
s = yacyCore.seedDB.get((String) i.next()); // should be getConnected; get only during testing time
if (s != null) l.add(s);
entry = (Map.Entry) i.next();
s = yacyCore.seedDB.get((String) entry.getKey()); // should be getConnected; get only during testing time
if (s != null) {
s.setAlternativeAddress((String) entry.getValue());
l.add(s);
}
}
yacySeed[] result = new yacySeed[l.size()];
for (int j = 0; j < l.size(); j++) {
@ -243,9 +247,9 @@ public class yacySearch extends Thread {
indexContainer containerCache, Map abstractCache,
int targets, plasmaURLPattern blacklist, plasmaSnippetCache snippetCache,
plasmaSearchTimingProfile timingProfile, plasmaSearchRankingProfile rankingProfile,
kelondroBitfield constraint, TreeSet clusterselection) {
kelondroBitfield constraint, TreeMap clusterselection) {
// check own peer status
if (yacyCore.seedDB.mySeed == null || yacyCore.seedDB.mySeed.getAddress() == null) { return null; }
if (yacyCore.seedDB.mySeed == null || yacyCore.seedDB.mySeed.getPublicAddress() == null) { return null; }
// prepare seed targets and threads
final yacySeed[] targetPeers = (clusterselection == null) ? selectDHTPeers(plasmaSearchQuery.hashes2Set(wordhashes), targets) : selectClusterPeers(clusterselection);
@ -269,7 +273,7 @@ public class yacySearch extends Thread {
plasmaSearchTimingProfile timingProfile, plasmaSearchRankingProfile rankingProfile,
kelondroBitfield constraint) {
// check own peer status
if (yacyCore.seedDB.mySeed == null || yacyCore.seedDB.mySeed.getAddress() == null) { return null; }
if (yacyCore.seedDB.mySeed == null || yacyCore.seedDB.mySeed.getPublicAddress() == null) { return null; }
// prepare seed targets and threads
final yacySeed targetPeer = yacyCore.seedDB.getConnected(targethash);

@ -174,6 +174,7 @@ public class yacySeed {
private final Map dna;
public int available;
public int selectscore = -1; // only for debugging
public String alternativeIP = null;
public yacySeed(String theHash, Map theDna) {
// create a seed with a pre-defined hash map
@ -245,6 +246,16 @@ public class yacySeed {
public static boolean isDefaultPeerName(String name) {
return name != null && name.length() > 10 && name.charAt(0) <= '9' && name.charAt(name.length() - 1) <= '9' && name.indexOf("dpn") > 0;
}
/**
* used when doing routing within a cluster; this can assign a ip and a port
* that is used instead the address stored in the seed DNA
*/
public void setAlternativeAddress(String ipport) {
if (ipport == null) return;
int p = ipport.indexOf(':');
if (p < 0) this.alternativeIP = ipport; else this.alternativeIP = ipport.substring(0, p);
}
/**
* try to get the IP<br>
@ -376,7 +387,7 @@ public class yacySeed {
}
}
public final String getAddress() {
public final String getPublicAddress() {
// returns an ip:port string
String ip = (String) this.dna.get(yacySeed.IP);
if (ip == null) { return null; }
@ -390,6 +401,22 @@ public class yacySeed {
return ip + ":" + port;
}
public final String getClusterAddress() {
// if this seed is part of a cluster, the peer has probably the
// alternativeIP object set to a local ip
// if this is present and the public ip of this peer is identical to the public ip of the own seed,
// construct an address using this ip; othervise return the public address
if ((this.alternativeIP == null) ||
(yacyCore.seedDB == null) ||
(!(this.getIP().equals(yacyCore.seedDB.mySeed.getIP())))) return getPublicAddress();
final String port = (String) this.dna.get(yacySeed.PORT);
if (port == null) { return null; }
if (port.length() < 2) { return null; }
return this.alternativeIP + ":" + port;
}
public final InetAddress getInetAddress() {
// returns the ip address

@ -58,6 +58,7 @@ import java.util.HashMap;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
import java.util.TreeSet;
import de.anomic.http.httpHeader;
@ -248,36 +249,51 @@ public final class yacySeedDB {
return new seedEnum(up, field, seedPotentialDB);
}
public TreeSet /* peer-b64-hashes */ clusterHashes(String clusterdefinition) {
public TreeMap /* peer-b64-hashes/ipport */ clusterHashes(String clusterdefinition) {
// collects seeds according to cluster definition string, which consists of
// comma-separated .yacy or .yacyh-domains
String[] cluster = clusterdefinition.split(",");
TreeSet clusterset = new TreeSet(kelondroBase64Order.enhancedCoder);
// the domain may be extended by an alternative address specification of the form
// <ip> or <ip>:<port>. The port must be identical to the port specified in the peer seed,
// therefore it is optional. The address specification is separated by a '='; the complete
// address has therefore the form
// address ::= (<peername>'.yacy'|<peerhexhash>'.yacyh'){'='<ip>{':'<port}}
// clusterdef ::= {address}{','address}*
String[] addresses = clusterdefinition.split(",");
TreeMap clustermap = new TreeMap(kelondroBase64Order.enhancedCoder);
yacySeed seed;
String hash;
for (int i = 0; i < cluster.length; i++) {
if (cluster[i].endsWith(".yacyh")) {
String hash, yacydom, ipport;
int p;
for (int i = 0; i < addresses.length; i++) {
p = addresses[i].indexOf('=');
if (p >= 0) {
yacydom = addresses[i].substring(0, p);
ipport = addresses[i].substring(p + 1);
} else {
yacydom = addresses[i];
ipport = null;
}
if (yacydom.endsWith(".yacyh")) {
// find a peer with its hexhash
hash = yacySeed.hexHash2b64Hash(cluster[i].substring(0, cluster[i].length() - 6));
hash = yacySeed.hexHash2b64Hash(yacydom.substring(0, yacydom.length() - 6));
seed = get(hash);
if (seed == null) {
yacyCore.log.logWarning("cluster peer '" + cluster[i] + "' was not found.");
yacyCore.log.logWarning("cluster peer '" + yacydom + "' was not found.");
} else {
clusterset.add(hash);
clustermap.put(hash, ipport);
}
} else if (cluster[i].endsWith(".yacy")) {
} else if (yacydom.endsWith(".yacy")) {
// find a peer with its name
seed = lookupByName(cluster[i].substring(0, cluster[i].length() - 5));
seed = lookupByName(yacydom.substring(0, yacydom.length() - 5));
if (seed == null) {
yacyCore.log.logWarning("cluster peer '" + cluster[i] + "' was not found.");
yacyCore.log.logWarning("cluster peer '" + yacydom + "' was not found.");
} else {
clusterset.add(seed.hash);
clustermap.put(seed.hash, ipport);
}
} else {
yacyCore.log.logWarning("cluster peer '" + cluster[i] + "' has wrong syntax. the name must end with .yacy or .yacyh");
yacyCore.log.logWarning("cluster peer '" + addresses[i] + "' has wrong syntax. the name must end with .yacy or .yacyh");
}
}
return clusterset;
return clustermap;
}
public Iterator /*of yacySeed*/ seedsInCluster(String firstHash, TreeSet clusterhashes) {
@ -594,7 +610,7 @@ public final class yacySeedDB {
try {
seed = (yacySeed) e.nextElement();
if (seed != null) {
addressStr = seed.getAddress();
addressStr = seed.getPublicAddress();
if (addressStr == null) {
serverLog.logWarning("YACY","lookupByIP: address of seed " + seed.getName() + " is null.");
continue;
@ -618,7 +634,7 @@ public final class yacySeedDB {
try {
seed = (yacySeed) e.nextElement();
if (seed != null) {
addressStr = seed.getAddress();
addressStr = seed.getPublicAddress();
if ((pos = addressStr.indexOf(":"))!= -1) {
addressStr = addressStr.substring(0,pos);
}
@ -637,7 +653,7 @@ public final class yacySeedDB {
while (e.hasMoreElements()) {
try {
seed = (yacySeed) e.nextElement();
if ((seed != null) && ((addressStr = seed.getAddress()) != null)) {
if ((seed != null) && ((addressStr = seed.getPublicAddress()) != null)) {
if ((pos = addressStr.indexOf(":"))!= -1) {
addressStr = addressStr.substring(0,pos);
}
@ -651,7 +667,7 @@ public final class yacySeedDB {
try {
// check local seed
addressStr = mySeed.getAddress();
addressStr = mySeed.getPublicAddress();
if ((pos = addressStr.indexOf(":"))!= -1) {
addressStr = addressStr.substring(0,pos);
}
@ -866,7 +882,7 @@ public final class yacySeedDB {
seed = mySeed;
else return null;
}
return seed.getAddress() + ((subdom == null) ? "" : ("/" + subdom));
return seed.getPublicAddress() + ((subdom == null) ? "" : ("/" + subdom));
} else if (host.endsWith(".yacy")) {
// identify subdomain
p = host.indexOf(".");
@ -882,7 +898,7 @@ public final class yacySeedDB {
// take local ip instead of external
return serverCore.publicIP() + ":" + serverCore.getPortNr(sb.getConfig("port", "8080")) + ((subdom == null) ? "" : ("/" + subdom));
}
return seed.getAddress() + ((subdom == null) ? "" : ("/" + subdom));
return seed.getPublicAddress() + ((subdom == null) ? "" : ("/" + subdom));
} else {
return null;
}

Loading…
Cancel
Save