boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

reverted changes to yacy protocol classes: they caused the sciencenet to loose connections

Browse Source 
a comparisment with the main release 0.57 had been made: this showed a stable network
This is an emergency operation to ensure availability of the sciencenet network.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4553 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent b664a53553
commit 5530b8e1ca
11 changed files with 302 additions and 370 deletions
Show Stats Download Patch File Download Diff File

60
htroot/yacy/crawlReceipt.java
Unescape View File

@ -62,34 +62,25 @@ import de.anomic.yacy.yacySeed;
public final class crawlReceipt { public final class crawlReceipt {
/* /*
* this is used to respond on a remote crawling request * this is used to respond on a remote crawling request
*/ */
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
// return variable that accumulates replacements // return variable that accumulates replacements
final serverObjects prop = new serverObjects(); plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
// defaults serverObjects prop = new serverObjects();
prop.put("delay", "3600"); if ((post == null) || (env == null)) return prop;
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) { serverLog log = switchboard.getLog();
return prop;
}
// seed hash of requester
String ohash = post.get("iam", "");
if (ohash == null || ohash.length() == 0 ) return prop;
yacySeed oseed = yacyCore.seedDB.get(ohash);
if (oseed == null) return prop;
oseed.setFlagDirectConnect(true);
oseed.setLastSeenUTC();
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverLog log = sb.getLog();
//int proxyPrefetchDepth = Integer.parseInt(env.getConfig("proxyPrefetchDepth", "0")); //int proxyPrefetchDepth = Integer.parseInt(env.getConfig("proxyPrefetchDepth", "0"));
//int crawlingDepth = Integer.parseInt(env.getConfig("crawlingDepth", "0")); //int crawlingDepth = Integer.parseInt(env.getConfig("crawlingDepth", "0"));
// request values // request values
String iam = post.get("iam", ""); // seed hash of requester
String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability
//String process = post.get("process", ""); // process type //String process = post.get("process", ""); // process type
String key = post.get("key", ""); // transmission key String key = post.get("key", ""); // transmission key
@ -120,50 +111,53 @@ public final class crawlReceipt {
*/ */
final yacySeed otherPeer = yacyCore.seedDB.get(iam);
final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));
if ((yacyCore.seedDB.mySeed() == null) || (!(yacyCore.seedDB.mySeed().hash.equals(youare)))) { if ((yacyCore.seedDB.mySeed() == null) || (!(yacyCore.seedDB.mySeed().hash.equals(youare)))) {
// no yacy connection / unknown peers // no yacy connection / unknown peers
prop.put("delay", "3600");
return prop; return prop;
} }
if (propStr == null) { if (propStr == null) {
// error with url / wrong key // error with url / wrong key
prop.put("delay", "3600");
return prop; return prop;
} }
if ((sb.isRobinsonMode()) && (!sb.isInMyCluster(oseed))) { if ((switchboard.isRobinsonMode()) && (!switchboard.isInMyCluster(otherPeer))) {
// we reject urls that are from outside our cluster // we reject urls that are from outside our cluster
prop.put("delay", "9999"); prop.put("delay", "9999");
return prop; // ???
} }
// generating a new loaded URL entry // generating a new loaded URL entry
indexURLEntry entry = sb.wordIndex.loadedURL.newEntry(propStr); indexURLEntry entry = switchboard.wordIndex.loadedURL.newEntry(propStr);
if (entry == null) { if (entry == null) {
log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (entry null) from peer " + ohash + "\n\tURL properties: "+ propStr); log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (entry null) from peer " + iam + "\n\tURL properties: "+ propStr);
prop.put("delay", "3600");
return prop; return prop;
} }
indexURLEntry.Components comp = entry.comp(); indexURLEntry.Components comp = entry.comp();
if (comp.url() == null) { if (comp.url() == null) {
log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (url null) for hash " + entry.hash() + " from peer " + ohash + "\n\tURL properties: "+ propStr); log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (url null) for hash " + entry.hash() + " from peer " + iam + "\n\tURL properties: "+ propStr);
prop.put("delay", "3600");
return prop; return prop;
} }
// check if the entry is in our network domain // check if the entry is in our network domain
if (!sb.acceptURL(comp.url())) { if (!switchboard.acceptURL(comp.url())) {
log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (url outside of our domain) for hash " + entry.hash() + " from peer " + ohash + "\n\tURL properties: "+ propStr); log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (url outside of our domain) for hash " + entry.hash() + " from peer " + iam + "\n\tURL properties: "+ propStr);
prop.put("delay", "9999"); prop.put("delay", "9999");
return prop; return prop;
} }
if (result.equals("fill")) try { if (result.equals("fill")) try {
// put new entry into database // put new entry into database
sb.wordIndex.loadedURL.store(entry); switchboard.wordIndex.loadedURL.store(entry);
sb.wordIndex.loadedURL.stack(entry, youare, ohash, 1); switchboard.wordIndex.loadedURL.stack(entry, youare, iam, 1);
sb.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work has been done switchboard.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work has been done
final String otherPeerName = ohash + ":" + oseed.getName() + "/" + oseed.getVersion();
log.logInfo("crawlReceipt: RECEIVED RECEIPT from " + otherPeerName + " for URL " + entry.hash() + ":" + comp.url().toNormalform(false, true)); log.logInfo("crawlReceipt: RECEIVED RECEIPT from " + otherPeerName + " for URL " + entry.hash() + ":" + comp.url().toNormalform(false, true));
// ready for more // ready for more
@ -171,14 +165,16 @@ public final class crawlReceipt {
return prop; return prop;
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
prop.put("delay", "3600");
return prop; return prop;
} }
sb.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work is transformed into an error case switchboard.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work is transformed into an error case
plasmaCrawlZURL.Entry ee = sb.crawlQueues.errorURL.newEntry(entry.toBalancerEntry(), youare, null, 0, result + ":" + reason); plasmaCrawlZURL.Entry ee = switchboard.crawlQueues.errorURL.newEntry(entry.toBalancerEntry(), youare, null, 0, result + ":" + reason);
ee.store(); ee.store();
sb.crawlQueues.errorURL.push(ee); switchboard.crawlQueues.errorURL.push(ee);
//switchboard.noticeURL.remove(receivedUrlhash); //switchboard.noticeURL.remove(receivedUrlhash);
prop.put("delay", "3600");
return prop; return prop;
// return rewrite properties // return rewrite properties

53
htroot/yacy/hello.java
Unescape View File

@ -65,9 +65,10 @@ import de.anomic.yacy.yacyVersion;
public final class hello { public final class hello {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws InterruptedException { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws InterruptedException {
// return variable that accumulates replacements plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects(); serverObjects prop = new serverObjects();
if (post == null || env == null) { prop.put("message", "none");
if ((post == null) || (env == null)) {
prop.put("message", "no post or no enviroment"); prop.put("message", "no post or no enviroment");
return prop; return prop;
} }
@ -75,7 +76,6 @@ public final class hello {
prop.put("message", "not in my network"); prop.put("message", "not in my network");
return prop; return prop;
} }
prop.put("message", "none");
// final String iam = (String) post.get("iam", ""); // complete seed of the requesting peer // final String iam = (String) post.get("iam", ""); // complete seed of the requesting peer
// final String mytime = (String) post.get(MYTIME, ""); // // final String mytime = (String) post.get(MYTIME, ""); //
@ -90,10 +90,10 @@ public final class hello {
prop.put("message", "your seed is too long (" + seed.length() + ")"); prop.put("message", "your seed is too long (" + seed.length() + ")");
return prop; return prop;
} }
final yacySeed oseed = yacySeed.genRemoteSeed(seed, key, false); final yacySeed remoteSeed = yacySeed.genRemoteSeed(seed, key, false);
// System.out.println("YACYHELLO: REMOTESEED=" + ((remoteSeed == null) ? "NULL" : remoteSeed.toString())); // System.out.println("YACYHELLO: REMOTESEED=" + ((remoteSeed == null) ? "NULL" : remoteSeed.toString()));
if (oseed == null || oseed.hash == null) { if ((remoteSeed == null) || (remoteSeed.hash == null)) {
prop.put("message", "cannot parse your seed"); prop.put("message", "cannot parse your seed");
return prop; return prop;
} }
@ -110,19 +110,18 @@ public final class hello {
return prop; return prop;
} }
final String userAgent = (String) header.get(httpHeader.USER_AGENT, "<unknown>"); final String userAgent = (String) header.get(httpHeader.USER_AGENT, "<unknown>");
final String reportedip = oseed.get(yacySeed.IP, ""); final String reportedip = remoteSeed.get(yacySeed.IP, "");
final String reportedPeerType = oseed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR); final String reportedPeerType = remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
final float clientversion = oseed.getVersion(); final float clientversion = remoteSeed.getVersion();
final plasmaSwitchboard sb = (plasmaSwitchboard) env; if ((sb.isRobinsonMode()) && (!sb.isPublicRobinson())) {
if (sb.isRobinsonMode() && !sb.isPublicRobinson()) { // if we are a robinson cluster, answer only if this client is known by our network definition
// if we are a robinson cluster, answer only if this client is known by our network definition
prop.put("message", "I am robinson, I do not answer"); prop.put("message", "I am robinson, I do not answer");
return prop; return prop;
} }
int urls = -1; int urls = -1;
if (sb.clusterhashes != null) oseed.setAlternativeAddress((String) sb.clusterhashes.get(oseed.hash)); if (sb.clusterhashes != null) remoteSeed.setAlternativeAddress((String) sb.clusterhashes.get(remoteSeed.hash));
// if the remote client has reported its own IP address and the client supports // if the remote client has reported its own IP address and the client supports
// the port forwarding feature (if client version >= 0.383) then we try to // the port forwarding feature (if client version >= 0.383) then we try to
@ -132,8 +131,8 @@ public final class hello {
// try first the reportedip, since this may be a connect from a port-forwarding host // try first the reportedip, since this may be a connect from a port-forwarding host
prop.put("yourip", reportedip); prop.put("yourip", reportedip);
oseed.put(yacySeed.IP, reportedip); remoteSeed.put(yacySeed.IP, reportedip);
urls = yacyClient.queryUrlCount(oseed); urls = yacyClient.queryUrlCount(remoteSeed);
} else { } else {
prop.put("yourip", "unknown"); prop.put("yourip", "unknown");
} }
@ -151,41 +150,41 @@ public final class hello {
serverCore.checkInterruption(); serverCore.checkInterruption();
prop.put("yourip", clientip); prop.put("yourip", clientip);
oseed.put(yacySeed.IP, clientip); remoteSeed.put(yacySeed.IP, clientip);
urls = yacyClient.queryUrlCount(oseed); urls = yacyClient.queryUrlCount(remoteSeed);
} }
} }
// System.out.println("YACYHELLO: YOUR IP=" + clientip); // System.out.println("YACYHELLO: YOUR IP=" + clientip);
// set lastseen value (we have seen that peer, it contacted us!) // set lastseen value (we have seen that peer, it contacted us!)
oseed.setLastSeenUTC(); remoteSeed.setLastSeenUTC();
// assign status // assign status
if (urls >= 0) { if (urls >= 0) {
if (oseed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) == null) { if (remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) == null) {
prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_SENIOR); prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_SENIOR);
oseed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); remoteSeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR);
} else if (oseed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_PRINCIPAL).equals(yacySeed.PEERTYPE_PRINCIPAL)) { } else if (remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_PRINCIPAL).equals(yacySeed.PEERTYPE_PRINCIPAL)) {
prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_PRINCIPAL); prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_PRINCIPAL);
} else { } else {
prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_SENIOR); prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_SENIOR);
oseed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); remoteSeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR);
} }
// connect the seed // connect the seed
yacyCore.peerActions.peerArrival(oseed, true); yacyCore.peerActions.peerArrival(remoteSeed, true);
} else { } else {
prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_JUNIOR); prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_JUNIOR);
yacyCore.peerActions.juniorConnects++; // update statistics yacyCore.peerActions.juniorConnects++; // update statistics
oseed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR); remoteSeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
yacyCore.log.logInfo("hello: responded remote junior peer '" + oseed.getName() + "' from " + reportedip); yacyCore.log.logInfo("hello: responded remote junior peer '" + remoteSeed.getName() + "' from " + reportedip);
// no connection here, instead store junior in connection cache // no connection here, instead store junior in connection cache
if ((oseed.hash != null) && (oseed.isProper() == null)) { if ((remoteSeed.hash != null) && (remoteSeed.isProper() == null)) {
yacyCore.peerActions.peerPing(oseed); yacyCore.peerActions.peerPing(remoteSeed);
} }
} }
yacyCore.peerActions.setUserAgent(clientip, userAgent); yacyCore.peerActions.setUserAgent(clientip, userAgent);
if (!((String)prop.get(yacySeed.YOURTYPE)).equals(reportedPeerType)) { if (!((String)prop.get(yacySeed.YOURTYPE)).equals(reportedPeerType)) {
yacyCore.log.logInfo("hello: changing remote peer '" + oseed.getName() + yacyCore.log.logInfo("hello: changing remote peer '" + remoteSeed.getName() +
"' [" + reportedip + "' [" + reportedip +
"] peerType from '" + reportedPeerType + "] peerType from '" + reportedPeerType +
"' to '" + prop.get(yacySeed.YOURTYPE) + "'."); "' to '" + prop.get(yacySeed.YOURTYPE) + "'.");

24
htroot/yacy/message.java
Unescape View File

@ -73,17 +73,16 @@ public final class message {
} }
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
// return variable that accumulates replacements if (post == null || env == null) { return null; }
final serverObjects prop = new serverObjects();
// defaults
prop.put("response", "-1"); // request rejected
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) { // return variable that accumulates replacements
return prop; plasmaSwitchboard sb = (plasmaSwitchboard) env;
} serverObjects prop = new serverObjects();
if ((post == null) || (env == null)) return prop;
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
String process = post.get("process", "permission"); String process = post.get("process", "permission");
String key = post.get("key", ""); String key = post.get("key", "");
int messagesize = 10240; int messagesize = 10240;
int attachmentsize = 0; int attachmentsize = 0;
@ -95,14 +94,15 @@ public final class message {
// check if we are the right target and requester has correct information about this peer // check if we are the right target and requester has correct information about this peer
if ((yacyCore.seedDB.mySeed() == null) || (!(yacyCore.seedDB.mySeed().hash.equals(youare)))) { if ((yacyCore.seedDB.mySeed() == null) || (!(yacyCore.seedDB.mySeed().hash.equals(youare)))) {
// this request has a wrong target // this request has a wrong target
prop.put("response", "-1"); // request rejected
return prop; return prop;
} }
final plasmaSwitchboard sb = (plasmaSwitchboard) env; if ((sb.isRobinsonMode()) &&
if (sb.isRobinsonMode() && (!((sb.isPublicRobinson()) ||
!(sb.isPublicRobinson() || (sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))))) {
sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))) {
// if we are a robinson cluster, answer only if this client is known by our network definition // if we are a robinson cluster, answer only if this client is known by our network definition
prop.put("response", "-1"); // request rejected
return prop; return prop;
} }

29
htroot/yacy/profile.java
Unescape View File

@ -64,32 +64,29 @@ public final class profile {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
// return variable that accumulates replacements // return variable that accumulates replacements
final serverObjects prop = new serverObjects(); serverObjects prop = new serverObjects();
// defaults plasmaSwitchboard sb = (plasmaSwitchboard) env;
prop.put("list", "0"); if ((post == null) || (env == null)) return prop;
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) { if ((sb.isRobinsonMode()) &&
return prop; (!sb.isPublicRobinson()) &&
} (!sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))) {
// if we are a robinson cluster, answer only if this client is known by our network definition
final plasmaSwitchboard sb = (plasmaSwitchboard) env; prop.put("list", "0");
if (sb.isRobinsonMode() &&
!sb.isPublicRobinson() &&
!sb.isInMyCluster((String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP))) {
// if we are a robinson cluster, answer only if this client is known by our network definition
return prop; return prop;
} }
Properties profile = new Properties(); Properties profile = new Properties();
int count = 0; int count=0;
String key = ""; String key="";
String value = ""; String value="";
FileInputStream fileIn = null; FileInputStream fileIn = null;
try { try {
fileIn = new FileInputStream(new File("DATA/SETTINGS/profile.txt")); fileIn = new FileInputStream(new File("DATA/SETTINGS/profile.txt"));
profile.load(fileIn); profile.load(fileIn);
} catch (IOException e) { } catch(IOException e) {
} finally { } finally {
if (fileIn != null) try { fileIn.close(); fileIn = null; } catch (Exception e) {} if (fileIn != null) try { fileIn.close(); fileIn = null; } catch (Exception e) {}
} }

47
htroot/yacy/query.java
Unescape View File

@ -53,46 +53,37 @@ import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNetwork; import de.anomic.yacy.yacyNetwork;
import de.anomic.yacy.yacySeed;
public final class query { public final class query {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch ss) {
if (post == null || ss == null) { return null; }
// return variable that accumulates replacements // return variable that accumulates replacements
final plasmaSwitchboard sb = (plasmaSwitchboard) ss;
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
if ((post == null) || (ss == null)) return prop;
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) { if (!yacyNetwork.authentifyRequest(post, ss)) return prop;
return prop;
}
if ((sb.isRobinsonMode()) &&
final plasmaSwitchboard sb = (plasmaSwitchboard) env; (!sb.isPublicRobinson()) &&
if (sb.isRobinsonMode() && (!sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))) {
!sb.isPublicRobinson() && // if we are a robinson cluster, answer only if we are public robinson peers,
!sb.isInMyCluster((String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP))) { // or we are a private cluster and the requester is in our cluster.
// if we are a robinson cluster, answer only if we are public robinson peers, // if we don't answer, the remote peer will recognize us as junior peer,
// or we are a private cluster and the requester is in our cluster. // what would mean that our peer ping does not work
// if we don't answer, the remote peer will recognize us as junior peer, prop.put("response", "-1"); // request rejected
// what would mean that our peer ping does not work
prop.put("response", "-1"); // request rejected
return prop; return prop;
} }
// System.out.println("YACYQUERY: RECEIVED POST = " + ((post == null) ? "NULL" : post.toString())); // System.out.println("YACYQUERY: RECEIVED POST = " + ((post == null) ? "NULL" : post.toString()));
final String ohash = post.get("iam", ""); // complete seed of the requesting peer // final String iam = post.get("iam", ""); // complete seed of the requesting peer
final String youare = post.get("youare", ""); // seed hash of the target peer, used for testing network stability final String youare = post.get("youare", ""); // seed hash of the target peer, used for testing network stability
// final String key = post.get("key", ""); // transmission key for response // final String key = post.get("key", ""); // transmission key for response
final String obj = post.get("object", ""); // keyword for query subject final String obj = post.get("object", ""); // keyword for query subject
final String qenv = post.get("env", ""); // argument to query final String env = post.get("env", ""); // argument to query
final yacySeed oseed = yacyCore.seedDB.get(ohash);
if (oseed == null) {
prop.put("response", "0");
return prop;
} else {
oseed.setFlagDirectConnect(true);
oseed.setLastSeenUTC();
}
prop.put("mytime", serverDate.formatShortSecond()); prop.put("mytime", serverDate.formatShortSecond());
@ -107,7 +98,7 @@ public final class query {
if (obj.equals("rwiurlcount")) { if (obj.equals("rwiurlcount")) {
// the total number of different urls in the rwi is returned // the total number of different urls in the rwi is returned
// <env> shall contain a word hash, the number of assigned lurls to this hash is returned // <env> shall contain a word hash, the number of assigned lurls to this hash is returned
prop.put("response", sb.wordIndex.indexSize(qenv)); prop.put("response", sb.wordIndex.indexSize(env));
return prop; return prop;
} }

119
htroot/yacy/search.java
Unescape View File

@ -60,17 +60,46 @@ public final class search {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
// return variable that accumulates replacements // return variable that accumulates replacements
final serverObjects prop = new serverObjects(); final plasmaSwitchboard sb = (plasmaSwitchboard) env;
// defaults sb.remoteSearchLastAccess = System.currentTimeMillis();
prop.put("links", "");
prop.put("linkcount", "0"); serverObjects prop = new serverObjects();
prop.put("references", ""); if ((post == null) || (env == null)) return prop;
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) {
return prop;
}
String client = (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP); String client = (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP);
//System.out.println("yacy: search received request = " + post.toString());
final String oseed = post.get("myseed", ""); // complete seed of the requesting peer
// final String youare = post.get("youare", ""); // seed hash of the target peer, used for testing network stability
final String key = post.get("key", ""); // transmission key for response
final String query = post.get("query", ""); // a string of word hashes that shall be searched and combined
final String exclude= post.get("exclude", "");// a string of word hashes that shall not be within the search result
String urls = post.get("urls", ""); // a string of url hashes that are preselected for the search: no other may be returned
String abstracts = post.get("abstracts", ""); // a string of word hashes for abstracts that shall be generated, or 'auto' (for maxcount-word), or '' (for none)
// final String fwdep = post.get("fwdep", ""); // forward depth. if "0" then peer may NOT ask another peer for more results
// final String fwden = post.get("fwden", ""); // forward deny, a list of seed hashes. They may NOT be target of forward hopping
final int count = Math.min(100, post.getInt("count", 10)); // maximum number of wanted results
final int maxdist= post.getInt("maxdist", Integer.MAX_VALUE);
final String prefer = post.get("prefer", "");
final String contentdom = post.get("contentdom", "text");
final String filter = post.get("filter", ".*");
final int partitions = post.getInt("partitions", 30);
String profile = post.get("profile", ""); // remote profile hand-over
if (profile.length() > 0) profile = crypt.simpleDecode(profile, null);
//final boolean includesnippet = post.get("includesnippet", "false").equals("true");
kelondroBitfield constraint = ((post.containsKey("constraint")) && (post.get("constraint", "").length() > 0)) ? new kelondroBitfield(4, post.get("constraint", "______")) : null;
if (constraint != null) {
// check bad handover parameter from older versions
boolean allon = true;
for (int i = 0; i < 32; i++) {
if (!constraint.get(i)) {allon = false; break;}
}
if (allon) constraint = null;
}
// final boolean global = ((String) post.get("resource", "global")).equals("global"); // if true, then result may consist of answers from other peers
// Date remoteTime = yacyCore.parseUniversalDate((String) post.get(yacySeed.MYTIME)); // read remote time
// test: // test:
// http://localhost:8080/yacy/search.html?query=4galTpdpDM5Q (search for linux) // http://localhost:8080/yacy/search.html?query=4galTpdpDM5Q (search for linux)
// http://localhost:8080/yacy/search.html?query=gh8DKIhGKXws (search for book) // http://localhost:8080/yacy/search.html?query=gh8DKIhGKXws (search for book)
@ -80,69 +109,34 @@ public final class search {
// http://localhost:8080/yacy/search.html?query=4galTpdpDM5Qgh8DKIhGKXws&abstracts=auto (search for linux and book, generate abstract automatically) // http://localhost:8080/yacy/search.html?query=4galTpdpDM5Qgh8DKIhGKXws&abstracts=auto (search for linux and book, generate abstract automatically)
// http://localhost:8080/yacy/search.html?query=&abstracts=4galTpdpDM5Q (only abstracts for linux) // http://localhost:8080/yacy/search.html?query=&abstracts=4galTpdpDM5Q (only abstracts for linux)
final plasmaSwitchboard sb = (plasmaSwitchboard) env; if ((sb.isRobinsonMode()) &&
if (sb.isRobinsonMode() && (!((sb.isPublicRobinson()) ||
!(sb.isPublicRobinson() || (sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))))) {
sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))) { // if we are a robinson cluster, answer only if this client is known by our network definition
// if we are a robinson cluster, answer only if this client is known by our network definition prop.put("links", "");
return prop; prop.put("linkcount", "0");
} prop.put("references", "");
return prop;
final String query = post.get("query", ""); // a string of word hashes that shall be searched and combined
final String abstracts = post.get("abstracts", ""); // a string of word hashes for abstracts that shall be generated, or 'auto' (for maxcount-word), or '' (for none)
if ((query == null || query.length() == 0) & (abstracts == null || abstracts.length() == 0)) {
return prop;
} }
// tell all threads to do nothing for a specific time // tell all threads to do nothing for a specific time
sb.intermissionAllThreads(3000); sb.intermissionAllThreads(3000);
sb.remoteSearchLastAccess = System.currentTimeMillis();
// myseed = complete seed of the requesting peer, key = transmission key for response TreeSet<String> abstractSet = ((abstracts.length() == 0) || (abstracts.equals("auto"))) ? null : plasmaSearchQuery.hashes2Set(abstracts);
final yacySeed oseed = yacySeed.genRemoteSeed(post.get("myseed", ""), post.get("key", ""), true);
// store accessing peer // store accessing peer
if (yacyCore.seedDB == null) { if (yacyCore.seedDB == null) {
yacyCore.log.logSevere("yacy.search: seed cache not initialized"); yacyCore.log.logSevere("yacy.search: seed cache not initialized");
} else { } else {
yacyCore.peerActions.peerArrival(oseed, true); yacyCore.peerActions.peerArrival(yacySeed.genRemoteSeed(oseed, key, true), true);
}
// final String youare = post.get("youare", ""); // seed hash of the target peer, used for testing network stability
// final String fwdep = post.get("fwdep", ""); // forward depth. if "0" then peer may NOT ask another peer for more results
// final String fwden = post.get("fwden", ""); // forward deny, a list of seed hashes. They may NOT be target of forward hopping
String profile = post.get("profile", ""); // remote profile hand-over
String urls = post.get("urls", ""); // a string of url hashes that are preselected for the search: no other may be returned
final String exclude = post.get("exclude", ""); // a string of word hashes that shall not be within the search result
final int count = Math.min(100, post.getInt("count", 10)); // maximum number of wanted results
final int maxdist = post.getInt("maxdist", Integer.MAX_VALUE);
final String prefer = post.get("prefer", "");
final String filter = post.get("filter", ".*");
final int partitions = post.getInt("partitions", 30);
final String contentdom = post.get("contentdom", "text");
if (profile.length() > 0) profile = crypt.simpleDecode(profile, null);
//final boolean includesnippet = post.get("includesnippet", "false").equals("true");
kelondroBitfield constraint = ((post.containsKey("constraint")) && (post.get("constraint", "").length() > 0)) ? new kelondroBitfield(4, post.get("constraint", "______")) : null;
if (constraint != null) {
// check bad handover parameter from older versions
boolean allon = true;
for (int i = 0; i < 32; i++) {
if (!constraint.get(i)) {allon = false; break;}
}
if (allon) constraint = null;
} }
// final boolean global = ((String) post.get("resource", "global")).equals("global"); // if true, then result may consist of answers from other peers
// Date remoteTime = yacyCore.parseUniversalDate((String) post.get(yacySeed.MYTIME)); // read remote time
// prepare search // prepare search
final TreeSet<String> queryhashes = plasmaSearchQuery.hashes2Set(query); final TreeSet<String> queryhashes = plasmaSearchQuery.hashes2Set(query);
final TreeSet<String> abstractSet = (abstracts.length() == 0 || abstracts.equals("auto")) ? null : plasmaSearchQuery.hashes2Set(abstracts);
final TreeSet<String> excludehashes = (exclude.length() == 0) ? new TreeSet<String>(kelondroBase64Order.enhancedComparator) : plasmaSearchQuery.hashes2Set(exclude); final TreeSet<String> excludehashes = (exclude.length() == 0) ? new TreeSet<String>(kelondroBase64Order.enhancedComparator) : plasmaSearchQuery.hashes2Set(exclude);
final long timestamp = System.currentTimeMillis(); final long timestamp = System.currentTimeMillis();
// prepare a search profile // prepare a search profile
plasmaSearchRankingProfile rankingProfile = (profile.length() == 0) ? new plasmaSearchRankingProfile(plasmaSearchQuery.contentdomParser(contentdom)) : new plasmaSearchRankingProfile("", profile); plasmaSearchRankingProfile rankingProfile = (profile.length() == 0) ? new plasmaSearchRankingProfile(plasmaSearchQuery.contentdomParser(contentdom)) : new plasmaSearchRankingProfile("", profile);
// prepare an abstract result // prepare an abstract result
@ -173,6 +167,7 @@ public final class search {
indexabstract.append("indexabstract." + wordhash + "=").append(indexContainer.compressIndex(container, null, 1000).toString()).append(serverCore.CRLF_STRING); indexabstract.append("indexabstract." + wordhash + "=").append(indexContainer.compressIndex(container, null, 1000).toString()).append(serverCore.CRLF_STRING);
} }
} }
prop.put("indexcount", ""); prop.put("indexcount", "");
prop.put("joincount", "0"); prop.put("joincount", "0");
prop.put("references", ""); prop.put("references", "");
@ -255,7 +250,14 @@ public final class search {
prop.put("indexabstract", indexabstract.toString()); prop.put("indexabstract", indexabstract.toString());
// prepare result // prepare result
if (joincount != 0 || accu != null) { if ((joincount == 0) || (accu == null)) {
// no results
prop.put("links", "");
prop.put("linkcount", "0");
prop.put("references", "");
} else {
// result is a List of urlEntry elements // result is a List of urlEntry elements
long timer = System.currentTimeMillis(); long timer = System.currentTimeMillis();
StringBuffer links = new StringBuffer(); StringBuffer links = new StringBuffer();
@ -299,9 +301,10 @@ public final class search {
prop.put("searchtime", System.currentTimeMillis() - timestamp); prop.put("searchtime", System.currentTimeMillis() - timestamp);
final int links = Integer.parseInt(prop.get("linkcount", "0")); final int links = Integer.parseInt(prop.get("linkcount","0"));
yacyCore.seedDB.mySeed().incSI(links); yacyCore.seedDB.mySeed().incSI(links);
yacyCore.seedDB.mySeed().incSU(links); yacyCore.seedDB.mySeed().incSU(links);
return prop; return prop;
} }
} }

55
htroot/yacy/transfer.java
Unescape View File

@ -61,19 +61,17 @@ import de.anomic.yacy.yacySeed;
public final class transfer { public final class transfer {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
// return variable that accumulates replacements plasmaSwitchboard sb = (plasmaSwitchboard) env;
final serverObjects prop = new serverObjects(); serverObjects prop = new serverObjects();
if ((post == null) || (env == null)) return prop;
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) { String process = post.get("process", ""); // permission or store
return prop; //String key = post.get("key", ""); // a transmission key from the client
} String otherpeer = post.get("iam", ""); // identification of the client (a peer-hash)
String purpose = post.get("purpose", ""); // declares how the file shall be treated
String process = post.get("process", ""); // permission or store String filename = post.get("filename", ""); // a name of a file without path
// String key = post.get("key", ""); // a transmission key from the client //long filesize = Long.parseLong((String) post.get("filesize", "")); // the size of the file
String ohash = post.get("iam", ""); // identification of the client (a peer-hash)
String purpose = post.get("purpose", ""); // declares how the file shall be treated
String filename = post.get("filename", ""); // a name of a file without path
// long filesize = Long.parseLong((String) post.get("filesize", "")); // the size of the file
prop.put("process", "0"); prop.put("process", "0");
prop.put("response", "denied"); // reject is default and is overwritten if ok prop.put("response", "denied"); // reject is default and is overwritten if ok
@ -83,34 +81,29 @@ public final class transfer {
prop.put("process_path", ""); prop.put("process_path", "");
prop.put("process_maxsize", "0"); prop.put("process_maxsize", "0");
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
if (sb.isRobinsonMode() || !sb.rankingOn) { if (sb.isRobinsonMode() || !sb.rankingOn) {
// in a robinson environment, do not answer. We do not do any transfer in a robinson cluster. // in a robinson environment, do not answer. We do not do any transfer in a robinson cluster.
return prop; return prop;
} }
final yacySeed oseed = yacyCore.seedDB.get(ohash); yacySeed otherseed = yacyCore.seedDB.get(otherpeer);
if (oseed == null) { if ((otherseed == null) || (filename.indexOf("..") >= 0)) {
// reject unknown peers: this does not appear fair, but anonymous senders are dangerous // reject unknown peers: this does not appear fair, but anonymous senders are dangerous
sb.getLog().logFine("RankingTransmission: rejected unknown peer '" + ohash + "', current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown"));
return prop;
}
oseed.setFlagDirectConnect(true);
oseed.setLastSeenUTC();
if (filename.indexOf("..") >= 0) {
// reject paths that contain '..' because they are dangerous // reject paths that contain '..' because they are dangerous
sb.getLog().logFine("RankingTransmission: rejected wrong path '" + filename + "' from peer " + oseed.getName() + "/" + oseed.getPublicAddress()+ ", current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown")); if (otherseed == null) sb.getLog().logFine("RankingTransmission: rejected unknown peer '" + otherpeer + "', current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown"));
if (filename.indexOf("..") >= 0) sb.getLog().logFine("RankingTransmission: rejected wrong path '" + filename + "' from peer " + otherseed.getName() + "/" + otherseed.getPublicAddress()+ ", current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown"));
return prop; return prop;
} }
String otherpeerName = otherseed.hash + ":" + otherseed.getName();
if (process.equals("permission")) { if (process.equals("permission")) {
prop.put("process", "0"); prop.put("process", "0");
if (((purpose.equals("crcon")) && (filename.startsWith("CRG")) && (filename.endsWith(".cr.gz"))) || ((filename.startsWith("domlist")) && (filename.endsWith(".txt.gz") || filename.endsWith(".zip")))) { if (((purpose.equals("crcon")) && (filename.startsWith("CRG")) && (filename.endsWith(".cr.gz"))) || ((filename.startsWith("domlist")) && (filename.endsWith(".txt.gz") || filename.endsWith(".zip")))) {
// consolidation of cr files // consolidation of cr files
//System.out.println("yacy/transfer:post=" + post.toString()); //System.out.println("yacy/transfer:post=" + post.toString());
//String cansendprotocol = (String) post.get("can-send-protocol", "http"); //String cansendprotocol = (String) post.get("can-send-protocol", "http");
String access = kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(ohash + ":" + filename)) + ":" + kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw("" + System.currentTimeMillis())); String access = kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(otherpeer + ":" + filename)) + ":" + kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw("" + System.currentTimeMillis()));
prop.put("response", "ok"); prop.put("response", "ok");
prop.put("process_access", access); prop.put("process_access", access);
prop.put("process_address", yacyCore.seedDB.mySeed().getPublicAddress()); prop.put("process_address", yacyCore.seedDB.mySeed().getPublicAddress());
@ -118,7 +111,7 @@ public final class transfer {
prop.put("process_path", ""); // currently empty; the store process will find a path prop.put("process_path", ""); // currently empty; the store process will find a path
prop.put("process_maxsize", "-1"); // if response is too big we return the size of the file prop.put("process_maxsize", "-1"); // if response is too big we return the size of the file
sb.rankingPermissions.put(serverCodings.encodeMD5Hex(kelondroBase64Order.standardCoder.encodeString(access)), filename); sb.rankingPermissions.put(serverCodings.encodeMD5Hex(kelondroBase64Order.standardCoder.encodeString(access)), filename);
sb.getLog().logFine("RankingTransmission: granted peer " + oseed.hash + ":" + oseed.getName() + " to send CR file " + filename); sb.getLog().logFine("RankingTransmission: granted peer " + otherpeerName + " to send CR file " + filename);
} }
return prop; return prop;
} }
@ -136,7 +129,7 @@ public final class transfer {
if ((grantedFile == null) || (!(grantedFile.equals(filename)))) { if ((grantedFile == null) || (!(grantedFile.equals(filename)))) {
// fraud-access of this interface // fraud-access of this interface
prop.put("response", "denied"); prop.put("response", "denied");
sb.getLog().logFine("RankingTransmission: denied " + oseed.hash + ":" + oseed.getName() + " to send CR file " + filename + ": wrong access code"); sb.getLog().logFine("RankingTransmission: denied " + otherpeerName + " to send CR file " + filename + ": wrong access code");
} else { } else {
sb.rankingPermissions.remove(accesscode); // not needed any more sb.rankingPermissions.remove(accesscode); // not needed any more
File path = new File(sb.rankingPath, plasmaRankingDistribution.CR_OTHER); File path = new File(sb.rankingPath, plasmaRankingDistribution.CR_OTHER);
@ -148,10 +141,10 @@ public final class transfer {
String md5t = serverCodings.encodeMD5Hex(file); String md5t = serverCodings.encodeMD5Hex(file);
if (md5t.equals(md5)) { if (md5t.equals(md5)) {
prop.put("response", "ok"); prop.put("response", "ok");
sb.getLog().logFine("RankingTransmission: received from peer " + oseed.hash + ":" + oseed.getName() + " CR file " + filename); sb.getLog().logFine("RankingTransmission: received from peer " + otherpeerName + " CR file " + filename);
} else { } else {
prop.put("response", "transfer failure"); prop.put("response", "transfer failure");
sb.getLog().logFine("RankingTransmission: transfer failure from peer " + oseed.hash + ":" + oseed.getName() + " for CR file " + filename); sb.getLog().logFine("RankingTransmission: transfer failure from peer " + otherpeerName + " for CR file " + filename);
} }
}else{ }else{
//exploit? //exploit?
@ -167,7 +160,7 @@ public final class transfer {
} }
// wrong access // wrong access
sb.getLog().logFine("RankingTransmission: rejected unknown process " + process + ":" + purpose + " from peer " + oseed.hash + ":" + oseed.getName()); sb.getLog().logFine("RankingTransmission: rejected unknown process " + process + ":" + purpose + " from peer " + otherpeerName);
return prop; return prop;
} }

51
htroot/yacy/transferRWI.java
Unescape View File

@ -66,41 +66,28 @@ import de.anomic.yacy.yacySeed;
public final class transferRWI { public final class transferRWI {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws InterruptedException { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws InterruptedException {
// return variable that accumulates replacements // return variable that accumulates replacements
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
final serverObjects prop = new serverObjects(); final serverObjects prop = new serverObjects();
if ((post == null) || (env == null)) return prop;
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) { if (!yacyNetwork.authentifyRequest(post, env)) return prop;
return prop;
}
if (!post.containsKey("wordc")) return prop; if (!post.containsKey("wordc")) return prop;
if (!post.containsKey("entryc")) return prop; if (!post.containsKey("entryc")) return prop;
// request values // request values
final String ohash = post.get("iam", ""); // seed hash of requester final String iam = post.get("iam", ""); // seed hash of requester
final String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability final String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability
// final String key = (String) post.get("key", ""); // transmission key // final String key = (String) post.get("key", ""); // transmission key
final int wordc = post.getInt("wordc", 0); // number of different words final int wordc = post.getInt("wordc", 0); // number of different words
final int entryc = post.getInt("entryc", 0); // number of entries in indexes final int entryc = post.getInt("entryc", 0); // number of entries in indexes
byte[] indexes = post.get("indexes", "").getBytes(); // the indexes, as list of word entries byte[] indexes = post.get("indexes", "").getBytes(); // the indexes, as list of word entries
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
boolean granted = sb.getConfig("allowReceiveIndex", "false").equals("true"); boolean granted = sb.getConfig("allowReceiveIndex", "false").equals("true");
boolean blockBlacklist = sb.getConfig("indexReceiveBlockBlacklist", "false").equals("true"); boolean blockBlacklist = sb.getConfig("indexReceiveBlockBlacklist", "false").equals("true");
boolean checkLimit = sb.getConfigBool("indexDistribution.transferRWIReceiptLimitEnabled", true); boolean checkLimit = sb.getConfigBool("indexDistribution.transferRWIReceiptLimitEnabled", true);
final long cachelimit = sb.getConfigLong("indexDistribution.dhtReceiptLimit", 10000); final long cachelimit = sb.getConfigLong("indexDistribution.dhtReceiptLimit", 10000);
final yacySeed otherPeer = yacyCore.seedDB.get(iam);
final yacySeed oseed = yacyCore.seedDB.get(ohash); final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));
if (oseed == null) {
prop.put("unknownURL", "");
prop.put("result", "busy");
prop.put("pause", "120000");
return prop;
} else {
oseed.setFlagDirectConnect(true);
oseed.setLastSeenUTC();
}
final String oname = ohash + ":" + oseed.getName() + "/" + oseed.getVersion();
// response values // response values
String result = "ok"; String result = "ok";
@ -108,17 +95,17 @@ public final class transferRWI {
int pause = 10000; int pause = 10000;
if ((youare == null) || (!youare.equals(yacyCore.seedDB.mySeed().hash))) { if ((youare == null) || (!youare.equals(yacyCore.seedDB.mySeed().hash))) {
sb.getLog().logInfo("Rejecting RWIs from peer " + oname + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash); sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash);
result = "wrong_target"; result = "wrong_target";
pause = 0; pause = 0;
} else if ((!granted) || (sb.isRobinsonMode())) { } else if ((!granted) || (sb.isRobinsonMode())) {
// we dont want to receive indexes // we dont want to receive indexes
sb.getLog().logInfo("Rejecting RWIs from peer " + oname + ". Not granted."); sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Not granted.");
result = "not_granted"; result = "not_granted";
pause = 0; pause = 0;
} else if (checkLimit && sb.wordIndex.dhtInCacheSize() > cachelimit) { } else if (checkLimit && sb.wordIndex.dhtInCacheSize() > cachelimit) {
// we are too busy to receive indexes // we are too busy to receive indexes
sb.getLog().logInfo("Rejecting RWIs from peer " + oname + ". We are too busy (buffersize=" + sb.wordIndex.dhtInCacheSize() + ")."); sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". We are too busy (buffersize=" + sb.wordIndex.dhtInCacheSize() + ").");
granted = false; // don't accept more words if there are too many words to flush granted = false; // don't accept more words if there are too many words to flush
result = "busy"; result = "busy";
pause = 60000; pause = 60000;
@ -131,7 +118,7 @@ public final class transferRWI {
} */ else { } */ else {
// we want and can receive indexes // we want and can receive indexes
// log value status (currently added to find outOfMemory error // log value status (currently added to find outOfMemory error
sb.getLog().logFine("Processing " + indexes.length + " bytes / " + wordc + " words / " + entryc + " entries from " + oname); sb.getLog().logFine("Processing " + indexes.length + " bytes / " + wordc + " words / " + entryc + " entries from " + otherPeerName);
final long startProcess = System.currentTimeMillis(); final long startProcess = System.currentTimeMillis();
// decode request // decode request
@ -174,7 +161,7 @@ public final class transferRWI {
// block blacklisted entries // block blacklisted entries
if ((blockBlacklist) && (plasmaSwitchboard.urlBlacklist.hashInBlacklistedCache(plasmaURLPattern.BLACKLIST_DHT, urlHash))) { if ((blockBlacklist) && (plasmaSwitchboard.urlBlacklist.hashInBlacklistedCache(plasmaURLPattern.BLACKLIST_DHT, urlHash))) {
int deleted = sb.wordIndex.tryRemoveURLs(urlHash); int deleted = sb.wordIndex.tryRemoveURLs(urlHash);
yacyCore.log.logFine("transferRWI: blocked blacklisted URLHash '" + urlHash + "' from peer " + oname + "; deleted " + deleted + " URL entries from RWIs"); yacyCore.log.logFine("transferRWI: blocked blacklisted URLHash '" + urlHash + "' from peer " + otherPeerName + "; deleted " + deleted + " URL entries from RWIs");
blocked++; blocked++;
continue; continue;
} }
@ -208,10 +195,10 @@ public final class transferRWI {
} }
if (unknownURLs.length() > 0) { unknownURLs.delete(0, 1); } if (unknownURLs.length() > 0) { unknownURLs.delete(0, 1); }
if ((wordhashes.length == 0) || (received == 0)) { if ((wordhashes.length == 0) || (received == 0)) {
sb.getLog().logInfo("Received 0 RWIs from " + oname + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs, blocked " + blocked + " RWIs"); sb.getLog().logInfo("Received 0 RWIs from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs, blocked " + blocked + " RWIs");
} else { } else {
final double avdist = (yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, wordhashes[0]) + yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, wordhashes[received - 1])) / 2.0; final double avdist = (yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, wordhashes[0]) + yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, wordhashes[received - 1])) / 2.0;
sb.getLog().logInfo("Received " + received + " Entries " + wordc + " Words [" + wordhashes[0] + " .. " + wordhashes[received - 1] + "]/" + avdist + " from " + oname + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + "/" + receivedURL + " URLs, blocked " + blocked + " RWIs"); sb.getLog().logInfo("Received " + received + " Entries " + wordc + " Words [" + wordhashes[0] + " .. " + wordhashes[received - 1] + "]/" + avdist + " from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + "/" + receivedURL + " URLs, blocked " + blocked + " RWIs");
} }
result = "ok"; result = "ok";

62
htroot/yacy/transferURL.java
Unescape View File

@ -62,48 +62,38 @@ import de.anomic.yacy.yacySeed;
public final class transferURL { public final class transferURL {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws InterruptedException {
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) {
return prop;
}
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws InterruptedException {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
long freshdate = 0; long freshdate = 0;
try {freshdate = serverDate.parseShortDay("20061101").getTime();} catch (ParseException e1) {} try {freshdate = serverDate.parseShortDay("20061101").getTime();} catch (ParseException e1) {}
// request values // return variable that accumulates replacements
final String ohash = post.get("iam", ""); // seed hash of requester
final String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability
// final String key = post.get("key", ""); // transmission key
final int urlc = post.getInt("urlc", 0); // number of transported urls
final plasmaSwitchboard sb = (plasmaSwitchboard) env; final plasmaSwitchboard sb = (plasmaSwitchboard) env;
final serverObjects prop = new serverObjects();
if ((post == null) || (env == null)) return prop;
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
// request values
final String iam = post.get("iam", ""); // seed hash of requester
final String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability
// final String key = post.get("key", ""); // transmission key
final int urlc = post.getInt("urlc", 0); // number of transported urls
final boolean granted = sb.getConfig("allowReceiveIndex", "false").equals("true"); final boolean granted = sb.getConfig("allowReceiveIndex", "false").equals("true");
final boolean blockBlacklist = sb.getConfig("indexReceiveBlockBlacklist", "false").equals("true"); final boolean blockBlacklist = sb.getConfig("indexReceiveBlockBlacklist", "false").equals("true");
final yacySeed oseed = yacyCore.seedDB.get(ohash);
if (oseed == null) {
prop.put("result", "error_not_granted");
prop.put("pause", "120000");
return prop;
} else {
oseed.setFlagDirectConnect(true);
oseed.setLastSeenUTC();
}
final String oname = ohash + ":" + oseed.getName() + "/" + oseed.getVersion();
// response values // response values
String result = ""; String result = "";
String doublevalues = "0"; String doublevalues = "0";
final yacySeed otherPeer = yacyCore.seedDB.get(iam);
final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));
if ((youare == null) || (!youare.equals(yacyCore.seedDB.mySeed().hash))) { if ((youare == null) || (!youare.equals(yacyCore.seedDB.mySeed().hash))) {
sb.getLog().logInfo("Rejecting URLs from peer " + oname + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash); sb.getLog().logInfo("Rejecting URLs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash);
result = "wrong_target"; result = "wrong_target";
} else if ((!granted) || (sb.isRobinsonMode())) { } else if ((!granted) || (sb.isRobinsonMode())) {
sb.getLog().logInfo("Rejecting URLs from peer " + oname + ". Not granted."); sb.getLog().logInfo("Rejecting URLs from peer " + otherPeerName + ". Not granted.");
result = "error_not_granted"; result = "error_not_granted";
} else { } else {
int received = 0; int received = 0;
@ -118,7 +108,7 @@ public final class transferURL {
// read new lurl-entry // read new lurl-entry
urls = (String) post.get("url" + i); urls = (String) post.get("url" + i);
if (urls == null) { if (urls == null) {
yacyCore.log.logFine("transferURL: got null URL-string from peer " + oname); yacyCore.log.logFine("transferURL: got null URL-string from peer " + otherPeerName);
blocked++; blocked++;
continue; continue;
} }
@ -126,7 +116,7 @@ public final class transferURL {
// parse new lurl-entry // parse new lurl-entry
lEntry = sb.wordIndex.loadedURL.newEntry(urls); lEntry = sb.wordIndex.loadedURL.newEntry(urls);
if (lEntry == null) { if (lEntry == null) {
yacyCore.log.logWarning("transferURL: received invalid URL (entry null) from peer " + oname + "\n\tURL Property: " + urls); yacyCore.log.logWarning("transferURL: received invalid URL (entry null) from peer " + otherPeerName + "\n\tURL Property: " + urls);
blocked++; blocked++;
continue; continue;
} }
@ -134,14 +124,14 @@ public final class transferURL {
// check if entry is well-formed // check if entry is well-formed
indexURLEntry.Components comp = lEntry.comp(); indexURLEntry.Components comp = lEntry.comp();
if (comp.url() == null) { if (comp.url() == null) {
yacyCore.log.logWarning("transferURL: received invalid URL from peer " + oname + "\n\tURL Property: " + urls); yacyCore.log.logWarning("transferURL: received invalid URL from peer " + otherPeerName + "\n\tURL Property: " + urls);
blocked++; blocked++;
continue; continue;
} }
// check whether entry is too old // check whether entry is too old
if (lEntry.freshdate().getTime() <= freshdate) { if (lEntry.freshdate().getTime() <= freshdate) {
yacyCore.log.logFine("transerURL: received too old URL from peer " + oname + ": " + lEntry.freshdate()); yacyCore.log.logFine("transerURL: received too old URL from peer " + otherPeerName + ": " + lEntry.freshdate());
blocked++; blocked++;
continue; continue;
} }
@ -149,7 +139,7 @@ public final class transferURL {
// check if the entry is blacklisted // check if the entry is blacklisted
if ((blockBlacklist) && (plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_DHT, comp.url()))) { if ((blockBlacklist) && (plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_DHT, comp.url()))) {
int deleted = sb.wordIndex.tryRemoveURLs(lEntry.hash()); int deleted = sb.wordIndex.tryRemoveURLs(lEntry.hash());
yacyCore.log.logFine("transferURL: blocked blacklisted URL '" + comp.url().toNormalform(false, true) + "' from peer " + oname + "; deleted " + deleted + " URL entries from RWIs"); yacyCore.log.logFine("transferURL: blocked blacklisted URL '" + comp.url().toNormalform(false, true) + "' from peer " + otherPeerName + "; deleted " + deleted + " URL entries from RWIs");
lEntry = null; lEntry = null;
blocked++; blocked++;
continue; continue;
@ -157,7 +147,7 @@ public final class transferURL {
// check if the entry is in our network domain // check if the entry is in our network domain
if (!sb.acceptURL(comp.url())) { if (!sb.acceptURL(comp.url())) {
yacyCore.log.logFine("transferURL: blocked URL outside of our domain '" + comp.url().toNormalform(false, true) + "' from peer " + oname); yacyCore.log.logFine("transferURL: blocked URL outside of our domain '" + comp.url().toNormalform(false, true) + "' from peer " + otherPeerName);
lEntry = null; lEntry = null;
blocked++; blocked++;
continue; continue;
@ -166,8 +156,8 @@ public final class transferURL {
// write entry to database // write entry to database
try { try {
sb.wordIndex.loadedURL.store(lEntry); sb.wordIndex.loadedURL.store(lEntry);
sb.wordIndex.loadedURL.stack(lEntry, ohash, ohash, 3); sb.wordIndex.loadedURL.stack(lEntry, iam, iam, 3);
yacyCore.log.logFine("transferURL: received URL '" + comp.url().toNormalform(false, true) + "' from peer " + oname); yacyCore.log.logFine("transferURL: received URL '" + comp.url().toNormalform(false, true) + "' from peer " + otherPeerName);
received++; received++;
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
@ -179,8 +169,8 @@ public final class transferURL {
// return rewrite properties // return rewrite properties
final int more = sb.wordIndex.loadedURL.size() - sizeBefore; final int more = sb.wordIndex.loadedURL.size() - sizeBefore;
doublevalues = Integer.toString(received - more); doublevalues = Integer.toString(received - more);
sb.getLog().logInfo("Received " + received + " URLs from peer " + oname + " in " + (System.currentTimeMillis() - start) + " ms, Blocked " + blocked + " URLs"); sb.getLog().logInfo("Received " + received + " URLs from peer " + otherPeerName + " in " + (System.currentTimeMillis() - start) + " ms, Blocked " + blocked + " URLs");
if ((received - more) > 0) sb.getLog().logSevere("Received " + doublevalues + " double URLs from peer " + oname); if ((received - more) > 0) sb.getLog().logSevere("Received " + doublevalues + " double URLs from peer " + otherPeerName);
result = "ok"; result = "ok";
} }

82
source/de/anomic/yacy/yacyCore.java
Unescape View File

@ -81,21 +81,23 @@ public class yacyCore {
// statics // statics
public static final ThreadGroup publishThreadGroup = new ThreadGroup("publishThreadGroup"); public static final ThreadGroup publishThreadGroup = new ThreadGroup("publishThreadGroup");
public static yacySeedDB seedDB; public static yacySeedDB seedDB = null;
public static yacyNewsPool newsPool; public static yacyNewsPool newsPool = null;
public static final HashMap<String, String> seedUploadMethods = new HashMap<String, String>(); public static final HashMap<String, String> seedUploadMethods = new HashMap<String, String>();
public static yacyPeerActions peerActions; public static yacyPeerActions peerActions = null;
public static yacyDHTAction dhtAgent; public static yacyDHTAction dhtAgent = null;
public static serverLog log; public static serverLog log;
public static long lastOnlineTime; public static long lastOnlineTime = 0;
/** pseudo-random key derived from a time-interval while YaCy startup. */ /** pseudo-random key derived from a time-interval while YaCy startup*/
public static long speedKey; public static long speedKey = 0;
public static File yacyDBPath; public static File yacyDBPath;
public static final Map<String, yacyAccessible> amIAccessibleDB = Collections.synchronizedMap(new HashMap<String, yacyAccessible>()); // Holds PeerHash / yacyAccessible Relations public static final Map<String, yacyAccessible> amIAccessibleDB = Collections.synchronizedMap(new HashMap<String, yacyAccessible>()); // Holds PeerHash / yacyAccessible Relations
// constants for PeerPing behaviour // constants for PeerPing behaviour
private static final int PING_INITIAL = 16; private static final int PING_INITIAL = 10;
private static final int PING_MIN_LASTSEEN = 240000; // in milliseconds private static final int PING_MAX_RUNNING = 3;
private static final int PING_MIN_PEERSEEN = 2; // min. accessible to force senior private static final int PING_MIN_RUNNING = 1;
private static final int PING_MIN_DBSIZE = 5;
private static final int PING_MIN_PEERSEEN = 1; // min. accessible to force senior
private static final long PING_MAX_DBAGE = 15 * 60 * 1000; // in milliseconds private static final long PING_MAX_DBAGE = 15 * 60 * 1000; // in milliseconds
// public static yacyShare shareManager = null; // public static yacyShare shareManager = null;
@ -369,33 +371,45 @@ public class yacyCore {
int attempts = seedDB.sizeConnected(); int attempts = seedDB.sizeConnected();
// getting a list of peers to contact // getting a list of peers to contact
if (seedDB.mySeed().isVirgin()) { if (seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN).equals(yacySeed.PEERTYPE_VIRGIN)) {
attempts = Math.min(attempts, PING_INITIAL); if (attempts > PING_INITIAL) { attempts = PING_INITIAL; }
final Map<String, String> ch = plasmaSwitchboard.getSwitchboard().clusterhashes; Map<String, String> ch = plasmaSwitchboard.getSwitchboard().clusterhashes;
seeds = seedDB.seedsByAge(true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection seeds = seedDB.seedsByAge(true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection
// add also all peers from cluster if this is a public robinson cluster // add also all peers from cluster if this is a public robinson cluster
if (plasmaSwitchboard.getSwitchboard().clusterhashes != null) { if (plasmaSwitchboard.getSwitchboard().clusterhashes != null) {
final Iterator<Map.Entry<String, String>> i = ch.entrySet().iterator(); Iterator<Map.Entry<String, String>> i = ch.entrySet().iterator();
String hash; String hash;
Map.Entry<String, String> entry; Map.Entry<String, String> entry;
yacySeed seed; yacySeed seed;
while (i.hasNext()) { while (i.hasNext()) {
entry = i.next(); entry = i.next();
hash = entry.getKey(); hash = entry.getKey();
seed = seeds.get(hash); seed = (yacySeed) seeds.get(hash);
if (seed == null) { if (seed == null) {
seed = seedDB.get(hash); seed = seedDB.get(hash);
if (seed == null) { continue; } if (seed == null) continue;
} }
seed.setAlternativeAddress(entry.getValue()); seed.setAlternativeAddress((String) entry.getValue());
seeds.put(hash, seed); seeds.put(hash, seed);
} }
} }
} else { } else {
seeds = seedDB.getOldestSeeds(attempts / 5, PING_MIN_LASTSEEN); int diff = PING_MIN_DBSIZE - amIAccessibleDB.size();
if (diff > PING_MIN_RUNNING) {
diff = Math.min(diff, PING_MAX_RUNNING);
if (attempts > diff) { attempts = diff; }
} else {
if (attempts > PING_MIN_RUNNING) { attempts = PING_MIN_RUNNING; }
}
seeds = seedDB.seedsByAge(false, attempts); // best for seed list maintenance/cleaning
} }
if (seeds == null || seeds.size() == 0) { return 0; }
attempts = seeds.size(); if ((seeds == null) || seeds.size() == 0) { return 0; }
if (seeds.size() < attempts) { attempts = seeds.size(); }
// This will try to get Peers that are not currently in amIAccessibleDB
Iterator<yacySeed> si = seeds.values().iterator();
yacySeed seed;
// include a YaCyNews record to my seed // include a YaCyNews record to my seed
try { try {
@ -420,14 +434,10 @@ public class yacyCore {
final List<Thread> syncList = Collections.synchronizedList(new LinkedList<Thread>()); // memory for threads final List<Thread> syncList = Collections.synchronizedList(new LinkedList<Thread>()); // memory for threads
final serverSemaphore sync = new serverSemaphore(attempts); final serverSemaphore sync = new serverSemaphore(attempts);
// This will try to get Peers that are not currently in amIAccessibleDB
Iterator<yacySeed> si = seeds.values().iterator();
yacySeed seed;
// going through the peer list and starting a new publisher thread for each peer // going through the peer list and starting a new publisher thread for each peer
int i = 0; int i = 0;
while (si.hasNext()) { while (si. hasNext()) {
seed = si.next(); seed = (yacySeed) si.next();
if (seed == null) { if (seed == null) {
sync.P(); sync.P();
continue; continue;
@ -436,8 +446,8 @@ public class yacyCore {
final String address = seed.getClusterAddress(); final String address = seed.getClusterAddress();
log.logFine("HELLO #" + i + " to peer '" + seed.get(yacySeed.NAME, "") + "' at " + address); // debug log.logFine("HELLO #" + i + " to peer '" + seed.get(yacySeed.NAME, "") + "' at " + address); // debug
final String seederror = seed.isProper(); String seederror = seed.isProper();
if (address == null || seederror != null) { if ((address == null) || (seederror != null)) {
// we don't like that address, delete it // we don't like that address, delete it
peerActions.peerDeparture(seed, "peer ping to peer resulted in address = " + address + "; seederror = " + seederror); peerActions.peerDeparture(seed, "peer ping to peer resulted in address = " + address + "; seederror = " + seederror);
sync.P(); sync.P();
@ -480,9 +490,9 @@ public class yacyCore {
final int dbSize; final int dbSize;
synchronized (amIAccessibleDB) { synchronized (amIAccessibleDB) {
dbSize = amIAccessibleDB.size(); dbSize = amIAccessibleDB.size();
final Iterator<String> ai = amIAccessibleDB.keySet().iterator(); Iterator<String> ai = amIAccessibleDB.keySet().iterator();
while (ai.hasNext()) { while (ai.hasNext()) {
final yacyAccessible ya = amIAccessibleDB.get(ai.next()); yacyAccessible ya = (yacyAccessible) amIAccessibleDB.get(ai.next());
if (ya.lastUpdated < cutofftime) { if (ya.lastUpdated < cutofftime) {
ai.remove(); ai.remove();
} else { } else {
@ -498,11 +508,11 @@ public class yacyCore {
log.logInfo("PeerPing: I am accessible for " + accessible + log.logInfo("PeerPing: I am accessible for " + accessible +
" peer(s), not accessible for " + notaccessible + " peer(s)."); " peer(s), not accessible for " + notaccessible + " peer(s).");
if (accessible + notaccessible > 0) { if ((accessible + notaccessible) > 0) {
final String newPeerType; final String newPeerType;
// At least one other Peer told us our type // At least one other Peer told us our type
if (accessible >= PING_MIN_PEERSEEN || if ((accessible >= PING_MIN_PEERSEEN) ||
accessible >= notaccessible) { (accessible >= notaccessible)) {
// We can be reached from a majority of other Peers // We can be reached from a majority of other Peers
if (yacyCore.seedDB.mySeed().isPrincipal()) { if (yacyCore.seedDB.mySeed().isPrincipal()) {
newPeerType = yacySeed.PEERTYPE_PRINCIPAL; newPeerType = yacySeed.PEERTYPE_PRINCIPAL;
@ -513,11 +523,11 @@ public class yacyCore {
// We cannot be reached from the outside // We cannot be reached from the outside
newPeerType = yacySeed.PEERTYPE_JUNIOR; newPeerType = yacySeed.PEERTYPE_JUNIOR;
} }
if (yacyCore.seedDB.mySeed().isType(newPeerType)) { if (yacyCore.seedDB.mySeed().orVirgin().equals(newPeerType)) {
log.logInfo("PeerPing: myType is " + yacyCore.seedDB.mySeed().orVirgin()); log.logInfo("PeerPing: myType is " + yacyCore.seedDB.mySeed().orVirgin());
} else { } else {
log.logInfo("PeerPing: changing myType from '" + yacyCore.seedDB.mySeed().orVirgin() + "' to '" + newPeerType + "'"); log.logInfo("PeerPing: changing myType from '" + yacyCore.seedDB.mySeed().orVirgin() + "' to '" + newPeerType + "'");
yacyCore.seedDB.mySeed().setType(newPeerType); yacyCore.seedDB.mySeed().put(yacySeed.PEERTYPE, newPeerType);
} }
} else { } else {
log.logInfo("PeerPing: No data, staying at myType: " + yacyCore.seedDB.mySeed().orVirgin()); log.logInfo("PeerPing: No data, staying at myType: " + yacyCore.seedDB.mySeed().orVirgin());

36
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -90,7 +90,7 @@ public final class yacySeedDB {
* these hashes all shall be generated by base64.enhancedCoder * these hashes all shall be generated by base64.enhancedCoder
*/ */
public static final int commonHashLength = 12; public static final int commonHashLength = 12;
public static final int dhtActivityMagic = 48; public static final int dhtActivityMagic = 32;
public static final String[] sortFields = new String[] {yacySeed.LCOUNT, yacySeed.ICOUNT, yacySeed.UPTIME, yacySeed.VERSION, yacySeed.LASTSEEN}; public static final String[] sortFields = new String[] {yacySeed.LCOUNT, yacySeed.ICOUNT, yacySeed.UPTIME, yacySeed.VERSION, yacySeed.LASTSEEN};
public static final String[] longaccFields = new String[] {yacySeed.LCOUNT, yacySeed.ICOUNT, yacySeed.ISPEED}; public static final String[] longaccFields = new String[] {yacySeed.LCOUNT, yacySeed.ICOUNT, yacySeed.ISPEED};
@ -378,40 +378,6 @@ public final class yacySeedDB {
} }
} }
public HashMap<String, yacySeed> getOldestSeeds(int count, int minage) {
// returns a peerhash/yacySeed relation
try {
final kelondroMScoreCluster<String> seedScore = new kelondroMScoreCluster<String>();
final Iterator<yacySeed> s = seedsConnected(true, false, null, (float) 0.0);
yacySeed ys;
int age;
int searchcount = 1000;
while (s.hasNext() && searchcount-- > 0) {
ys = s.next();
if (ys == null) { continue; }
age = (int) Math.abs(System.currentTimeMillis() + serverDate.dayMillis - ys.getLastSeenUTC());
if (age < minage) { continue; }
seedScore.addScore(ys.hash, age); // the higher age, the older is the peer
}
if (seedScore.size() == 0) { return null; }
// result is now in the score object; create a result vector
final HashMap<String, yacySeed> result = new HashMap<String, yacySeed>();
final Iterator<String> it = seedScore.scores(false);
searchcount = Math.min(count, seedScore.size());
int c = 0;
while (c++ < searchcount && it.hasNext()) {
ys = getConnected(it.next());
if (ys != null && ys.hash != null) { result.put(ys.hash, ys); }
}
return result;
} catch (kelondroException e) {
seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
yacyCore.log.logFine("Internal Error at yacySeedDB.seedsByAge: " + e.getMessage(), e);
return null;
}
}
public int sizeConnected() { public int sizeConnected() {
return seedActiveDB.size(); return seedActiveDB.size();
/* /*

Write Preview
Loading…
Cancel
Save