boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

reverted changes to yacy protocol classes: they caused the sciencenet to loose connections

Browse Source 
a comparisment with the main release 0.57 had been made: this showed a stable network
This is an emergency operation to ensure availability of the sciencenet network.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4553 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent b664a53553
commit 5530b8e1ca
11 changed files with 302 additions and 370 deletions
Show Stats Download Patch File Download Diff File

60
htroot/yacy/crawlReceipt.java
Unescape View File

@ -62,34 +62,25 @@ import de.anomic.yacy.yacySeed;
public final class crawlReceipt {
/*
* this is used to respond on a remote crawling request
*/
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
// defaults
prop.put("delay", "3600");
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) {
return prop;
}
// seed hash of requester
String ohash = post.get("iam", "");
if (ohash == null || ohash.length() == 0 ) return prop;
yacySeed oseed = yacyCore.seedDB.get(ohash);
if (oseed == null) return prop;
oseed.setFlagDirectConnect(true);
oseed.setLastSeenUTC();
plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
if ((post == null) || (env == null)) return prop;
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverLog log = sb.getLog();
serverLog log = switchboard.getLog();
//int proxyPrefetchDepth = Integer.parseInt(env.getConfig("proxyPrefetchDepth", "0"));
//int crawlingDepth = Integer.parseInt(env.getConfig("crawlingDepth", "0"));
// request values
String iam = post.get("iam", ""); // seed hash of requester
String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability
//String process = post.get("process", ""); // process type
String key = post.get("key", ""); // transmission key
@ -120,50 +111,53 @@ public final class crawlReceipt {
*/
final yacySeed otherPeer = yacyCore.seedDB.get(iam);
final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));
if ((yacyCore.seedDB.mySeed() == null) || (!(yacyCore.seedDB.mySeed().hash.equals(youare)))) {
// no yacy connection / unknown peers
prop.put("delay", "3600");
return prop;
}
if (propStr == null) {
// error with url / wrong key
prop.put("delay", "3600");
return prop;
}
if ((sb.isRobinsonMode()) && (!sb.isInMyCluster(oseed))) {
if ((switchboard.isRobinsonMode()) && (!switchboard.isInMyCluster(otherPeer))) {
// we reject urls that are from outside our cluster
prop.put("delay", "9999");
return prop; // ???
}
// generating a new loaded URL entry
indexURLEntry entry = sb.wordIndex.loadedURL.newEntry(propStr);
indexURLEntry entry = switchboard.wordIndex.loadedURL.newEntry(propStr);
if (entry == null) {
log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (entry null) from peer " + ohash + "\n\tURL properties: "+ propStr);
log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (entry null) from peer " + iam + "\n\tURL properties: "+ propStr);
prop.put("delay", "3600");
return prop;
}
indexURLEntry.Components comp = entry.comp();
if (comp.url() == null) {
log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (url null) for hash " + entry.hash() + " from peer " + ohash + "\n\tURL properties: "+ propStr);
log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (url null) for hash " + entry.hash() + " from peer " + iam + "\n\tURL properties: "+ propStr);
prop.put("delay", "3600");
return prop;
}
// check if the entry is in our network domain
if (!sb.acceptURL(comp.url())) {
log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (url outside of our domain) for hash " + entry.hash() + " from peer " + ohash + "\n\tURL properties: "+ propStr);
if (!switchboard.acceptURL(comp.url())) {
log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (url outside of our domain) for hash " + entry.hash() + " from peer " + iam + "\n\tURL properties: "+ propStr);
prop.put("delay", "9999");
return prop;
}
if (result.equals("fill")) try {
// put new entry into database
sb.wordIndex.loadedURL.store(entry);
sb.wordIndex.loadedURL.stack(entry, youare, ohash, 1);
sb.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work has been done
final String otherPeerName = ohash + ":" + oseed.getName() + "/" + oseed.getVersion();
switchboard.wordIndex.loadedURL.store(entry);
switchboard.wordIndex.loadedURL.stack(entry, youare, iam, 1);
switchboard.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work has been done
log.logInfo("crawlReceipt: RECEIVED RECEIPT from " + otherPeerName + " for URL " + entry.hash() + ":" + comp.url().toNormalform(false, true));
// ready for more
@ -171,14 +165,16 @@ public final class crawlReceipt {
return prop;
} catch (IOException e) {
e.printStackTrace();
prop.put("delay", "3600");
return prop;
}
sb.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work is transformed into an error case
plasmaCrawlZURL.Entry ee = sb.crawlQueues.errorURL.newEntry(entry.toBalancerEntry(), youare, null, 0, result + ":" + reason);
switchboard.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work is transformed into an error case
plasmaCrawlZURL.Entry ee = switchboard.crawlQueues.errorURL.newEntry(entry.toBalancerEntry(), youare, null, 0, result + ":" + reason);
ee.store();
sb.crawlQueues.errorURL.push(ee);
switchboard.crawlQueues.errorURL.push(ee);
//switchboard.noticeURL.remove(receivedUrlhash);
prop.put("delay", "3600");
return prop;
// return rewrite properties

51
htroot/yacy/hello.java
Unescape View File

@ -65,9 +65,10 @@ import de.anomic.yacy.yacyVersion;
public final class hello {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws InterruptedException {
// return variable that accumulates replacements
plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
if (post == null || env == null) {
prop.put("message", "none");
if ((post == null) || (env == null)) {
prop.put("message", "no post or no enviroment");
return prop;
}
@ -75,7 +76,6 @@ public final class hello {
prop.put("message", "not in my network");
return prop;
}
prop.put("message", "none");
// final String iam = (String) post.get("iam", ""); // complete seed of the requesting peer
// final String mytime = (String) post.get(MYTIME, ""); //
@ -90,10 +90,10 @@ public final class hello {
prop.put("message", "your seed is too long (" + seed.length() + ")");
return prop;
}
final yacySeed oseed = yacySeed.genRemoteSeed(seed, key, false);
final yacySeed remoteSeed = yacySeed.genRemoteSeed(seed, key, false);
// System.out.println("YACYHELLO: REMOTESEED=" + ((remoteSeed == null) ? "NULL" : remoteSeed.toString()));
if (oseed == null || oseed.hash == null) {
if ((remoteSeed == null) || (remoteSeed.hash == null)) {
prop.put("message", "cannot parse your seed");
return prop;
}
@ -110,19 +110,18 @@ public final class hello {
return prop;
}
final String userAgent = (String) header.get(httpHeader.USER_AGENT, "<unknown>");
final String reportedip = oseed.get(yacySeed.IP, "");
final String reportedPeerType = oseed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
final float clientversion = oseed.getVersion();
final String reportedip = remoteSeed.get(yacySeed.IP, "");
final String reportedPeerType = remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
final float clientversion = remoteSeed.getVersion();
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
if (sb.isRobinsonMode() && !sb.isPublicRobinson()) {
if ((sb.isRobinsonMode()) && (!sb.isPublicRobinson())) {
// if we are a robinson cluster, answer only if this client is known by our network definition
prop.put("message", "I am robinson, I do not answer");
return prop;
}
int urls = -1;
if (sb.clusterhashes != null) oseed.setAlternativeAddress((String) sb.clusterhashes.get(oseed.hash));
if (sb.clusterhashes != null) remoteSeed.setAlternativeAddress((String) sb.clusterhashes.get(remoteSeed.hash));
// if the remote client has reported its own IP address and the client supports
// the port forwarding feature (if client version >= 0.383) then we try to
@ -132,8 +131,8 @@ public final class hello {
// try first the reportedip, since this may be a connect from a port-forwarding host
prop.put("yourip", reportedip);
oseed.put(yacySeed.IP, reportedip);
urls = yacyClient.queryUrlCount(oseed);
remoteSeed.put(yacySeed.IP, reportedip);
urls = yacyClient.queryUrlCount(remoteSeed);
} else {
prop.put("yourip", "unknown");
}
@ -151,41 +150,41 @@ public final class hello {
serverCore.checkInterruption();
prop.put("yourip", clientip);
oseed.put(yacySeed.IP, clientip);
urls = yacyClient.queryUrlCount(oseed);
remoteSeed.put(yacySeed.IP, clientip);
urls = yacyClient.queryUrlCount(remoteSeed);
}
}
// System.out.println("YACYHELLO: YOUR IP=" + clientip);
// set lastseen value (we have seen that peer, it contacted us!)
oseed.setLastSeenUTC();
remoteSeed.setLastSeenUTC();
// assign status
if (urls >= 0) {
if (oseed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) == null) {
if (remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) == null) {
prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_SENIOR);
oseed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR);
} else if (oseed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_PRINCIPAL).equals(yacySeed.PEERTYPE_PRINCIPAL)) {
remoteSeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR);
} else if (remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_PRINCIPAL).equals(yacySeed.PEERTYPE_PRINCIPAL)) {
prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_PRINCIPAL);
} else {
prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_SENIOR);
oseed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR);
remoteSeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR);
}
// connect the seed
yacyCore.peerActions.peerArrival(oseed, true);
yacyCore.peerActions.peerArrival(remoteSeed, true);
} else {
prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_JUNIOR);
yacyCore.peerActions.juniorConnects++; // update statistics
oseed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
yacyCore.log.logInfo("hello: responded remote junior peer '" + oseed.getName() + "' from " + reportedip);
remoteSeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
yacyCore.log.logInfo("hello: responded remote junior peer '" + remoteSeed.getName() + "' from " + reportedip);
// no connection here, instead store junior in connection cache
if ((oseed.hash != null) && (oseed.isProper() == null)) {
yacyCore.peerActions.peerPing(oseed);
if ((remoteSeed.hash != null) && (remoteSeed.isProper() == null)) {
yacyCore.peerActions.peerPing(remoteSeed);
}
}
yacyCore.peerActions.setUserAgent(clientip, userAgent);
if (!((String)prop.get(yacySeed.YOURTYPE)).equals(reportedPeerType)) {
yacyCore.log.logInfo("hello: changing remote peer '" + oseed.getName() +
yacyCore.log.logInfo("hello: changing remote peer '" + remoteSeed.getName() +
"' [" + reportedip +
"] peerType from '" + reportedPeerType +
"' to '" + prop.get(yacySeed.YOURTYPE) + "'.");

22
htroot/yacy/message.java
Unescape View File

@ -73,14 +73,13 @@ public final class message {
}
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
// defaults
prop.put("response", "-1"); // request rejected
if (post == null || env == null) { return null; }
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) {
return prop;
}
// return variable that accumulates replacements
plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
if ((post == null) || (env == null)) return prop;
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
String process = post.get("process", "permission");
String key = post.get("key", "");
@ -95,14 +94,15 @@ public final class message {
// check if we are the right target and requester has correct information about this peer
if ((yacyCore.seedDB.mySeed() == null) || (!(yacyCore.seedDB.mySeed().hash.equals(youare)))) {
// this request has a wrong target
prop.put("response", "-1"); // request rejected
return prop;
}
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
if (sb.isRobinsonMode() &&
!(sb.isPublicRobinson() ||
sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))) {
if ((sb.isRobinsonMode()) &&
(!((sb.isPublicRobinson()) ||
(sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))))) {
// if we are a robinson cluster, answer only if this client is known by our network definition
prop.put("response", "-1"); // request rejected
return prop;
}

27
htroot/yacy/profile.java
Unescape View File

@ -64,32 +64,29 @@ public final class profile {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
// defaults
prop.put("list", "0");
serverObjects prop = new serverObjects();
plasmaSwitchboard sb = (plasmaSwitchboard) env;
if ((post == null) || (env == null)) return prop;
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) {
return prop;
}
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
if (sb.isRobinsonMode() &&
!sb.isPublicRobinson() &&
!sb.isInMyCluster((String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP))) {
if ((sb.isRobinsonMode()) &&
(!sb.isPublicRobinson()) &&
(!sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))) {
// if we are a robinson cluster, answer only if this client is known by our network definition
prop.put("list", "0");
return prop;
}
Properties profile = new Properties();
int count = 0;
String key = "";
String value = "";
int count=0;
String key="";
String value="";
FileInputStream fileIn = null;
try {
fileIn = new FileInputStream(new File("DATA/SETTINGS/profile.txt"));
profile.load(fileIn);
} catch (IOException e) {
} catch(IOException e) {
} finally {
if (fileIn != null) try { fileIn.close(); fileIn = null; } catch (Exception e) {}
}

33
htroot/yacy/query.java
Unescape View File

@ -53,22 +53,22 @@ import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNetwork;
import de.anomic.yacy.yacySeed;
public final class query {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch ss) {
if (post == null || ss == null) { return null; }
// return variable that accumulates replacements
final plasmaSwitchboard sb = (plasmaSwitchboard) ss;
final serverObjects prop = new serverObjects();
if ((post == null) || (ss == null)) return prop;
if (!yacyNetwork.authentifyRequest(post, ss)) return prop;
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) {
return prop;
}
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
if (sb.isRobinsonMode() &&
!sb.isPublicRobinson() &&
!sb.isInMyCluster((String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP))) {
if ((sb.isRobinsonMode()) &&
(!sb.isPublicRobinson()) &&
(!sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))) {
// if we are a robinson cluster, answer only if we are public robinson peers,
// or we are a private cluster and the requester is in our cluster.
// if we don't answer, the remote peer will recognize us as junior peer,
@ -79,20 +79,11 @@ public final class query {
// System.out.println("YACYQUERY: RECEIVED POST = " + ((post == null) ? "NULL" : post.toString()));
final String ohash = post.get("iam", ""); // complete seed of the requesting peer
// final String iam = post.get("iam", ""); // complete seed of the requesting peer
final String youare = post.get("youare", ""); // seed hash of the target peer, used for testing network stability
// final String key = post.get("key", ""); // transmission key for response
final String obj = post.get("object", ""); // keyword for query subject
final String qenv = post.get("env", ""); // argument to query
final yacySeed oseed = yacyCore.seedDB.get(ohash);
if (oseed == null) {
prop.put("response", "0");
return prop;
} else {
oseed.setFlagDirectConnect(true);
oseed.setLastSeenUTC();
}
final String env = post.get("env", ""); // argument to query
prop.put("mytime", serverDate.formatShortSecond());
@ -107,7 +98,7 @@ public final class query {
if (obj.equals("rwiurlcount")) {
// the total number of different urls in the rwi is returned
// <env> shall contain a word hash, the number of assigned lurls to this hash is returned
prop.put("response", sb.wordIndex.indexSize(qenv));
prop.put("response", sb.wordIndex.indexSize(env));
return prop;
}

109
htroot/yacy/search.java
Unescape View File

@ -60,69 +60,34 @@ public final class search {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
// defaults
prop.put("links", "");
prop.put("linkcount", "0");
prop.put("references", "");
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) {
return prop;
}
String client = (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP);
// test:
// http://localhost:8080/yacy/search.html?query=4galTpdpDM5Q (search for linux)
// http://localhost:8080/yacy/search.html?query=gh8DKIhGKXws (search for book)
// http://localhost:8080/yacy/search.html?query=UEhMGfGv2vOE (search for kernel)
// http://localhost:8080/yacy/search.html?query=ZX-LjaYo74PP (search for help)
// http://localhost:8080/yacy/search.html?query=uDqIalxDfM2a (search for mail)
// http://localhost:8080/yacy/search.html?query=4galTpdpDM5Qgh8DKIhGKXws&abstracts=auto (search for linux and book, generate abstract automatically)
// http://localhost:8080/yacy/search.html?query=&abstracts=4galTpdpDM5Q (only abstracts for linux)
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
if (sb.isRobinsonMode() &&
!(sb.isPublicRobinson() ||
sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))) {
// if we are a robinson cluster, answer only if this client is known by our network definition
return prop;
}
final String query = post.get("query", ""); // a string of word hashes that shall be searched and combined
final String abstracts = post.get("abstracts", ""); // a string of word hashes for abstracts that shall be generated, or 'auto' (for maxcount-word), or '' (for none)
if ((query == null || query.length() == 0) & (abstracts == null || abstracts.length() == 0)) {
return prop;
}
// tell all threads to do nothing for a specific time
sb.intermissionAllThreads(3000);
sb.remoteSearchLastAccess = System.currentTimeMillis();
// myseed = complete seed of the requesting peer, key = transmission key for response
final yacySeed oseed = yacySeed.genRemoteSeed(post.get("myseed", ""), post.get("key", ""), true);
// store accessing peer
if (yacyCore.seedDB == null) {
yacyCore.log.logSevere("yacy.search: seed cache not initialized");
} else {
yacyCore.peerActions.peerArrival(oseed, true);
}
serverObjects prop = new serverObjects();
if ((post == null) || (env == null)) return prop;
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
String client = (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP);
//System.out.println("yacy: search received request = " + post.toString());
final String oseed = post.get("myseed", ""); // complete seed of the requesting peer
// final String youare = post.get("youare", ""); // seed hash of the target peer, used for testing network stability
final String key = post.get("key", ""); // transmission key for response
final String query = post.get("query", ""); // a string of word hashes that shall be searched and combined
final String exclude= post.get("exclude", "");// a string of word hashes that shall not be within the search result
String urls = post.get("urls", ""); // a string of url hashes that are preselected for the search: no other may be returned
String abstracts = post.get("abstracts", ""); // a string of word hashes for abstracts that shall be generated, or 'auto' (for maxcount-word), or '' (for none)
// final String fwdep = post.get("fwdep", ""); // forward depth. if "0" then peer may NOT ask another peer for more results
// final String fwden = post.get("fwden", ""); // forward deny, a list of seed hashes. They may NOT be target of forward hopping
String profile = post.get("profile", ""); // remote profile hand-over
String urls = post.get("urls", ""); // a string of url hashes that are preselected for the search: no other may be returned
final String exclude = post.get("exclude", ""); // a string of word hashes that shall not be within the search result
final int count = Math.min(100, post.getInt("count", 10)); // maximum number of wanted results
final int maxdist = post.getInt("maxdist", Integer.MAX_VALUE);
final int maxdist= post.getInt("maxdist", Integer.MAX_VALUE);
final String prefer = post.get("prefer", "");
final String contentdom = post.get("contentdom", "text");
final String filter = post.get("filter", ".*");
final int partitions = post.getInt("partitions", 30);
final String contentdom = post.get("contentdom", "text");
String profile = post.get("profile", ""); // remote profile hand-over
if (profile.length() > 0) profile = crypt.simpleDecode(profile, null);
//final boolean includesnippet = post.get("includesnippet", "false").equals("true");
kelondroBitfield constraint = ((post.containsKey("constraint")) && (post.get("constraint", "").length() > 0)) ? new kelondroBitfield(4, post.get("constraint", "______")) : null;
if (constraint != null) {
// check bad handover parameter from older versions
@ -135,10 +100,39 @@ public final class search {
// final boolean global = ((String) post.get("resource", "global")).equals("global"); // if true, then result may consist of answers from other peers
// Date remoteTime = yacyCore.parseUniversalDate((String) post.get(yacySeed.MYTIME)); // read remote time
// test:
// http://localhost:8080/yacy/search.html?query=4galTpdpDM5Q (search for linux)
// http://localhost:8080/yacy/search.html?query=gh8DKIhGKXws (search for book)
// http://localhost:8080/yacy/search.html?query=UEhMGfGv2vOE (search for kernel)
// http://localhost:8080/yacy/search.html?query=ZX-LjaYo74PP (search for help)
// http://localhost:8080/yacy/search.html?query=uDqIalxDfM2a (search for mail)
// http://localhost:8080/yacy/search.html?query=4galTpdpDM5Qgh8DKIhGKXws&abstracts=auto (search for linux and book, generate abstract automatically)
// http://localhost:8080/yacy/search.html?query=&abstracts=4galTpdpDM5Q (only abstracts for linux)
if ((sb.isRobinsonMode()) &&
(!((sb.isPublicRobinson()) ||
(sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))))) {
// if we are a robinson cluster, answer only if this client is known by our network definition
prop.put("links", "");
prop.put("linkcount", "0");
prop.put("references", "");
return prop;
}
// tell all threads to do nothing for a specific time
sb.intermissionAllThreads(3000);
TreeSet<String> abstractSet = ((abstracts.length() == 0) || (abstracts.equals("auto"))) ? null : plasmaSearchQuery.hashes2Set(abstracts);
// store accessing peer
if (yacyCore.seedDB == null) {
yacyCore.log.logSevere("yacy.search: seed cache not initialized");
} else {
yacyCore.peerActions.peerArrival(yacySeed.genRemoteSeed(oseed, key, true), true);
}
// prepare search
final TreeSet<String> queryhashes = plasmaSearchQuery.hashes2Set(query);
final TreeSet<String> abstractSet = (abstracts.length() == 0 || abstracts.equals("auto")) ? null : plasmaSearchQuery.hashes2Set(abstracts);
final TreeSet<String> excludehashes = (exclude.length() == 0) ? new TreeSet<String>(kelondroBase64Order.enhancedComparator) : plasmaSearchQuery.hashes2Set(exclude);
final long timestamp = System.currentTimeMillis();
@ -173,6 +167,7 @@ public final class search {
indexabstract.append("indexabstract." + wordhash + "=").append(indexContainer.compressIndex(container, null, 1000).toString()).append(serverCore.CRLF_STRING);
}
}
prop.put("indexcount", "");
prop.put("joincount", "0");
prop.put("references", "");
@ -255,7 +250,14 @@ public final class search {
prop.put("indexabstract", indexabstract.toString());
// prepare result
if (joincount != 0 || accu != null) {
if ((joincount == 0) || (accu == null)) {
// no results
prop.put("links", "");
prop.put("linkcount", "0");
prop.put("references", "");
} else {
// result is a List of urlEntry elements
long timer = System.currentTimeMillis();
StringBuffer links = new StringBuffer();
@ -299,9 +301,10 @@ public final class search {
prop.put("searchtime", System.currentTimeMillis() - timestamp);
final int links = Integer.parseInt(prop.get("linkcount", "0"));
final int links = Integer.parseInt(prop.get("linkcount","0"));
yacyCore.seedDB.mySeed().incSI(links);
yacyCore.seedDB.mySeed().incSU(links);
return prop;
}
}

45
htroot/yacy/transfer.java
Unescape View File

@ -61,19 +61,17 @@ import de.anomic.yacy.yacySeed;
public final class transfer {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) {
return prop;
}
plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
if ((post == null) || (env == null)) return prop;
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
String process = post.get("process", ""); // permission or store
// String key = post.get("key", ""); // a transmission key from the client
String ohash = post.get("iam", ""); // identification of the client (a peer-hash)
//String key = post.get("key", ""); // a transmission key from the client
String otherpeer = post.get("iam", ""); // identification of the client (a peer-hash)
String purpose = post.get("purpose", ""); // declares how the file shall be treated
String filename = post.get("filename", ""); // a name of a file without path
// long filesize = Long.parseLong((String) post.get("filesize", "")); // the size of the file
//long filesize = Long.parseLong((String) post.get("filesize", "")); // the size of the file
prop.put("process", "0");
prop.put("response", "denied"); // reject is default and is overwritten if ok
@ -83,34 +81,29 @@ public final class transfer {
prop.put("process_path", "");
prop.put("process_maxsize", "0");
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
if (sb.isRobinsonMode() || !sb.rankingOn) {
// in a robinson environment, do not answer. We do not do any transfer in a robinson cluster.
return prop;
}
final yacySeed oseed = yacyCore.seedDB.get(ohash);
if (oseed == null) {
yacySeed otherseed = yacyCore.seedDB.get(otherpeer);
if ((otherseed == null) || (filename.indexOf("..") >= 0)) {
// reject unknown peers: this does not appear fair, but anonymous senders are dangerous
sb.getLog().logFine("RankingTransmission: rejected unknown peer '" + ohash + "', current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown"));
return prop;
}
oseed.setFlagDirectConnect(true);
oseed.setLastSeenUTC();
if (filename.indexOf("..") >= 0) {
// reject paths that contain '..' because they are dangerous
sb.getLog().logFine("RankingTransmission: rejected wrong path '" + filename + "' from peer " + oseed.getName() + "/" + oseed.getPublicAddress()+ ", current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown"));
if (otherseed == null) sb.getLog().logFine("RankingTransmission: rejected unknown peer '" + otherpeer + "', current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown"));
if (filename.indexOf("..") >= 0) sb.getLog().logFine("RankingTransmission: rejected wrong path '" + filename + "' from peer " + otherseed.getName() + "/" + otherseed.getPublicAddress()+ ", current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown"));
return prop;
}
String otherpeerName = otherseed.hash + ":" + otherseed.getName();
if (process.equals("permission")) {
prop.put("process", "0");
if (((purpose.equals("crcon")) && (filename.startsWith("CRG")) && (filename.endsWith(".cr.gz"))) || ((filename.startsWith("domlist")) && (filename.endsWith(".txt.gz") || filename.endsWith(".zip")))) {
// consolidation of cr files
//System.out.println("yacy/transfer:post=" + post.toString());
//String cansendprotocol = (String) post.get("can-send-protocol", "http");
String access = kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(ohash + ":" + filename)) + ":" + kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw("" + System.currentTimeMillis()));
String access = kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(otherpeer + ":" + filename)) + ":" + kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw("" + System.currentTimeMillis()));
prop.put("response", "ok");
prop.put("process_access", access);
prop.put("process_address", yacyCore.seedDB.mySeed().getPublicAddress());
@ -118,7 +111,7 @@ public final class transfer {
prop.put("process_path", ""); // currently empty; the store process will find a path
prop.put("process_maxsize", "-1"); // if response is too big we return the size of the file
sb.rankingPermissions.put(serverCodings.encodeMD5Hex(kelondroBase64Order.standardCoder.encodeString(access)), filename);
sb.getLog().logFine("RankingTransmission: granted peer " + oseed.hash + ":" + oseed.getName() + " to send CR file " + filename);
sb.getLog().logFine("RankingTransmission: granted peer " + otherpeerName + " to send CR file " + filename);
}
return prop;
}
@ -136,7 +129,7 @@ public final class transfer {
if ((grantedFile == null) || (!(grantedFile.equals(filename)))) {
// fraud-access of this interface
prop.put("response", "denied");
sb.getLog().logFine("RankingTransmission: denied " + oseed.hash + ":" + oseed.getName() + " to send CR file " + filename + ": wrong access code");
sb.getLog().logFine("RankingTransmission: denied " + otherpeerName + " to send CR file " + filename + ": wrong access code");
} else {
sb.rankingPermissions.remove(accesscode); // not needed any more
File path = new File(sb.rankingPath, plasmaRankingDistribution.CR_OTHER);
@ -148,10 +141,10 @@ public final class transfer {
String md5t = serverCodings.encodeMD5Hex(file);
if (md5t.equals(md5)) {
prop.put("response", "ok");
sb.getLog().logFine("RankingTransmission: received from peer " + oseed.hash + ":" + oseed.getName() + " CR file " + filename);
sb.getLog().logFine("RankingTransmission: received from peer " + otherpeerName + " CR file " + filename);
} else {
prop.put("response", "transfer failure");
sb.getLog().logFine("RankingTransmission: transfer failure from peer " + oseed.hash + ":" + oseed.getName() + " for CR file " + filename);
sb.getLog().logFine("RankingTransmission: transfer failure from peer " + otherpeerName + " for CR file " + filename);
}
}else{
//exploit?
@ -167,7 +160,7 @@ public final class transfer {
}
// wrong access
sb.getLog().logFine("RankingTransmission: rejected unknown process " + process + ":" + purpose + " from peer " + oseed.hash + ":" + oseed.getName());
sb.getLog().logFine("RankingTransmission: rejected unknown process " + process + ":" + purpose + " from peer " + otherpeerName);
return prop;
}

41
htroot/yacy/transferRWI.java
Unescape View File

@ -66,41 +66,28 @@ import de.anomic.yacy.yacySeed;
public final class transferRWI {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws InterruptedException {
// return variable that accumulates replacements
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
final serverObjects prop = new serverObjects();
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) {
return prop;
}
if ((post == null) || (env == null)) return prop;
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
if (!post.containsKey("wordc")) return prop;
if (!post.containsKey("entryc")) return prop;
// request values
final String ohash = post.get("iam", ""); // seed hash of requester
final String iam = post.get("iam", ""); // seed hash of requester
final String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability
// final String key = (String) post.get("key", ""); // transmission key
final int wordc = post.getInt("wordc", 0); // number of different words
final int entryc = post.getInt("entryc", 0); // number of entries in indexes
byte[] indexes = post.get("indexes", "").getBytes(); // the indexes, as list of word entries
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
boolean granted = sb.getConfig("allowReceiveIndex", "false").equals("true");
boolean blockBlacklist = sb.getConfig("indexReceiveBlockBlacklist", "false").equals("true");
boolean checkLimit = sb.getConfigBool("indexDistribution.transferRWIReceiptLimitEnabled", true);
final long cachelimit = sb.getConfigLong("indexDistribution.dhtReceiptLimit", 10000);
final yacySeed oseed = yacyCore.seedDB.get(ohash);
if (oseed == null) {
prop.put("unknownURL", "");
prop.put("result", "busy");
prop.put("pause", "120000");
return prop;
} else {
oseed.setFlagDirectConnect(true);
oseed.setLastSeenUTC();
}
final String oname = ohash + ":" + oseed.getName() + "/" + oseed.getVersion();
final yacySeed otherPeer = yacyCore.seedDB.get(iam);
final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));
// response values
String result = "ok";
@ -108,17 +95,17 @@ public final class transferRWI {
int pause = 10000;
if ((youare == null) || (!youare.equals(yacyCore.seedDB.mySeed().hash))) {
sb.getLog().logInfo("Rejecting RWIs from peer " + oname + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash);
sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash);
result = "wrong_target";
pause = 0;
} else if ((!granted) || (sb.isRobinsonMode())) {
// we dont want to receive indexes
sb.getLog().logInfo("Rejecting RWIs from peer " + oname + ". Not granted.");
sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Not granted.");
result = "not_granted";
pause = 0;
} else if (checkLimit && sb.wordIndex.dhtInCacheSize() > cachelimit) {
// we are too busy to receive indexes
sb.getLog().logInfo("Rejecting RWIs from peer " + oname + ". We are too busy (buffersize=" + sb.wordIndex.dhtInCacheSize() + ").");
sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". We are too busy (buffersize=" + sb.wordIndex.dhtInCacheSize() + ").");
granted = false; // don't accept more words if there are too many words to flush
result = "busy";
pause = 60000;
@ -131,7 +118,7 @@ public final class transferRWI {
} */ else {
// we want and can receive indexes
// log value status (currently added to find outOfMemory error
sb.getLog().logFine("Processing " + indexes.length + " bytes / " + wordc + " words / " + entryc + " entries from " + oname);
sb.getLog().logFine("Processing " + indexes.length + " bytes / " + wordc + " words / " + entryc + " entries from " + otherPeerName);
final long startProcess = System.currentTimeMillis();
// decode request
@ -174,7 +161,7 @@ public final class transferRWI {
// block blacklisted entries
if ((blockBlacklist) && (plasmaSwitchboard.urlBlacklist.hashInBlacklistedCache(plasmaURLPattern.BLACKLIST_DHT, urlHash))) {
int deleted = sb.wordIndex.tryRemoveURLs(urlHash);
yacyCore.log.logFine("transferRWI: blocked blacklisted URLHash '" + urlHash + "' from peer " + oname + "; deleted " + deleted + " URL entries from RWIs");
yacyCore.log.logFine("transferRWI: blocked blacklisted URLHash '" + urlHash + "' from peer " + otherPeerName + "; deleted " + deleted + " URL entries from RWIs");
blocked++;
continue;
}
@ -208,10 +195,10 @@ public final class transferRWI {
}
if (unknownURLs.length() > 0) { unknownURLs.delete(0, 1); }
if ((wordhashes.length == 0) || (received == 0)) {
sb.getLog().logInfo("Received 0 RWIs from " + oname + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs, blocked " + blocked + " RWIs");
sb.getLog().logInfo("Received 0 RWIs from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs, blocked " + blocked + " RWIs");
} else {
final double avdist = (yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, wordhashes[0]) + yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, wordhashes[received - 1])) / 2.0;
sb.getLog().logInfo("Received " + received + " Entries " + wordc + " Words [" + wordhashes[0] + " .. " + wordhashes[received - 1] + "]/" + avdist + " from " + oname + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + "/" + receivedURL + " URLs, blocked " + blocked + " RWIs");
sb.getLog().logInfo("Received " + received + " Entries " + wordc + " Words [" + wordhashes[0] + " .. " + wordhashes[received - 1] + "]/" + avdist + " from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + "/" + receivedURL + " URLs, blocked " + blocked + " RWIs");
}
result = "ok";

56
htroot/yacy/transferURL.java
Unescape View File

@ -62,48 +62,38 @@ import de.anomic.yacy.yacySeed;
public final class transferURL {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws InterruptedException {
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) {
return prop;
}
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws InterruptedException {
long start = System.currentTimeMillis();
long freshdate = 0;
try {freshdate = serverDate.parseShortDay("20061101").getTime();} catch (ParseException e1) {}
// return variable that accumulates replacements
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
final serverObjects prop = new serverObjects();
if ((post == null) || (env == null)) return prop;
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
// request values
final String ohash = post.get("iam", ""); // seed hash of requester
final String iam = post.get("iam", ""); // seed hash of requester
final String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability
// final String key = post.get("key", ""); // transmission key
final int urlc = post.getInt("urlc", 0); // number of transported urls
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
final boolean granted = sb.getConfig("allowReceiveIndex", "false").equals("true");
final boolean blockBlacklist = sb.getConfig("indexReceiveBlockBlacklist", "false").equals("true");
final yacySeed oseed = yacyCore.seedDB.get(ohash);
if (oseed == null) {
prop.put("result", "error_not_granted");
prop.put("pause", "120000");
return prop;
} else {
oseed.setFlagDirectConnect(true);
oseed.setLastSeenUTC();
}
final String oname = ohash + ":" + oseed.getName() + "/" + oseed.getVersion();
// response values
String result = "";
String doublevalues = "0";
final yacySeed otherPeer = yacyCore.seedDB.get(iam);
final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));
if ((youare == null) || (!youare.equals(yacyCore.seedDB.mySeed().hash))) {
sb.getLog().logInfo("Rejecting URLs from peer " + oname + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash);
sb.getLog().logInfo("Rejecting URLs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash);
result = "wrong_target";
} else if ((!granted) || (sb.isRobinsonMode())) {
sb.getLog().logInfo("Rejecting URLs from peer " + oname + ". Not granted.");
sb.getLog().logInfo("Rejecting URLs from peer " + otherPeerName + ". Not granted.");
result = "error_not_granted";
} else {
int received = 0;
@ -118,7 +108,7 @@ public final class transferURL {
// read new lurl-entry
urls = (String) post.get("url" + i);
if (urls == null) {
yacyCore.log.logFine("transferURL: got null URL-string from peer " + oname);
yacyCore.log.logFine("transferURL: got null URL-string from peer " + otherPeerName);
blocked++;
continue;
}
@ -126,7 +116,7 @@ public final class transferURL {
// parse new lurl-entry
lEntry = sb.wordIndex.loadedURL.newEntry(urls);
if (lEntry == null) {
yacyCore.log.logWarning("transferURL: received invalid URL (entry null) from peer " + oname + "\n\tURL Property: " + urls);
yacyCore.log.logWarning("transferURL: received invalid URL (entry null) from peer " + otherPeerName + "\n\tURL Property: " + urls);
blocked++;
continue;
}
@ -134,14 +124,14 @@ public final class transferURL {
// check if entry is well-formed
indexURLEntry.Components comp = lEntry.comp();
if (comp.url() == null) {
yacyCore.log.logWarning("transferURL: received invalid URL from peer " + oname + "\n\tURL Property: " + urls);
yacyCore.log.logWarning("transferURL: received invalid URL from peer " + otherPeerName + "\n\tURL Property: " + urls);
blocked++;
continue;
}
// check whether entry is too old
if (lEntry.freshdate().getTime() <= freshdate) {
yacyCore.log.logFine("transerURL: received too old URL from peer " + oname + ": " + lEntry.freshdate());
yacyCore.log.logFine("transerURL: received too old URL from peer " + otherPeerName + ": " + lEntry.freshdate());
blocked++;
continue;
}
@ -149,7 +139,7 @@ public final class transferURL {
// check if the entry is blacklisted
if ((blockBlacklist) && (plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_DHT, comp.url()))) {
int deleted = sb.wordIndex.tryRemoveURLs(lEntry.hash());
yacyCore.log.logFine("transferURL: blocked blacklisted URL '" + comp.url().toNormalform(false, true) + "' from peer " + oname + "; deleted " + deleted + " URL entries from RWIs");
yacyCore.log.logFine("transferURL: blocked blacklisted URL '" + comp.url().toNormalform(false, true) + "' from peer " + otherPeerName + "; deleted " + deleted + " URL entries from RWIs");
lEntry = null;
blocked++;
continue;
@ -157,7 +147,7 @@ public final class transferURL {
// check if the entry is in our network domain
if (!sb.acceptURL(comp.url())) {
yacyCore.log.logFine("transferURL: blocked URL outside of our domain '" + comp.url().toNormalform(false, true) + "' from peer " + oname);
yacyCore.log.logFine("transferURL: blocked URL outside of our domain '" + comp.url().toNormalform(false, true) + "' from peer " + otherPeerName);
lEntry = null;
blocked++;
continue;
@ -166,8 +156,8 @@ public final class transferURL {
// write entry to database
try {
sb.wordIndex.loadedURL.store(lEntry);
sb.wordIndex.loadedURL.stack(lEntry, ohash, ohash, 3);
yacyCore.log.logFine("transferURL: received URL '" + comp.url().toNormalform(false, true) + "' from peer " + oname);
sb.wordIndex.loadedURL.stack(lEntry, iam, iam, 3);
yacyCore.log.logFine("transferURL: received URL '" + comp.url().toNormalform(false, true) + "' from peer " + otherPeerName);
received++;
} catch (IOException e) {
e.printStackTrace();
@ -179,8 +169,8 @@ public final class transferURL {
// return rewrite properties
final int more = sb.wordIndex.loadedURL.size() - sizeBefore;
doublevalues = Integer.toString(received - more);
sb.getLog().logInfo("Received " + received + " URLs from peer " + oname + " in " + (System.currentTimeMillis() - start) + " ms, Blocked " + blocked + " URLs");
if ((received - more) > 0) sb.getLog().logSevere("Received " + doublevalues + " double URLs from peer " + oname);
sb.getLog().logInfo("Received " + received + " URLs from peer " + otherPeerName + " in " + (System.currentTimeMillis() - start) + " ms, Blocked " + blocked + " URLs");
if ((received - more) > 0) sb.getLog().logSevere("Received " + doublevalues + " double URLs from peer " + otherPeerName);
result = "ok";
}

80
source/de/anomic/yacy/yacyCore.java
Unescape View File

@ -81,21 +81,23 @@ public class yacyCore {
// statics
public static final ThreadGroup publishThreadGroup = new ThreadGroup("publishThreadGroup");
public static yacySeedDB seedDB;
public static yacyNewsPool newsPool;
public static yacySeedDB seedDB = null;
public static yacyNewsPool newsPool = null;
public static final HashMap<String, String> seedUploadMethods = new HashMap<String, String>();
public static yacyPeerActions peerActions;
public static yacyDHTAction dhtAgent;
public static yacyPeerActions peerActions = null;
public static yacyDHTAction dhtAgent = null;
public static serverLog log;
public static long lastOnlineTime;
/** pseudo-random key derived from a time-interval while YaCy startup. */
public static long speedKey;
public static long lastOnlineTime = 0;
/** pseudo-random key derived from a time-interval while YaCy startup*/
public static long speedKey = 0;
public static File yacyDBPath;
public static final Map<String, yacyAccessible> amIAccessibleDB = Collections.synchronizedMap(new HashMap<String, yacyAccessible>()); // Holds PeerHash / yacyAccessible Relations
// constants for PeerPing behaviour
private static final int PING_INITIAL = 16;
private static final int PING_MIN_LASTSEEN = 240000; // in milliseconds
private static final int PING_MIN_PEERSEEN = 2; // min. accessible to force senior
private static final int PING_INITIAL = 10;
private static final int PING_MAX_RUNNING = 3;
private static final int PING_MIN_RUNNING = 1;
private static final int PING_MIN_DBSIZE = 5;
private static final int PING_MIN_PEERSEEN = 1; // min. accessible to force senior
private static final long PING_MAX_DBAGE = 15 * 60 * 1000; // in milliseconds
// public static yacyShare shareManager = null;
@ -369,33 +371,45 @@ public class yacyCore {
int attempts = seedDB.sizeConnected();
// getting a list of peers to contact
if (seedDB.mySeed().isVirgin()) {
attempts = Math.min(attempts, PING_INITIAL);
final Map<String, String> ch = plasmaSwitchboard.getSwitchboard().clusterhashes;
if (seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN).equals(yacySeed.PEERTYPE_VIRGIN)) {
if (attempts > PING_INITIAL) { attempts = PING_INITIAL; }
Map<String, String> ch = plasmaSwitchboard.getSwitchboard().clusterhashes;
seeds = seedDB.seedsByAge(true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection
// add also all peers from cluster if this is a public robinson cluster
if (plasmaSwitchboard.getSwitchboard().clusterhashes != null) {
final Iterator<Map.Entry<String, String>> i = ch.entrySet().iterator();
Iterator<Map.Entry<String, String>> i = ch.entrySet().iterator();
String hash;
Map.Entry<String, String> entry;
yacySeed seed;
while (i.hasNext()) {
entry = i.next();
hash = entry.getKey();
seed = seeds.get(hash);
seed = (yacySeed) seeds.get(hash);
if (seed == null) {
seed = seedDB.get(hash);
if (seed == null) { continue; }
if (seed == null) continue;
}
seed.setAlternativeAddress(entry.getValue());
seed.setAlternativeAddress((String) entry.getValue());
seeds.put(hash, seed);
}
}
} else {
seeds = seedDB.getOldestSeeds(attempts / 5, PING_MIN_LASTSEEN);
int diff = PING_MIN_DBSIZE - amIAccessibleDB.size();
if (diff > PING_MIN_RUNNING) {
diff = Math.min(diff, PING_MAX_RUNNING);
if (attempts > diff) { attempts = diff; }
} else {
if (attempts > PING_MIN_RUNNING) { attempts = PING_MIN_RUNNING; }
}
seeds = seedDB.seedsByAge(false, attempts); // best for seed list maintenance/cleaning
}
if (seeds == null || seeds.size() == 0) { return 0; }
attempts = seeds.size();
if ((seeds == null) || seeds.size() == 0) { return 0; }
if (seeds.size() < attempts) { attempts = seeds.size(); }
// This will try to get Peers that are not currently in amIAccessibleDB
Iterator<yacySeed> si = seeds.values().iterator();
yacySeed seed;
// include a YaCyNews record to my seed
try {
@ -420,14 +434,10 @@ public class yacyCore {
final List<Thread> syncList = Collections.synchronizedList(new LinkedList<Thread>()); // memory for threads
final serverSemaphore sync = new serverSemaphore(attempts);
// This will try to get Peers that are not currently in amIAccessibleDB
Iterator<yacySeed> si = seeds.values().iterator();
yacySeed seed;
// going through the peer list and starting a new publisher thread for each peer
int i = 0;
while (si.hasNext()) {
seed = si.next();
while (si. hasNext()) {
seed = (yacySeed) si.next();
if (seed == null) {
sync.P();
continue;
@ -436,8 +446,8 @@ public class yacyCore {
final String address = seed.getClusterAddress();
log.logFine("HELLO #" + i + " to peer '" + seed.get(yacySeed.NAME, "") + "' at " + address); // debug
final String seederror = seed.isProper();
if (address == null || seederror != null) {
String seederror = seed.isProper();
if ((address == null) || (seederror != null)) {
// we don't like that address, delete it
peerActions.peerDeparture(seed, "peer ping to peer resulted in address = " + address + "; seederror = " + seederror);
sync.P();
@ -480,9 +490,9 @@ public class yacyCore {
final int dbSize;
synchronized (amIAccessibleDB) {
dbSize = amIAccessibleDB.size();
final Iterator<String> ai = amIAccessibleDB.keySet().iterator();
Iterator<String> ai = amIAccessibleDB.keySet().iterator();
while (ai.hasNext()) {
final yacyAccessible ya = amIAccessibleDB.get(ai.next());
yacyAccessible ya = (yacyAccessible) amIAccessibleDB.get(ai.next());
if (ya.lastUpdated < cutofftime) {
ai.remove();
} else {
@ -498,11 +508,11 @@ public class yacyCore {
log.logInfo("PeerPing: I am accessible for " + accessible +
" peer(s), not accessible for " + notaccessible + " peer(s).");
if (accessible + notaccessible > 0) {
if ((accessible + notaccessible) > 0) {
final String newPeerType;
// At least one other Peer told us our type
if (accessible >= PING_MIN_PEERSEEN ||
accessible >= notaccessible) {
if ((accessible >= PING_MIN_PEERSEEN) ||
(accessible >= notaccessible)) {
// We can be reached from a majority of other Peers
if (yacyCore.seedDB.mySeed().isPrincipal()) {
newPeerType = yacySeed.PEERTYPE_PRINCIPAL;
@ -513,11 +523,11 @@ public class yacyCore {
// We cannot be reached from the outside
newPeerType = yacySeed.PEERTYPE_JUNIOR;
}
if (yacyCore.seedDB.mySeed().isType(newPeerType)) {
if (yacyCore.seedDB.mySeed().orVirgin().equals(newPeerType)) {
log.logInfo("PeerPing: myType is " + yacyCore.seedDB.mySeed().orVirgin());
} else {
log.logInfo("PeerPing: changing myType from '" + yacyCore.seedDB.mySeed().orVirgin() + "' to '" + newPeerType + "'");
yacyCore.seedDB.mySeed().setType(newPeerType);
yacyCore.seedDB.mySeed().put(yacySeed.PEERTYPE, newPeerType);
}
} else {
log.logInfo("PeerPing: No data, staying at myType: " + yacyCore.seedDB.mySeed().orVirgin());

36
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -90,7 +90,7 @@ public final class yacySeedDB {
* these hashes all shall be generated by base64.enhancedCoder
*/
public static final int commonHashLength = 12;
public static final int dhtActivityMagic = 48;
public static final int dhtActivityMagic = 32;
public static final String[] sortFields = new String[] {yacySeed.LCOUNT, yacySeed.ICOUNT, yacySeed.UPTIME, yacySeed.VERSION, yacySeed.LASTSEEN};
public static final String[] longaccFields = new String[] {yacySeed.LCOUNT, yacySeed.ICOUNT, yacySeed.ISPEED};
@ -378,40 +378,6 @@ public final class yacySeedDB {
}
}
public HashMap<String, yacySeed> getOldestSeeds(int count, int minage) {
// returns a peerhash/yacySeed relation
try {
final kelondroMScoreCluster<String> seedScore = new kelondroMScoreCluster<String>();
final Iterator<yacySeed> s = seedsConnected(true, false, null, (float) 0.0);
yacySeed ys;
int age;
int searchcount = 1000;
while (s.hasNext() && searchcount-- > 0) {
ys = s.next();
if (ys == null) { continue; }
age = (int) Math.abs(System.currentTimeMillis() + serverDate.dayMillis - ys.getLastSeenUTC());
if (age < minage) { continue; }
seedScore.addScore(ys.hash, age); // the higher age, the older is the peer
}
if (seedScore.size() == 0) { return null; }
// result is now in the score object; create a result vector
final HashMap<String, yacySeed> result = new HashMap<String, yacySeed>();
final Iterator<String> it = seedScore.scores(false);
searchcount = Math.min(count, seedScore.size());
int c = 0;
while (c++ < searchcount && it.hasNext()) {
ys = getConnected(it.next());
if (ys != null && ys.hash != null) { result.put(ys.hash, ys); }
}
return result;
} catch (kelondroException e) {
seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
yacyCore.log.logFine("Internal Error at yacySeedDB.seedsByAge: " + e.getMessage(), e);
return null;
}
}
public int sizeConnected() {
return seedActiveDB.size();
/*

Write Preview
Loading…
Cancel
Save