From 77ba44633271800571b88b2be530acaa072b96cd Mon Sep 17 00:00:00 2001 From: borg-0300 Date: Fri, 8 Feb 2008 14:06:34 +0000 Subject: [PATCH] seedDB helpers update/cleanup git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4461 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/yacy/crawlReceipt.java | 68 +++++++++++++-------------- htroot/yacy/hello.java | 59 +++++++++++------------ htroot/yacy/message.java | 28 +++++------ htroot/yacy/profile.java | 33 +++++++------ htroot/yacy/query.java | 46 +++++++++--------- htroot/yacy/search.java | 11 ++--- htroot/yacy/transfer.java | 32 +++++++------ htroot/yacy/transferRWI.java | 50 +++++++++++--------- htroot/yacy/transferURL.java | 57 +++++++++++----------- source/de/anomic/yacy/yacyClient.java | 19 ++++---- source/de/anomic/yacy/yacyCore.java | 8 ++-- source/de/anomic/yacy/yacySeed.java | 4 +- 12 files changed, 214 insertions(+), 201 deletions(-) diff --git a/htroot/yacy/crawlReceipt.java b/htroot/yacy/crawlReceipt.java index 4e3ace6f8..cfb4adcb2 100644 --- a/htroot/yacy/crawlReceipt.java +++ b/htroot/yacy/crawlReceipt.java @@ -62,25 +62,34 @@ import de.anomic.yacy.yacySeed; public final class crawlReceipt { - /* * this is used to respond on a remote crawling request */ - public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { // return variable that accumulates replacements - plasmaSwitchboard switchboard = (plasmaSwitchboard) env; - serverObjects prop = new serverObjects(); - if ((post == null) || (env == null)) return prop; - if (!yacyNetwork.authentifyRequest(post, env)) return prop; - - serverLog log = switchboard.getLog(); + final serverObjects prop = new serverObjects(); + // defaults + prop.put("delay", "3600"); + + if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) { + return prop; + } + + // seed hash of requester + String ohash = post.get("iam", ""); + if (ohash == null || ohash.length() == 0 ) return prop; + yacySeed oseed = yacyCore.seedDB.get(ohash); + if (oseed == null) return prop; + oseed.setFlagDirectConnect(true); + oseed.setLastSeenUTC(); + + final plasmaSwitchboard sb = (plasmaSwitchboard) env; + serverLog log = sb.getLog(); //int proxyPrefetchDepth = Integer.parseInt(env.getConfig("proxyPrefetchDepth", "0")); //int crawlingDepth = Integer.parseInt(env.getConfig("crawlingDepth", "0")); // request values - String iam = post.get("iam", ""); // seed hash of requester String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability //String process = post.get("process", ""); // process type String key = post.get("key", ""); // transmission key @@ -111,59 +120,50 @@ public final class crawlReceipt { */ - final yacySeed otherPeer = yacyCore.seedDB.get(iam); - if (otherPeer == null) { - prop.put("delay", "3600"); - return prop; - } else { - otherPeer.setLastSeenUTC(); - } - final String otherPeerName = iam + ":" + otherPeer.getName() + "/" + otherPeer.getVersion(); + if ((yacyCore.seedDB.mySeed() == null) || (!(yacyCore.seedDB.mySeed().hash.equals(youare)))) { // no yacy connection / unknown peers - prop.put("delay", "3600"); return prop; } if (propStr == null) { // error with url / wrong key - prop.put("delay", "3600"); return prop; } - if ((switchboard.isRobinsonMode()) && (!switchboard.isInMyCluster(otherPeer))) { + if ((sb.isRobinsonMode()) && (!sb.isInMyCluster(oseed))) { // we reject urls that are from outside our cluster prop.put("delay", "9999"); + return prop; // ??? } // generating a new loaded URL entry - indexURLEntry entry = switchboard.wordIndex.loadedURL.newEntry(propStr); + indexURLEntry entry = sb.wordIndex.loadedURL.newEntry(propStr); if (entry == null) { - log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (entry null) from peer " + iam + "\n\tURL properties: "+ propStr); - prop.put("delay", "3600"); + log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (entry null) from peer " + ohash + "\n\tURL properties: "+ propStr); return prop; } indexURLEntry.Components comp = entry.comp(); if (comp.url() == null) { - log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (url null) for hash " + entry.hash() + " from peer " + iam + "\n\tURL properties: "+ propStr); - prop.put("delay", "3600"); + log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (url null) for hash " + entry.hash() + " from peer " + ohash + "\n\tURL properties: "+ propStr); return prop; } // check if the entry is in our network domain - if (!switchboard.acceptURL(comp.url())) { - log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (url outside of our domain) for hash " + entry.hash() + " from peer " + iam + "\n\tURL properties: "+ propStr); + if (!sb.acceptURL(comp.url())) { + log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (url outside of our domain) for hash " + entry.hash() + " from peer " + ohash + "\n\tURL properties: "+ propStr); prop.put("delay", "9999"); return prop; } if (result.equals("fill")) try { // put new entry into database - switchboard.wordIndex.loadedURL.store(entry); - switchboard.wordIndex.loadedURL.stack(entry, youare, iam, 1); - switchboard.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work has been done + sb.wordIndex.loadedURL.store(entry); + sb.wordIndex.loadedURL.stack(entry, youare, ohash, 1); + sb.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work has been done + final String otherPeerName = ohash + ":" + oseed.getName() + "/" + oseed.getVersion(); log.logInfo("crawlReceipt: RECEIVED RECEIPT from " + otherPeerName + " for URL " + entry.hash() + ":" + comp.url().toNormalform(false, true)); // ready for more @@ -171,16 +171,14 @@ public final class crawlReceipt { return prop; } catch (IOException e) { e.printStackTrace(); - prop.put("delay", "3600"); return prop; } - switchboard.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work is transformed into an error case - plasmaCrawlZURL.Entry ee = switchboard.crawlQueues.errorURL.newEntry(entry.toBalancerEntry(), youare, null, 0, result + ":" + reason); + sb.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work is transformed into an error case + plasmaCrawlZURL.Entry ee = sb.crawlQueues.errorURL.newEntry(entry.toBalancerEntry(), youare, null, 0, result + ":" + reason); ee.store(); - switchboard.crawlQueues.errorURL.push(ee); + sb.crawlQueues.errorURL.push(ee); //switchboard.noticeURL.remove(receivedUrlhash); - prop.put("delay", "3600"); return prop; // return rewrite properties diff --git a/htroot/yacy/hello.java b/htroot/yacy/hello.java index a25bb8f9c..193eaaf2e 100644 --- a/htroot/yacy/hello.java +++ b/htroot/yacy/hello.java @@ -65,10 +65,9 @@ import de.anomic.yacy.yacyVersion; public final class hello { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws InterruptedException { - plasmaSwitchboard sb = (plasmaSwitchboard) env; + // return variable that accumulates replacements serverObjects prop = new serverObjects(); - prop.put("message", "none"); - if ((post == null) || (env == null)) { + if (post == null || env == null) { prop.put("message", "no post or no enviroment"); return prop; } @@ -76,7 +75,8 @@ public final class hello { prop.put("message", "not in my network"); return prop; } - + prop.put("message", "none"); + // final String iam = (String) post.get("iam", ""); // complete seed of the requesting peer // final String mytime = (String) post.get(MYTIME, ""); // final String key = post.get("key", ""); // transmission key for response @@ -90,14 +90,14 @@ public final class hello { prop.put("message", "your seed is too long (" + seed.length() + ")"); return prop; } - final yacySeed remoteSeed = yacySeed.genRemoteSeed(seed, key, false); + final yacySeed oseed = yacySeed.genRemoteSeed(seed, key, false); // System.out.println("YACYHELLO: REMOTESEED=" + ((remoteSeed == null) ? "NULL" : remoteSeed.toString())); - if ((remoteSeed == null) || (remoteSeed.hash == null)) { + if (oseed == null || oseed.hash == null) { prop.put("message", "cannot parse your seed"); return prop; } - + // final String properTest = remoteSeed.isProper(); // The remote peer might not know its IP yet, so don't abort if the IP check fails // if ((properTest != null) && (! properTest.substring(0,1).equals("IP"))) { return null; } @@ -110,18 +110,19 @@ public final class hello { return prop; } final String userAgent = (String) header.get(httpHeader.USER_AGENT, ""); - final String reportedip = remoteSeed.get(yacySeed.IP, ""); - final String reportedPeerType = remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR); - final float clientversion = remoteSeed.getVersion(); + final String reportedip = oseed.get(yacySeed.IP, ""); + final String reportedPeerType = oseed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR); + final float clientversion = oseed.getVersion(); - if ((sb.isRobinsonMode()) && (!sb.isPublicRobinson())) { - // if we are a robinson cluster, answer only if this client is known by our network definition + final plasmaSwitchboard sb = (plasmaSwitchboard) env; + if (sb.isRobinsonMode() && !sb.isPublicRobinson()) { + // if we are a robinson cluster, answer only if this client is known by our network definition prop.put("message", "I am robinson, I do not answer"); return prop; } - + int urls = -1; - if (sb.clusterhashes != null) remoteSeed.setAlternativeAddress((String) sb.clusterhashes.get(remoteSeed.hash)); + if (sb.clusterhashes != null) oseed.setAlternativeAddress((String) sb.clusterhashes.get(oseed.hash)); // if the remote client has reported its own IP address and the client supports // the port forwarding feature (if client version >= 0.383) then we try to @@ -131,8 +132,8 @@ public final class hello { // try first the reportedip, since this may be a connect from a port-forwarding host prop.put("yourip", reportedip); - remoteSeed.put(yacySeed.IP, reportedip); - urls = yacyClient.queryUrlCount(remoteSeed); + oseed.put(yacySeed.IP, reportedip); + urls = yacyClient.queryUrlCount(oseed); } else { prop.put("yourip", "unknown"); } @@ -150,41 +151,41 @@ public final class hello { serverCore.checkInterruption(); prop.put("yourip", clientip); - remoteSeed.put(yacySeed.IP, clientip); - urls = yacyClient.queryUrlCount(remoteSeed); + oseed.put(yacySeed.IP, clientip); + urls = yacyClient.queryUrlCount(oseed); } } // System.out.println("YACYHELLO: YOUR IP=" + clientip); // set lastseen value (we have seen that peer, it contacted us!) - remoteSeed.setLastSeenUTC(); + oseed.setLastSeenUTC(); // assign status if (urls >= 0) { - if (remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) == null) { + if (oseed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) == null) { prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_SENIOR); - remoteSeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); - } else if (remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_PRINCIPAL).equals(yacySeed.PEERTYPE_PRINCIPAL)) { + oseed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); + } else if (oseed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_PRINCIPAL).equals(yacySeed.PEERTYPE_PRINCIPAL)) { prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_PRINCIPAL); } else { prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_SENIOR); - remoteSeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); + oseed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); } // connect the seed - yacyCore.peerActions.peerArrival(remoteSeed, true); + yacyCore.peerActions.peerArrival(oseed, true); } else { prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_JUNIOR); yacyCore.peerActions.juniorConnects++; // update statistics - remoteSeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR); - yacyCore.log.logInfo("hello: responded remote junior peer '" + remoteSeed.getName() + "' from " + reportedip); + oseed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR); + yacyCore.log.logInfo("hello: responded remote junior peer '" + oseed.getName() + "' from " + reportedip); // no connection here, instead store junior in connection cache - if ((remoteSeed.hash != null) && (remoteSeed.isProper() == null)) { - yacyCore.peerActions.peerPing(remoteSeed); + if ((oseed.hash != null) && (oseed.isProper() == null)) { + yacyCore.peerActions.peerPing(oseed); } } yacyCore.peerActions.setUserAgent(clientip, userAgent); if (!((String)prop.get(yacySeed.YOURTYPE)).equals(reportedPeerType)) { - yacyCore.log.logInfo("hello: changing remote peer '" + remoteSeed.getName() + + yacyCore.log.logInfo("hello: changing remote peer '" + oseed.getName() + "' [" + reportedip + "] peerType from '" + reportedPeerType + "' to '" + prop.get(yacySeed.YOURTYPE) + "'."); diff --git a/htroot/yacy/message.java b/htroot/yacy/message.java index 197547c60..9b359e773 100644 --- a/htroot/yacy/message.java +++ b/htroot/yacy/message.java @@ -73,16 +73,17 @@ public final class message { } public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { - if (post == null || env == null) { return null; } - // return variable that accumulates replacements - plasmaSwitchboard sb = (plasmaSwitchboard) env; - serverObjects prop = new serverObjects(); - if ((post == null) || (env == null)) return prop; - if (!yacyNetwork.authentifyRequest(post, env)) return prop; + final serverObjects prop = new serverObjects(); + // defaults + prop.put("response", "-1"); // request rejected + + if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) { + return prop; + } String process = post.get("process", "permission"); - String key = post.get("key", ""); + String key = post.get("key", ""); int messagesize = 10240; int attachmentsize = 0; @@ -94,18 +95,17 @@ public final class message { // check if we are the right target and requester has correct information about this peer if ((yacyCore.seedDB.mySeed() == null) || (!(yacyCore.seedDB.mySeed().hash.equals(youare)))) { // this request has a wrong target - prop.put("response", "-1"); // request rejected return prop; } - if ((sb.isRobinsonMode()) && - (!((sb.isPublicRobinson()) || - (sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))))) { - // if we are a robinson cluster, answer only if this client is known by our network definition - prop.put("response", "-1"); // request rejected + final plasmaSwitchboard sb = (plasmaSwitchboard) env; + if (sb.isRobinsonMode() && + !(sb.isPublicRobinson() || + sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))) { + // if we are a robinson cluster, answer only if this client is known by our network definition return prop; } - + prop.put("messagesize", Integer.toString(messagesize)); prop.put("attachmentsize", Integer.toString(attachmentsize)); diff --git a/htroot/yacy/profile.java b/htroot/yacy/profile.java index dd10cdfd0..8ae3afe61 100644 --- a/htroot/yacy/profile.java +++ b/htroot/yacy/profile.java @@ -64,29 +64,32 @@ public final class profile { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { // return variable that accumulates replacements - serverObjects prop = new serverObjects(); - plasmaSwitchboard sb = (plasmaSwitchboard) env; - if ((post == null) || (env == null)) return prop; - if (!yacyNetwork.authentifyRequest(post, env)) return prop; + final serverObjects prop = new serverObjects(); + // defaults + prop.put("list", "0"); - if ((sb.isRobinsonMode()) && - (!sb.isPublicRobinson()) && - (!sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))) { - // if we are a robinson cluster, answer only if this client is known by our network definition - prop.put("list", "0"); + if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) { return prop; } - + + final plasmaSwitchboard sb = (plasmaSwitchboard) env; + if (sb.isRobinsonMode() && + !sb.isPublicRobinson() && + !sb.isInMyCluster((String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP))) { + // if we are a robinson cluster, answer only if this client is known by our network definition + return prop; + } + Properties profile = new Properties(); - int count=0; - String key=""; - String value=""; + int count = 0; + String key = ""; + String value = ""; FileInputStream fileIn = null; try { fileIn = new FileInputStream(new File("DATA/SETTINGS/profile.txt")); - profile.load(fileIn); - } catch(IOException e) { + profile.load(fileIn); + } catch (IOException e) { } finally { if (fileIn != null) try { fileIn.close(); fileIn = null; } catch (Exception e) {} } diff --git a/htroot/yacy/query.java b/htroot/yacy/query.java index 22b65c729..b7fe0e531 100644 --- a/htroot/yacy/query.java +++ b/htroot/yacy/query.java @@ -57,41 +57,41 @@ import de.anomic.yacy.yacySeed; public final class query { - public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch ss) { - if (post == null || ss == null) { return null; } - + public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { // return variable that accumulates replacements - final plasmaSwitchboard sb = (plasmaSwitchboard) ss; final serverObjects prop = new serverObjects(); - if ((post == null) || (ss == null)) return prop; - if (!yacyNetwork.authentifyRequest(post, ss)) return prop; - - - if ((sb.isRobinsonMode()) && - (!sb.isPublicRobinson()) && - (!sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))) { - // if we are a robinson cluster, answer only if we are public robinson peers, - // or we are a private cluster and the requester is in our cluster. - // if we don't answer, the remote peer will recognize us as junior peer, - // what would mean that our peer ping does not work - prop.put("response", "-1"); // request rejected + + if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) { return prop; } - + + final plasmaSwitchboard sb = (plasmaSwitchboard) env; + if (sb.isRobinsonMode() && + !sb.isPublicRobinson() && + !sb.isInMyCluster((String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP))) { + // if we are a robinson cluster, answer only if we are public robinson peers, + // or we are a private cluster and the requester is in our cluster. + // if we don't answer, the remote peer will recognize us as junior peer, + // what would mean that our peer ping does not work + prop.put("response", "-1"); // request rejected + return prop; + } + // System.out.println("YACYQUERY: RECEIVED POST = " + ((post == null) ? "NULL" : post.toString())); - final String iam = post.get("iam", ""); // complete seed of the requesting peer + final String ohash = post.get("iam", ""); // complete seed of the requesting peer final String youare = post.get("youare", ""); // seed hash of the target peer, used for testing network stability // final String key = post.get("key", ""); // transmission key for response final String obj = post.get("object", ""); // keyword for query subject - final String env = post.get("env", ""); // argument to query + final String qenv = post.get("env", ""); // argument to query - final yacySeed otherPeer = yacyCore.seedDB.get(iam); - if (otherPeer == null) { + final yacySeed oseed = yacyCore.seedDB.get(ohash); + if (oseed == null) { prop.put("response", "0"); return prop; } else { - otherPeer.setLastSeenUTC(); + oseed.setFlagDirectConnect(true); + oseed.setLastSeenUTC(); } prop.put("mytime", serverDate.formatShortSecond()); @@ -107,7 +107,7 @@ public final class query { if (obj.equals("rwiurlcount")) { // the total number of different urls in the rwi is returned // shall contain a word hash, the number of assigned lurls to this hash is returned - prop.put("response", sb.wordIndex.indexSize(env)); + prop.put("response", sb.wordIndex.indexSize(qenv)); return prop; } diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java index ff8e2a33e..bbd25cc8c 100644 --- a/htroot/yacy/search.java +++ b/htroot/yacy/search.java @@ -59,7 +59,7 @@ public final class search { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { // return variable that accumulates replacements - serverObjects prop = new serverObjects(); + final serverObjects prop = new serverObjects(); // defaults prop.put("links", ""); prop.put("linkcount", "0"); @@ -86,8 +86,8 @@ public final class search { return prop; } - final String query = post.get("query", ""); // a string of word hashes that shall be searched and combined - String abstracts = post.get("abstracts", ""); // a string of word hashes for abstracts that shall be generated, or 'auto' (for maxcount-word), or '' (for none) + final String query = post.get("query", ""); // a string of word hashes that shall be searched and combined + final String abstracts = post.get("abstracts", ""); // a string of word hashes for abstracts that shall be generated, or 'auto' (for maxcount-word), or '' (for none) if ((query == null || query.length() == 0) & (abstracts == null || abstracts.length() == 0)) { return prop; } @@ -96,9 +96,8 @@ public final class search { sb.intermissionAllThreads(3000); sb.remoteSearchLastAccess = System.currentTimeMillis(); - final String oseed = post.get("myseed", ""); // complete seed of the requesting peer - final String key = post.get("key", ""); // transmission key for response - final yacySeed opeer = yacySeed.genRemoteSeed(oseed, key, true); + // myseed = complete seed of the requesting peer, key = transmission key for response + final yacySeed opeer = yacySeed.genRemoteSeed(post.get("myseed", ""), post.get("key", ""), true); // store accessing peer if (yacyCore.seedDB == null) { yacyCore.log.logSevere("yacy.search: seed cache not initialized"); diff --git a/htroot/yacy/transfer.java b/htroot/yacy/transfer.java index c3e5a3fb1..c7d76328c 100644 --- a/htroot/yacy/transfer.java +++ b/htroot/yacy/transfer.java @@ -61,11 +61,13 @@ import de.anomic.yacy.yacySeed; public final class transfer { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { - plasmaSwitchboard sb = (plasmaSwitchboard) env; - serverObjects prop = new serverObjects(); - if ((post == null) || (env == null)) return prop; - if (!yacyNetwork.authentifyRequest(post, env)) return prop; - + // return variable that accumulates replacements + final serverObjects prop = new serverObjects(); + + if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) { + return prop; + } + String process = post.get("process", ""); // permission or store // String key = post.get("key", ""); // a transmission key from the client String ohash = post.get("iam", ""); // identification of the client (a peer-hash) @@ -81,22 +83,24 @@ public final class transfer { prop.put("process_path", ""); prop.put("process_maxsize", "0"); + final plasmaSwitchboard sb = (plasmaSwitchboard) env; if (sb.isRobinsonMode() || !sb.rankingOn) { // in a robinson environment, do not answer. We do not do any transfer in a robinson cluster. return prop; } - final yacySeed opeer = yacyCore.seedDB.get(ohash); - if (opeer == null) { + final yacySeed oseed = yacyCore.seedDB.get(ohash); + if (oseed == null) { // reject unknown peers: this does not appear fair, but anonymous senders are dangerous sb.getLog().logFine("RankingTransmission: rejected unknown peer '" + ohash + "', current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown")); return prop; } - opeer.setLastSeenUTC(); + oseed.setFlagDirectConnect(true); + oseed.setLastSeenUTC(); if (filename.indexOf("..") >= 0) { // reject paths that contain '..' because they are dangerous - sb.getLog().logFine("RankingTransmission: rejected wrong path '" + filename + "' from peer " + opeer.getName() + "/" + opeer.getPublicAddress()+ ", current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown")); + sb.getLog().logFine("RankingTransmission: rejected wrong path '" + filename + "' from peer " + oseed.getName() + "/" + oseed.getPublicAddress()+ ", current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown")); return prop; } @@ -114,7 +118,7 @@ public final class transfer { prop.put("process_path", ""); // currently empty; the store process will find a path prop.put("process_maxsize", "-1"); // if response is too big we return the size of the file sb.rankingPermissions.put(serverCodings.encodeMD5Hex(kelondroBase64Order.standardCoder.encodeString(access)), filename); - sb.getLog().logFine("RankingTransmission: granted peer " + opeer.hash + ":" + opeer.getName() + " to send CR file " + filename); + sb.getLog().logFine("RankingTransmission: granted peer " + oseed.hash + ":" + oseed.getName() + " to send CR file " + filename); } return prop; } @@ -132,7 +136,7 @@ public final class transfer { if ((grantedFile == null) || (!(grantedFile.equals(filename)))) { // fraud-access of this interface prop.put("response", "denied"); - sb.getLog().logFine("RankingTransmission: denied " + opeer.hash + ":" + opeer.getName() + " to send CR file " + filename + ": wrong access code"); + sb.getLog().logFine("RankingTransmission: denied " + oseed.hash + ":" + oseed.getName() + " to send CR file " + filename + ": wrong access code"); } else { sb.rankingPermissions.remove(accesscode); // not needed any more File path = new File(sb.rankingPath, plasmaRankingDistribution.CR_OTHER); @@ -144,10 +148,10 @@ public final class transfer { String md5t = serverCodings.encodeMD5Hex(file); if (md5t.equals(md5)) { prop.put("response", "ok"); - sb.getLog().logFine("RankingTransmission: received from peer " + opeer.hash + ":" + opeer.getName() + " CR file " + filename); + sb.getLog().logFine("RankingTransmission: received from peer " + oseed.hash + ":" + oseed.getName() + " CR file " + filename); } else { prop.put("response", "transfer failure"); - sb.getLog().logFine("RankingTransmission: transfer failure from peer " + opeer.hash + ":" + opeer.getName() + " for CR file " + filename); + sb.getLog().logFine("RankingTransmission: transfer failure from peer " + oseed.hash + ":" + oseed.getName() + " for CR file " + filename); } }else{ //exploit? @@ -163,7 +167,7 @@ public final class transfer { } // wrong access - sb.getLog().logFine("RankingTransmission: rejected unknown process " + process + ":" + purpose + " from peer " + opeer.hash + ":" + opeer.getName()); + sb.getLog().logFine("RankingTransmission: rejected unknown process " + process + ":" + purpose + " from peer " + oseed.hash + ":" + oseed.getName()); return prop; } diff --git a/htroot/yacy/transferRWI.java b/htroot/yacy/transferRWI.java index 2e796afb9..90b5c80c9 100644 --- a/htroot/yacy/transferRWI.java +++ b/htroot/yacy/transferRWI.java @@ -67,55 +67,59 @@ import de.anomic.yacy.yacySeed; public final class transferRWI { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws InterruptedException { - // return variable that accumulates replacements - final plasmaSwitchboard sb = (plasmaSwitchboard) env; final serverObjects prop = new serverObjects(); - if ((post == null) || (env == null)) return prop; - if (!yacyNetwork.authentifyRequest(post, env)) return prop; + + if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) { + return prop; + } + if (!post.containsKey("wordc")) return prop; if (!post.containsKey("entryc")) return prop; - + // request values - final String iam = post.get("iam", ""); // seed hash of requester - final String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability -// final String key = (String) post.get("key", ""); // transmission key - final int wordc = post.getInt("wordc", 0); // number of different words - final int entryc = post.getInt("entryc", 0); // number of entries in indexes - byte[] indexes = post.get("indexes", "").getBytes(); // the indexes, as list of word entries + final String ohash = post.get("iam", ""); // seed hash of requester + final String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability +// final String key = (String) post.get("key", ""); // transmission key + final int wordc = post.getInt("wordc", 0); // number of different words + final int entryc = post.getInt("entryc", 0); // number of entries in indexes + byte[] indexes = post.get("indexes", "").getBytes(); // the indexes, as list of word entries + + final plasmaSwitchboard sb = (plasmaSwitchboard) env; boolean granted = sb.getConfig("allowReceiveIndex", "false").equals("true"); boolean blockBlacklist = sb.getConfig("indexReceiveBlockBlacklist", "false").equals("true"); boolean checkLimit = sb.getConfigBool("indexDistribution.transferRWIReceiptLimitEnabled", true); final long cachelimit = sb.getConfigLong("indexDistribution.dhtReceiptLimit", 10000); - final yacySeed otherPeer = yacyCore.seedDB.get(iam); - if (otherPeer == null) { + final yacySeed oseed = yacyCore.seedDB.get(ohash); + if (oseed == null) { prop.put("unknownURL", ""); prop.put("result", "busy"); prop.put("pause", "120000"); return prop; } else { - otherPeer.setLastSeenUTC(); + oseed.setFlagDirectConnect(true); + oseed.setLastSeenUTC(); } - final String otherPeerName = iam + ":" + otherPeer.getName() + "/" + otherPeer.getVersion(); + final String oname = ohash + ":" + oseed.getName() + "/" + oseed.getVersion(); // response values String result = "ok"; StringBuffer unknownURLs = new StringBuffer(); int pause = 10000; - + if ((youare == null) || (!youare.equals(yacyCore.seedDB.mySeed().hash))) { - sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash); + sb.getLog().logInfo("Rejecting RWIs from peer " + oname + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash); result = "wrong_target"; pause = 0; } else if ((!granted) || (sb.isRobinsonMode())) { // we dont want to receive indexes - sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Not granted."); + sb.getLog().logInfo("Rejecting RWIs from peer " + oname + ". Not granted."); result = "not_granted"; pause = 0; } else if (checkLimit && sb.wordIndex.dhtInCacheSize() > cachelimit) { // we are too busy to receive indexes - sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". We are too busy (buffersize=" + sb.wordIndex.dhtInCacheSize() + ")."); + sb.getLog().logInfo("Rejecting RWIs from peer " + oname + ". We are too busy (buffersize=" + sb.wordIndex.dhtInCacheSize() + ")."); granted = false; // don't accept more words if there are too many words to flush result = "busy"; pause = 60000; @@ -128,7 +132,7 @@ public final class transferRWI { } */ else { // we want and can receive indexes // log value status (currently added to find outOfMemory error - sb.getLog().logFine("Processing " + indexes.length + " bytes / " + wordc + " words / " + entryc + " entries from " + otherPeerName); + sb.getLog().logFine("Processing " + indexes.length + " bytes / " + wordc + " words / " + entryc + " entries from " + oname); final long startProcess = System.currentTimeMillis(); // decode request @@ -171,7 +175,7 @@ public final class transferRWI { // block blacklisted entries if ((blockBlacklist) && (plasmaSwitchboard.urlBlacklist.hashInBlacklistedCache(plasmaURLPattern.BLACKLIST_DHT, urlHash))) { int deleted = sb.wordIndex.tryRemoveURLs(urlHash); - yacyCore.log.logFine("transferRWI: blocked blacklisted URLHash '" + urlHash + "' from peer " + otherPeerName + "; deleted " + deleted + " URL entries from RWIs"); + yacyCore.log.logFine("transferRWI: blocked blacklisted URLHash '" + urlHash + "' from peer " + oname + "; deleted " + deleted + " URL entries from RWIs"); blocked++; continue; } @@ -205,10 +209,10 @@ public final class transferRWI { } if (unknownURLs.length() > 0) { unknownURLs.delete(0, 1); } if ((wordhashes.length == 0) || (received == 0)) { - sb.getLog().logInfo("Received 0 RWIs from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs, blocked " + blocked + " RWIs"); + sb.getLog().logInfo("Received 0 RWIs from " + oname + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs, blocked " + blocked + " RWIs"); } else { final double avdist = (yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, wordhashes[0]) + yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, wordhashes[received - 1])) / 2.0; - sb.getLog().logInfo("Received " + received + " Entries " + wordc + " Words [" + wordhashes[0] + " .. " + wordhashes[received - 1] + "]/" + avdist + " from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + "/" + receivedURL + " URLs, blocked " + blocked + " RWIs"); + sb.getLog().logInfo("Received " + received + " Entries " + wordc + " Words [" + wordhashes[0] + " .. " + wordhashes[received - 1] + "]/" + avdist + " from " + oname + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + "/" + receivedURL + " URLs, blocked " + blocked + " RWIs"); } result = "ok"; diff --git a/htroot/yacy/transferURL.java b/htroot/yacy/transferURL.java index 07f2f25ff..2d482de5e 100644 --- a/htroot/yacy/transferURL.java +++ b/htroot/yacy/transferURL.java @@ -62,45 +62,48 @@ import de.anomic.yacy.yacySeed; public final class transferURL { - public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws InterruptedException { + // return variable that accumulates replacements + final serverObjects prop = new serverObjects(); + + if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) { + return prop; + } + long start = System.currentTimeMillis(); long freshdate = 0; try {freshdate = serverDate.parseShortDay("20061101").getTime();} catch (ParseException e1) {} - - // return variable that accumulates replacements - final plasmaSwitchboard sb = (plasmaSwitchboard) env; - final serverObjects prop = new serverObjects(); - if ((post == null) || (env == null)) return prop; - if (!yacyNetwork.authentifyRequest(post, env)) return prop; // request values - final String iam = post.get("iam", ""); // seed hash of requester - final String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability -// final String key = post.get("key", ""); // transmission key - final int urlc = post.getInt("urlc", 0); // number of transported urls + final String ohash = post.get("iam", ""); // seed hash of requester + final String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability +// final String key = post.get("key", ""); // transmission key + final int urlc = post.getInt("urlc", 0); // number of transported urls + + final plasmaSwitchboard sb = (plasmaSwitchboard) env; final boolean granted = sb.getConfig("allowReceiveIndex", "false").equals("true"); final boolean blockBlacklist = sb.getConfig("indexReceiveBlockBlacklist", "false").equals("true"); - final yacySeed otherPeer = yacyCore.seedDB.get(iam); - if (otherPeer == null) { + final yacySeed oseed = yacyCore.seedDB.get(ohash); + if (oseed == null) { prop.put("result", "error_not_granted"); prop.put("pause", "120000"); return prop; } else { - otherPeer.setLastSeenUTC(); + oseed.setFlagDirectConnect(true); + oseed.setLastSeenUTC(); } - final String otherPeerName = iam + ":" + otherPeer.getName() + "/" + otherPeer.getVersion(); + final String oname = ohash + ":" + oseed.getName() + "/" + oseed.getVersion(); // response values String result = ""; String doublevalues = "0"; if ((youare == null) || (!youare.equals(yacyCore.seedDB.mySeed().hash))) { - sb.getLog().logInfo("Rejecting URLs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash); + sb.getLog().logInfo("Rejecting URLs from peer " + oname + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash); result = "wrong_target"; } else if ((!granted) || (sb.isRobinsonMode())) { - sb.getLog().logInfo("Rejecting URLs from peer " + otherPeerName + ". Not granted."); + sb.getLog().logInfo("Rejecting URLs from peer " + oname + ". Not granted."); result = "error_not_granted"; } else { int received = 0; @@ -115,7 +118,7 @@ public final class transferURL { // read new lurl-entry urls = (String) post.get("url" + i); if (urls == null) { - yacyCore.log.logFine("transferURL: got null URL-string from peer " + otherPeerName); + yacyCore.log.logFine("transferURL: got null URL-string from peer " + oname); blocked++; continue; } @@ -123,7 +126,7 @@ public final class transferURL { // parse new lurl-entry lEntry = sb.wordIndex.loadedURL.newEntry(urls); if (lEntry == null) { - yacyCore.log.logWarning("transferURL: received invalid URL (entry null) from peer " + otherPeerName + "\n\tURL Property: " + urls); + yacyCore.log.logWarning("transferURL: received invalid URL (entry null) from peer " + oname + "\n\tURL Property: " + urls); blocked++; continue; } @@ -131,14 +134,14 @@ public final class transferURL { // check if entry is well-formed indexURLEntry.Components comp = lEntry.comp(); if (comp.url() == null) { - yacyCore.log.logWarning("transferURL: received invalid URL from peer " + otherPeerName + "\n\tURL Property: " + urls); + yacyCore.log.logWarning("transferURL: received invalid URL from peer " + oname + "\n\tURL Property: " + urls); blocked++; continue; } // check whether entry is too old if (lEntry.freshdate().getTime() <= freshdate) { - yacyCore.log.logFine("transerURL: received too old URL from peer " + otherPeerName + ": " + lEntry.freshdate()); + yacyCore.log.logFine("transerURL: received too old URL from peer " + oname + ": " + lEntry.freshdate()); blocked++; continue; } @@ -146,7 +149,7 @@ public final class transferURL { // check if the entry is blacklisted if ((blockBlacklist) && (plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_DHT, comp.url()))) { int deleted = sb.wordIndex.tryRemoveURLs(lEntry.hash()); - yacyCore.log.logFine("transferURL: blocked blacklisted URL '" + comp.url().toNormalform(false, true) + "' from peer " + otherPeerName + "; deleted " + deleted + " URL entries from RWIs"); + yacyCore.log.logFine("transferURL: blocked blacklisted URL '" + comp.url().toNormalform(false, true) + "' from peer " + oname + "; deleted " + deleted + " URL entries from RWIs"); lEntry = null; blocked++; continue; @@ -154,7 +157,7 @@ public final class transferURL { // check if the entry is in our network domain if (!sb.acceptURL(comp.url())) { - yacyCore.log.logFine("transferURL: blocked URL outside of our domain '" + comp.url().toNormalform(false, true) + "' from peer " + otherPeerName); + yacyCore.log.logFine("transferURL: blocked URL outside of our domain '" + comp.url().toNormalform(false, true) + "' from peer " + oname); lEntry = null; blocked++; continue; @@ -163,8 +166,8 @@ public final class transferURL { // write entry to database try { sb.wordIndex.loadedURL.store(lEntry); - sb.wordIndex.loadedURL.stack(lEntry, iam, iam, 3); - yacyCore.log.logFine("transferURL: received URL '" + comp.url().toNormalform(false, true) + "' from peer " + otherPeerName); + sb.wordIndex.loadedURL.stack(lEntry, ohash, ohash, 3); + yacyCore.log.logFine("transferURL: received URL '" + comp.url().toNormalform(false, true) + "' from peer " + oname); received++; } catch (IOException e) { e.printStackTrace(); @@ -176,8 +179,8 @@ public final class transferURL { // return rewrite properties final int more = sb.wordIndex.loadedURL.size() - sizeBefore; doublevalues = Integer.toString(received - more); - sb.getLog().logInfo("Received " + received + " URLs from peer " + otherPeerName + " in " + (System.currentTimeMillis() - start) + " ms, Blocked " + blocked + " URLs"); - if ((received - more) > 0) sb.getLog().logSevere("Received " + doublevalues + " double URLs from peer " + otherPeerName); + sb.getLog().logInfo("Received " + received + " URLs from peer " + oname + " in " + (System.currentTimeMillis() - start) + " ms, Blocked " + blocked + " URLs"); + if ((received - more) > 0) sb.getLog().logSevere("Received " + doublevalues + " double URLs from peer " + oname); result = "ok"; } diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index b7b304b8b..9f9802c96 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -259,6 +259,7 @@ public final class yacyClient { ); if (result == null || result.size() == 0) { return null; } + target.setFlagDirectConnect(true); target.setLastSeenUTC(); //final Date remoteTime = yacyCore.parseUniversalDate((String) result.get(yacySeed.MYTIME)); // read remote time return yacySeed.genRemoteSeed((String) result.get("response"), post.get("key", ""), true); @@ -294,6 +295,7 @@ public final class yacyClient { if (resp == null) { return -1; } else try { + target.setFlagDirectConnect(true); target.setLastSeenUTC(); return Integer.parseInt(resp); } catch (NumberFormatException e) { @@ -334,6 +336,7 @@ public final class yacyClient { if (resp == null) { return -1; } else try { + target.setFlagDirectConnect(true); target.setLastSeenUTC(); return Integer.parseInt(resp); } catch (NumberFormatException e) { @@ -862,7 +865,7 @@ public final class yacyClient { } } - public static HashMap transferIndex(yacySeed targetSeed, indexContainer[] indexes, HashMap urlCache, boolean gzipBody, int timeout) { + public static HashMap transferIndex(yacySeed target, indexContainer[] indexes, HashMap urlCache, boolean gzipBody, int timeout) { HashMap resultObj = new HashMap(); int payloadSize = 0; @@ -882,7 +885,7 @@ public final class yacyClient { } // transfer the RWI without the URLs - HashMap in = transferRWI(targetSeed, indexes, gzipBody, timeout); + HashMap in = transferRWI(target, indexes, gzipBody, timeout); resultObj.put("resultTransferRWI", in); if (in == null) { @@ -897,11 +900,9 @@ public final class yacyClient { return resultObj; } - targetSeed.setLastSeenUTC(); - if (!(result.equals("ok"))) { - targetSeed.setFlagAcceptRemoteIndex(false); - yacyCore.seedDB.update(targetSeed.hash, targetSeed); + target.setFlagAcceptRemoteIndex(false); + yacyCore.seedDB.update(target.hash, target); resultObj.put("result", result); return resultObj; } @@ -926,7 +927,7 @@ public final class yacyClient { } } - in = transferURL(targetSeed, urls, gzipBody, timeout); + in = transferURL(target, urls, gzipBody, timeout); resultObj.put("resultTransferURL", in); if (in == null) { @@ -941,8 +942,8 @@ public final class yacyClient { return resultObj; } if (!(result.equals("ok"))) { - targetSeed.setFlagAcceptRemoteIndex(false); - yacyCore.seedDB.update(targetSeed.hash, targetSeed); + target.setFlagAcceptRemoteIndex(false); + yacyCore.seedDB.update(target.hash, target); resultObj.put("result",result); return resultObj; } diff --git a/source/de/anomic/yacy/yacyCore.java b/source/de/anomic/yacy/yacyCore.java index e742d7057..1bef2c3d8 100644 --- a/source/de/anomic/yacy/yacyCore.java +++ b/source/de/anomic/yacy/yacyCore.java @@ -93,11 +93,11 @@ public class yacyCore { public static File yacyDBPath; public static final Map amIAccessibleDB = Collections.synchronizedMap(new HashMap()); // Holds PeerHash / yacyAccessible Relations // constants for PeerPing behaviour - private static final int PING_INITIAL = 18; + private static final int PING_INITIAL = 12; private static final int PING_MAX_RUNNING = 6; private static final int PING_MIN_RUNNING = 3; - private static final int PING_MIN_DBSIZE = 5; - private static final int PING_MIN_PEERSEEN = 1; // min. accessible to force senior + private static final int PING_MIN_DBSIZE = 3; + private static final int PING_MIN_PEERSEEN = 2; // min. accessible to force senior private static final long PING_MAX_DBAGE = 15 * 60 * 1000; // in milliseconds // public static yacyShare shareManager = null; @@ -438,7 +438,7 @@ public class yacyCore { // going through the peer list and starting a new publisher thread for each peer int i = 0; - while (si. hasNext()) { + while (si.hasNext()) { seed = (yacySeed) si.next(); if (seed == null) { sync.P(); diff --git a/source/de/anomic/yacy/yacySeed.java b/source/de/anomic/yacy/yacySeed.java index 17ad265c0..975eeabf4 100644 --- a/source/de/anomic/yacy/yacySeed.java +++ b/source/de/anomic/yacy/yacySeed.java @@ -815,7 +815,7 @@ public class yacySeed { public final String genSeedStr(String key) { // use a default encoding - return this.genSeedStr('b', key); + return this.genSeedStr('z', key); } public final synchronized String genSeedStr(char method, String key) { @@ -838,7 +838,7 @@ public class yacySeed { } public final void save(File f) throws IOException { - final String out = this.genSeedStr('p', null); + final String out = this.genSeedStr('z', null); final FileWriter fw = new FileWriter(f); fw.write(out, 0, out.length()); fw.close();