diff --git a/htroot/yacy/crawlReceipt.java b/htroot/yacy/crawlReceipt.java index cfb4adcb2..eaf825ce0 100644 --- a/htroot/yacy/crawlReceipt.java +++ b/htroot/yacy/crawlReceipt.java @@ -62,34 +62,25 @@ import de.anomic.yacy.yacySeed; public final class crawlReceipt { + /* * this is used to respond on a remote crawling request */ + public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { // return variable that accumulates replacements - final serverObjects prop = new serverObjects(); - // defaults - prop.put("delay", "3600"); - - if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) { - return prop; - } - - // seed hash of requester - String ohash = post.get("iam", ""); - if (ohash == null || ohash.length() == 0 ) return prop; - yacySeed oseed = yacyCore.seedDB.get(ohash); - if (oseed == null) return prop; - oseed.setFlagDirectConnect(true); - oseed.setLastSeenUTC(); - - final plasmaSwitchboard sb = (plasmaSwitchboard) env; - serverLog log = sb.getLog(); + plasmaSwitchboard switchboard = (plasmaSwitchboard) env; + serverObjects prop = new serverObjects(); + if ((post == null) || (env == null)) return prop; + if (!yacyNetwork.authentifyRequest(post, env)) return prop; + + serverLog log = switchboard.getLog(); //int proxyPrefetchDepth = Integer.parseInt(env.getConfig("proxyPrefetchDepth", "0")); //int crawlingDepth = Integer.parseInt(env.getConfig("crawlingDepth", "0")); // request values + String iam = post.get("iam", ""); // seed hash of requester String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability //String process = post.get("process", ""); // process type String key = post.get("key", ""); // transmission key @@ -119,51 +110,54 @@ public final class crawlReceipt { stale - the resource was reloaded but not processed because source had no changes */ - + final yacySeed otherPeer = yacyCore.seedDB.get(iam); + final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion())); if ((yacyCore.seedDB.mySeed() == null) || (!(yacyCore.seedDB.mySeed().hash.equals(youare)))) { // no yacy connection / unknown peers + prop.put("delay", "3600"); return prop; } if (propStr == null) { // error with url / wrong key + prop.put("delay", "3600"); return prop; } - if ((sb.isRobinsonMode()) && (!sb.isInMyCluster(oseed))) { + if ((switchboard.isRobinsonMode()) && (!switchboard.isInMyCluster(otherPeer))) { // we reject urls that are from outside our cluster prop.put("delay", "9999"); - return prop; // ??? } // generating a new loaded URL entry - indexURLEntry entry = sb.wordIndex.loadedURL.newEntry(propStr); + indexURLEntry entry = switchboard.wordIndex.loadedURL.newEntry(propStr); if (entry == null) { - log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (entry null) from peer " + ohash + "\n\tURL properties: "+ propStr); + log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (entry null) from peer " + iam + "\n\tURL properties: "+ propStr); + prop.put("delay", "3600"); return prop; } indexURLEntry.Components comp = entry.comp(); if (comp.url() == null) { - log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (url null) for hash " + entry.hash() + " from peer " + ohash + "\n\tURL properties: "+ propStr); + log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (url null) for hash " + entry.hash() + " from peer " + iam + "\n\tURL properties: "+ propStr); + prop.put("delay", "3600"); return prop; } // check if the entry is in our network domain - if (!sb.acceptURL(comp.url())) { - log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (url outside of our domain) for hash " + entry.hash() + " from peer " + ohash + "\n\tURL properties: "+ propStr); + if (!switchboard.acceptURL(comp.url())) { + log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (url outside of our domain) for hash " + entry.hash() + " from peer " + iam + "\n\tURL properties: "+ propStr); prop.put("delay", "9999"); return prop; } if (result.equals("fill")) try { // put new entry into database - sb.wordIndex.loadedURL.store(entry); - sb.wordIndex.loadedURL.stack(entry, youare, ohash, 1); - sb.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work has been done - final String otherPeerName = ohash + ":" + oseed.getName() + "/" + oseed.getVersion(); + switchboard.wordIndex.loadedURL.store(entry); + switchboard.wordIndex.loadedURL.stack(entry, youare, iam, 1); + switchboard.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work has been done log.logInfo("crawlReceipt: RECEIVED RECEIPT from " + otherPeerName + " for URL " + entry.hash() + ":" + comp.url().toNormalform(false, true)); // ready for more @@ -171,14 +165,16 @@ public final class crawlReceipt { return prop; } catch (IOException e) { e.printStackTrace(); + prop.put("delay", "3600"); return prop; } - sb.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work is transformed into an error case - plasmaCrawlZURL.Entry ee = sb.crawlQueues.errorURL.newEntry(entry.toBalancerEntry(), youare, null, 0, result + ":" + reason); + switchboard.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work is transformed into an error case + plasmaCrawlZURL.Entry ee = switchboard.crawlQueues.errorURL.newEntry(entry.toBalancerEntry(), youare, null, 0, result + ":" + reason); ee.store(); - sb.crawlQueues.errorURL.push(ee); + switchboard.crawlQueues.errorURL.push(ee); //switchboard.noticeURL.remove(receivedUrlhash); + prop.put("delay", "3600"); return prop; // return rewrite properties diff --git a/htroot/yacy/hello.java b/htroot/yacy/hello.java index 193eaaf2e..a25bb8f9c 100644 --- a/htroot/yacy/hello.java +++ b/htroot/yacy/hello.java @@ -65,9 +65,10 @@ import de.anomic.yacy.yacyVersion; public final class hello { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws InterruptedException { - // return variable that accumulates replacements + plasmaSwitchboard sb = (plasmaSwitchboard) env; serverObjects prop = new serverObjects(); - if (post == null || env == null) { + prop.put("message", "none"); + if ((post == null) || (env == null)) { prop.put("message", "no post or no enviroment"); return prop; } @@ -75,8 +76,7 @@ public final class hello { prop.put("message", "not in my network"); return prop; } - prop.put("message", "none"); - + // final String iam = (String) post.get("iam", ""); // complete seed of the requesting peer // final String mytime = (String) post.get(MYTIME, ""); // final String key = post.get("key", ""); // transmission key for response @@ -90,14 +90,14 @@ public final class hello { prop.put("message", "your seed is too long (" + seed.length() + ")"); return prop; } - final yacySeed oseed = yacySeed.genRemoteSeed(seed, key, false); + final yacySeed remoteSeed = yacySeed.genRemoteSeed(seed, key, false); // System.out.println("YACYHELLO: REMOTESEED=" + ((remoteSeed == null) ? "NULL" : remoteSeed.toString())); - if (oseed == null || oseed.hash == null) { + if ((remoteSeed == null) || (remoteSeed.hash == null)) { prop.put("message", "cannot parse your seed"); return prop; } - + // final String properTest = remoteSeed.isProper(); // The remote peer might not know its IP yet, so don't abort if the IP check fails // if ((properTest != null) && (! properTest.substring(0,1).equals("IP"))) { return null; } @@ -110,19 +110,18 @@ public final class hello { return prop; } final String userAgent = (String) header.get(httpHeader.USER_AGENT, ""); - final String reportedip = oseed.get(yacySeed.IP, ""); - final String reportedPeerType = oseed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR); - final float clientversion = oseed.getVersion(); + final String reportedip = remoteSeed.get(yacySeed.IP, ""); + final String reportedPeerType = remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR); + final float clientversion = remoteSeed.getVersion(); - final plasmaSwitchboard sb = (plasmaSwitchboard) env; - if (sb.isRobinsonMode() && !sb.isPublicRobinson()) { - // if we are a robinson cluster, answer only if this client is known by our network definition + if ((sb.isRobinsonMode()) && (!sb.isPublicRobinson())) { + // if we are a robinson cluster, answer only if this client is known by our network definition prop.put("message", "I am robinson, I do not answer"); return prop; } - + int urls = -1; - if (sb.clusterhashes != null) oseed.setAlternativeAddress((String) sb.clusterhashes.get(oseed.hash)); + if (sb.clusterhashes != null) remoteSeed.setAlternativeAddress((String) sb.clusterhashes.get(remoteSeed.hash)); // if the remote client has reported its own IP address and the client supports // the port forwarding feature (if client version >= 0.383) then we try to @@ -132,8 +131,8 @@ public final class hello { // try first the reportedip, since this may be a connect from a port-forwarding host prop.put("yourip", reportedip); - oseed.put(yacySeed.IP, reportedip); - urls = yacyClient.queryUrlCount(oseed); + remoteSeed.put(yacySeed.IP, reportedip); + urls = yacyClient.queryUrlCount(remoteSeed); } else { prop.put("yourip", "unknown"); } @@ -151,41 +150,41 @@ public final class hello { serverCore.checkInterruption(); prop.put("yourip", clientip); - oseed.put(yacySeed.IP, clientip); - urls = yacyClient.queryUrlCount(oseed); + remoteSeed.put(yacySeed.IP, clientip); + urls = yacyClient.queryUrlCount(remoteSeed); } } // System.out.println("YACYHELLO: YOUR IP=" + clientip); // set lastseen value (we have seen that peer, it contacted us!) - oseed.setLastSeenUTC(); + remoteSeed.setLastSeenUTC(); // assign status if (urls >= 0) { - if (oseed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) == null) { + if (remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) == null) { prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_SENIOR); - oseed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); - } else if (oseed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_PRINCIPAL).equals(yacySeed.PEERTYPE_PRINCIPAL)) { + remoteSeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); + } else if (remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_PRINCIPAL).equals(yacySeed.PEERTYPE_PRINCIPAL)) { prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_PRINCIPAL); } else { prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_SENIOR); - oseed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); + remoteSeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); } // connect the seed - yacyCore.peerActions.peerArrival(oseed, true); + yacyCore.peerActions.peerArrival(remoteSeed, true); } else { prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_JUNIOR); yacyCore.peerActions.juniorConnects++; // update statistics - oseed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR); - yacyCore.log.logInfo("hello: responded remote junior peer '" + oseed.getName() + "' from " + reportedip); + remoteSeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR); + yacyCore.log.logInfo("hello: responded remote junior peer '" + remoteSeed.getName() + "' from " + reportedip); // no connection here, instead store junior in connection cache - if ((oseed.hash != null) && (oseed.isProper() == null)) { - yacyCore.peerActions.peerPing(oseed); + if ((remoteSeed.hash != null) && (remoteSeed.isProper() == null)) { + yacyCore.peerActions.peerPing(remoteSeed); } } yacyCore.peerActions.setUserAgent(clientip, userAgent); if (!((String)prop.get(yacySeed.YOURTYPE)).equals(reportedPeerType)) { - yacyCore.log.logInfo("hello: changing remote peer '" + oseed.getName() + + yacyCore.log.logInfo("hello: changing remote peer '" + remoteSeed.getName() + "' [" + reportedip + "] peerType from '" + reportedPeerType + "' to '" + prop.get(yacySeed.YOURTYPE) + "'."); diff --git a/htroot/yacy/message.java b/htroot/yacy/message.java index 9b359e773..197547c60 100644 --- a/htroot/yacy/message.java +++ b/htroot/yacy/message.java @@ -73,17 +73,16 @@ public final class message { } public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { - // return variable that accumulates replacements - final serverObjects prop = new serverObjects(); - // defaults - prop.put("response", "-1"); // request rejected + if (post == null || env == null) { return null; } - if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) { - return prop; - } + // return variable that accumulates replacements + plasmaSwitchboard sb = (plasmaSwitchboard) env; + serverObjects prop = new serverObjects(); + if ((post == null) || (env == null)) return prop; + if (!yacyNetwork.authentifyRequest(post, env)) return prop; String process = post.get("process", "permission"); - String key = post.get("key", ""); + String key = post.get("key", ""); int messagesize = 10240; int attachmentsize = 0; @@ -95,17 +94,18 @@ public final class message { // check if we are the right target and requester has correct information about this peer if ((yacyCore.seedDB.mySeed() == null) || (!(yacyCore.seedDB.mySeed().hash.equals(youare)))) { // this request has a wrong target + prop.put("response", "-1"); // request rejected return prop; } - final plasmaSwitchboard sb = (plasmaSwitchboard) env; - if (sb.isRobinsonMode() && - !(sb.isPublicRobinson() || - sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))) { - // if we are a robinson cluster, answer only if this client is known by our network definition + if ((sb.isRobinsonMode()) && + (!((sb.isPublicRobinson()) || + (sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))))) { + // if we are a robinson cluster, answer only if this client is known by our network definition + prop.put("response", "-1"); // request rejected return prop; } - + prop.put("messagesize", Integer.toString(messagesize)); prop.put("attachmentsize", Integer.toString(attachmentsize)); diff --git a/htroot/yacy/profile.java b/htroot/yacy/profile.java index 8ae3afe61..dd10cdfd0 100644 --- a/htroot/yacy/profile.java +++ b/htroot/yacy/profile.java @@ -64,32 +64,29 @@ public final class profile { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { // return variable that accumulates replacements - final serverObjects prop = new serverObjects(); - // defaults - prop.put("list", "0"); + serverObjects prop = new serverObjects(); + plasmaSwitchboard sb = (plasmaSwitchboard) env; + if ((post == null) || (env == null)) return prop; + if (!yacyNetwork.authentifyRequest(post, env)) return prop; - if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) { + if ((sb.isRobinsonMode()) && + (!sb.isPublicRobinson()) && + (!sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))) { + // if we are a robinson cluster, answer only if this client is known by our network definition + prop.put("list", "0"); return prop; } - - final plasmaSwitchboard sb = (plasmaSwitchboard) env; - if (sb.isRobinsonMode() && - !sb.isPublicRobinson() && - !sb.isInMyCluster((String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP))) { - // if we are a robinson cluster, answer only if this client is known by our network definition - return prop; - } - + Properties profile = new Properties(); - int count = 0; - String key = ""; - String value = ""; + int count=0; + String key=""; + String value=""; FileInputStream fileIn = null; try { fileIn = new FileInputStream(new File("DATA/SETTINGS/profile.txt")); - profile.load(fileIn); - } catch (IOException e) { + profile.load(fileIn); + } catch(IOException e) { } finally { if (fileIn != null) try { fileIn.close(); fileIn = null; } catch (Exception e) {} } diff --git a/htroot/yacy/query.java b/htroot/yacy/query.java index b7fe0e531..b25d9e249 100644 --- a/htroot/yacy/query.java +++ b/htroot/yacy/query.java @@ -53,46 +53,37 @@ import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacyNetwork; -import de.anomic.yacy.yacySeed; public final class query { - public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { + public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch ss) { + if (post == null || ss == null) { return null; } + // return variable that accumulates replacements + final plasmaSwitchboard sb = (plasmaSwitchboard) ss; final serverObjects prop = new serverObjects(); - - if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) { - return prop; - } - - final plasmaSwitchboard sb = (plasmaSwitchboard) env; - if (sb.isRobinsonMode() && - !sb.isPublicRobinson() && - !sb.isInMyCluster((String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP))) { - // if we are a robinson cluster, answer only if we are public robinson peers, - // or we are a private cluster and the requester is in our cluster. - // if we don't answer, the remote peer will recognize us as junior peer, - // what would mean that our peer ping does not work - prop.put("response", "-1"); // request rejected + if ((post == null) || (ss == null)) return prop; + if (!yacyNetwork.authentifyRequest(post, ss)) return prop; + + + if ((sb.isRobinsonMode()) && + (!sb.isPublicRobinson()) && + (!sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))) { + // if we are a robinson cluster, answer only if we are public robinson peers, + // or we are a private cluster and the requester is in our cluster. + // if we don't answer, the remote peer will recognize us as junior peer, + // what would mean that our peer ping does not work + prop.put("response", "-1"); // request rejected return prop; } - + // System.out.println("YACYQUERY: RECEIVED POST = " + ((post == null) ? "NULL" : post.toString())); - final String ohash = post.get("iam", ""); // complete seed of the requesting peer +// final String iam = post.get("iam", ""); // complete seed of the requesting peer final String youare = post.get("youare", ""); // seed hash of the target peer, used for testing network stability // final String key = post.get("key", ""); // transmission key for response final String obj = post.get("object", ""); // keyword for query subject - final String qenv = post.get("env", ""); // argument to query - - final yacySeed oseed = yacyCore.seedDB.get(ohash); - if (oseed == null) { - prop.put("response", "0"); - return prop; - } else { - oseed.setFlagDirectConnect(true); - oseed.setLastSeenUTC(); - } + final String env = post.get("env", ""); // argument to query prop.put("mytime", serverDate.formatShortSecond()); @@ -107,7 +98,7 @@ public final class query { if (obj.equals("rwiurlcount")) { // the total number of different urls in the rwi is returned // shall contain a word hash, the number of assigned lurls to this hash is returned - prop.put("response", sb.wordIndex.indexSize(qenv)); + prop.put("response", sb.wordIndex.indexSize(env)); return prop; } diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java index 8eeabc775..a373b33e0 100644 --- a/htroot/yacy/search.java +++ b/htroot/yacy/search.java @@ -60,17 +60,46 @@ public final class search { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { // return variable that accumulates replacements - final serverObjects prop = new serverObjects(); - // defaults - prop.put("links", ""); - prop.put("linkcount", "0"); - prop.put("references", ""); + final plasmaSwitchboard sb = (plasmaSwitchboard) env; + sb.remoteSearchLastAccess = System.currentTimeMillis(); + + serverObjects prop = new serverObjects(); + if ((post == null) || (env == null)) return prop; + if (!yacyNetwork.authentifyRequest(post, env)) return prop; + String client = (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP); - if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) { - return prop; + //System.out.println("yacy: search received request = " + post.toString()); + + final String oseed = post.get("myseed", ""); // complete seed of the requesting peer +// final String youare = post.get("youare", ""); // seed hash of the target peer, used for testing network stability + final String key = post.get("key", ""); // transmission key for response + final String query = post.get("query", ""); // a string of word hashes that shall be searched and combined + final String exclude= post.get("exclude", "");// a string of word hashes that shall not be within the search result + String urls = post.get("urls", ""); // a string of url hashes that are preselected for the search: no other may be returned + String abstracts = post.get("abstracts", ""); // a string of word hashes for abstracts that shall be generated, or 'auto' (for maxcount-word), or '' (for none) +// final String fwdep = post.get("fwdep", ""); // forward depth. if "0" then peer may NOT ask another peer for more results +// final String fwden = post.get("fwden", ""); // forward deny, a list of seed hashes. They may NOT be target of forward hopping + final int count = Math.min(100, post.getInt("count", 10)); // maximum number of wanted results + final int maxdist= post.getInt("maxdist", Integer.MAX_VALUE); + final String prefer = post.get("prefer", ""); + final String contentdom = post.get("contentdom", "text"); + final String filter = post.get("filter", ".*"); + final int partitions = post.getInt("partitions", 30); + String profile = post.get("profile", ""); // remote profile hand-over + if (profile.length() > 0) profile = crypt.simpleDecode(profile, null); + //final boolean includesnippet = post.get("includesnippet", "false").equals("true"); + kelondroBitfield constraint = ((post.containsKey("constraint")) && (post.get("constraint", "").length() > 0)) ? new kelondroBitfield(4, post.get("constraint", "______")) : null; + if (constraint != null) { + // check bad handover parameter from older versions + boolean allon = true; + for (int i = 0; i < 32; i++) { + if (!constraint.get(i)) {allon = false; break;} + } + if (allon) constraint = null; } - String client = (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP); - +// final boolean global = ((String) post.get("resource", "global")).equals("global"); // if true, then result may consist of answers from other peers +// Date remoteTime = yacyCore.parseUniversalDate((String) post.get(yacySeed.MYTIME)); // read remote time + // test: // http://localhost:8080/yacy/search.html?query=4galTpdpDM5Q (search for linux) // http://localhost:8080/yacy/search.html?query=gh8DKIhGKXws (search for book) @@ -80,71 +109,36 @@ public final class search { // http://localhost:8080/yacy/search.html?query=4galTpdpDM5Qgh8DKIhGKXws&abstracts=auto (search for linux and book, generate abstract automatically) // http://localhost:8080/yacy/search.html?query=&abstracts=4galTpdpDM5Q (only abstracts for linux) - final plasmaSwitchboard sb = (plasmaSwitchboard) env; - if (sb.isRobinsonMode() && - !(sb.isPublicRobinson() || - sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))) { - // if we are a robinson cluster, answer only if this client is known by our network definition - return prop; - } - - final String query = post.get("query", ""); // a string of word hashes that shall be searched and combined - final String abstracts = post.get("abstracts", ""); // a string of word hashes for abstracts that shall be generated, or 'auto' (for maxcount-word), or '' (for none) - if ((query == null || query.length() == 0) & (abstracts == null || abstracts.length() == 0)) { - return prop; + if ((sb.isRobinsonMode()) && + (!((sb.isPublicRobinson()) || + (sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))))) { + // if we are a robinson cluster, answer only if this client is known by our network definition + prop.put("links", ""); + prop.put("linkcount", "0"); + prop.put("references", ""); + return prop; } - + // tell all threads to do nothing for a specific time sb.intermissionAllThreads(3000); - sb.remoteSearchLastAccess = System.currentTimeMillis(); - // myseed = complete seed of the requesting peer, key = transmission key for response - final yacySeed oseed = yacySeed.genRemoteSeed(post.get("myseed", ""), post.get("key", ""), true); + TreeSet abstractSet = ((abstracts.length() == 0) || (abstracts.equals("auto"))) ? null : plasmaSearchQuery.hashes2Set(abstracts); + // store accessing peer if (yacyCore.seedDB == null) { yacyCore.log.logSevere("yacy.search: seed cache not initialized"); } else { - yacyCore.peerActions.peerArrival(oseed, true); - } - -// final String youare = post.get("youare", ""); // seed hash of the target peer, used for testing network stability -// final String fwdep = post.get("fwdep", ""); // forward depth. if "0" then peer may NOT ask another peer for more results -// final String fwden = post.get("fwden", ""); // forward deny, a list of seed hashes. They may NOT be target of forward hopping - String profile = post.get("profile", ""); // remote profile hand-over - String urls = post.get("urls", ""); // a string of url hashes that are preselected for the search: no other may be returned - final String exclude = post.get("exclude", ""); // a string of word hashes that shall not be within the search result - final int count = Math.min(100, post.getInt("count", 10)); // maximum number of wanted results - final int maxdist = post.getInt("maxdist", Integer.MAX_VALUE); - final String prefer = post.get("prefer", ""); - final String filter = post.get("filter", ".*"); - final int partitions = post.getInt("partitions", 30); - final String contentdom = post.get("contentdom", "text"); - - if (profile.length() > 0) profile = crypt.simpleDecode(profile, null); - //final boolean includesnippet = post.get("includesnippet", "false").equals("true"); - - kelondroBitfield constraint = ((post.containsKey("constraint")) && (post.get("constraint", "").length() > 0)) ? new kelondroBitfield(4, post.get("constraint", "______")) : null; - if (constraint != null) { - // check bad handover parameter from older versions - boolean allon = true; - for (int i = 0; i < 32; i++) { - if (!constraint.get(i)) {allon = false; break;} - } - if (allon) constraint = null; + yacyCore.peerActions.peerArrival(yacySeed.genRemoteSeed(oseed, key, true), true); } -// final boolean global = ((String) post.get("resource", "global")).equals("global"); // if true, then result may consist of answers from other peers -// Date remoteTime = yacyCore.parseUniversalDate((String) post.get(yacySeed.MYTIME)); // read remote time - // prepare search final TreeSet queryhashes = plasmaSearchQuery.hashes2Set(query); - final TreeSet abstractSet = (abstracts.length() == 0 || abstracts.equals("auto")) ? null : plasmaSearchQuery.hashes2Set(abstracts); final TreeSet excludehashes = (exclude.length() == 0) ? new TreeSet(kelondroBase64Order.enhancedComparator) : plasmaSearchQuery.hashes2Set(exclude); final long timestamp = System.currentTimeMillis(); - - // prepare a search profile + + // prepare a search profile plasmaSearchRankingProfile rankingProfile = (profile.length() == 0) ? new plasmaSearchRankingProfile(plasmaSearchQuery.contentdomParser(contentdom)) : new plasmaSearchRankingProfile("", profile); - + // prepare an abstract result StringBuffer indexabstract = new StringBuffer(); int indexabstractContainercount = 0; @@ -173,19 +167,20 @@ public final class search { indexabstract.append("indexabstract." + wordhash + "=").append(indexContainer.compressIndex(container, null, 1000).toString()).append(serverCore.CRLF_STRING); } } + prop.put("indexcount", ""); prop.put("joincount", "0"); prop.put("references", ""); - + } else { // retrieve index containers from search request theQuery = new plasmaSearchQuery(null, queryhashes, excludehashes, rankingProfile, maxdist, prefer, plasmaSearchQuery.contentdomParser(contentdom), false, count, 0, filter, plasmaSearchQuery.SEARCHDOM_LOCAL, null, -1, constraint, false, yacyURL.TLD_any_zone_filter, client); theQuery.domType = plasmaSearchQuery.SEARCHDOM_LOCAL; yacyCore.log.logInfo("INIT HASH SEARCH (query-" + abstracts + "): " + plasmaSearchQuery.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.displayResults() + " links"); - + // make event theSearch = plasmaSearchEvent.getEvent(theQuery, rankingProfile, sb.wordIndex, null, true); - + // set statistic details of search result and find best result index set if (theSearch.getRankingResult().getLocalResourceSize() == 0) { prop.put("indexcount", ""); @@ -210,7 +205,7 @@ public final class search { } } prop.put("indexcount", indexcount.toString()); - + if (theSearch.getRankingResult().getLocalResourceSize() == 0) { joincount = 0; prop.put("joincount", "0"); @@ -219,7 +214,7 @@ public final class search { prop.put("joincount", Integer.toString(joincount)); accu = theSearch.completeResults(3000); } - + // generate compressed index for maxcounthash // this is not needed if the search is restricted to specific // urls, because it is a re-search @@ -240,7 +235,7 @@ public final class search { } } if (partitions > 0) sb.requestedQueries = sb.requestedQueries + 1d / partitions; // increase query counter - + // prepare reference hints long timer = System.currentTimeMillis(); Set ws = theSearch.references(10); @@ -253,9 +248,16 @@ public final class search { serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), "reference collection", ws.size(), System.currentTimeMillis() - timer)); } prop.put("indexabstract", indexabstract.toString()); - + // prepare result - if (joincount != 0 || accu != null) { + if ((joincount == 0) || (accu == null)) { + + // no results + prop.put("links", ""); + prop.put("linkcount", "0"); + prop.put("references", ""); + + } else { // result is a List of urlEntry elements long timer = System.currentTimeMillis(); StringBuffer links = new StringBuffer(); @@ -272,7 +274,7 @@ public final class search { prop.put("linkcount", accu.size()); serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), "result list preparation", accu.size(), System.currentTimeMillis() - timer)); } - + // add information about forward peers prop.put("fwhop", ""); // hops (depth) of forwards that had been performed to construct this result prop.put("fwsrc", ""); // peers that helped to construct this result @@ -289,19 +291,20 @@ public final class search { if (handles == null) handles = new TreeSet(); handles.add(theQuery.handle); sb.remoteSearchTracker.put(client, handles); - + // log yacyCore.log.logInfo("EXIT HASH SEARCH: " + plasmaSearchQuery.anonymizedQueryHashes(theQuery.queryHashes) + " - " + joincount + " links found, " + prop.get("linkcount", "?") + " links selected, " + indexabstractContainercount + " index abstracts, " + (System.currentTimeMillis() - timestamp) + " milliseconds"); - + prop.put("searchtime", System.currentTimeMillis() - timestamp); - final int links = Integer.parseInt(prop.get("linkcount", "0")); + final int links = Integer.parseInt(prop.get("linkcount","0")); yacyCore.seedDB.mySeed().incSI(links); yacyCore.seedDB.mySeed().incSU(links); return prop; } + } diff --git a/htroot/yacy/transfer.java b/htroot/yacy/transfer.java index c7d76328c..e7346fa22 100644 --- a/htroot/yacy/transfer.java +++ b/htroot/yacy/transfer.java @@ -61,19 +61,17 @@ import de.anomic.yacy.yacySeed; public final class transfer { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { - // return variable that accumulates replacements - final serverObjects prop = new serverObjects(); - - if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) { - return prop; - } - - String process = post.get("process", ""); // permission or store -// String key = post.get("key", ""); // a transmission key from the client - String ohash = post.get("iam", ""); // identification of the client (a peer-hash) - String purpose = post.get("purpose", ""); // declares how the file shall be treated - String filename = post.get("filename", ""); // a name of a file without path -// long filesize = Long.parseLong((String) post.get("filesize", "")); // the size of the file + plasmaSwitchboard sb = (plasmaSwitchboard) env; + serverObjects prop = new serverObjects(); + if ((post == null) || (env == null)) return prop; + if (!yacyNetwork.authentifyRequest(post, env)) return prop; + + String process = post.get("process", ""); // permission or store + //String key = post.get("key", ""); // a transmission key from the client + String otherpeer = post.get("iam", ""); // identification of the client (a peer-hash) + String purpose = post.get("purpose", ""); // declares how the file shall be treated + String filename = post.get("filename", ""); // a name of a file without path + //long filesize = Long.parseLong((String) post.get("filesize", "")); // the size of the file prop.put("process", "0"); prop.put("response", "denied"); // reject is default and is overwritten if ok @@ -83,34 +81,29 @@ public final class transfer { prop.put("process_path", ""); prop.put("process_maxsize", "0"); - final plasmaSwitchboard sb = (plasmaSwitchboard) env; if (sb.isRobinsonMode() || !sb.rankingOn) { - // in a robinson environment, do not answer. We do not do any transfer in a robinson cluster. - return prop; + // in a robinson environment, do not answer. We do not do any transfer in a robinson cluster. + return prop; } - final yacySeed oseed = yacyCore.seedDB.get(ohash); - if (oseed == null) { + yacySeed otherseed = yacyCore.seedDB.get(otherpeer); + if ((otherseed == null) || (filename.indexOf("..") >= 0)) { // reject unknown peers: this does not appear fair, but anonymous senders are dangerous - sb.getLog().logFine("RankingTransmission: rejected unknown peer '" + ohash + "', current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown")); - return prop; - } - oseed.setFlagDirectConnect(true); - oseed.setLastSeenUTC(); - - if (filename.indexOf("..") >= 0) { // reject paths that contain '..' because they are dangerous - sb.getLog().logFine("RankingTransmission: rejected wrong path '" + filename + "' from peer " + oseed.getName() + "/" + oseed.getPublicAddress()+ ", current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown")); + if (otherseed == null) sb.getLog().logFine("RankingTransmission: rejected unknown peer '" + otherpeer + "', current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown")); + if (filename.indexOf("..") >= 0) sb.getLog().logFine("RankingTransmission: rejected wrong path '" + filename + "' from peer " + otherseed.getName() + "/" + otherseed.getPublicAddress()+ ", current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown")); return prop; } - + + String otherpeerName = otherseed.hash + ":" + otherseed.getName(); + if (process.equals("permission")) { prop.put("process", "0"); if (((purpose.equals("crcon")) && (filename.startsWith("CRG")) && (filename.endsWith(".cr.gz"))) || ((filename.startsWith("domlist")) && (filename.endsWith(".txt.gz") || filename.endsWith(".zip")))) { // consolidation of cr files //System.out.println("yacy/transfer:post=" + post.toString()); //String cansendprotocol = (String) post.get("can-send-protocol", "http"); - String access = kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(ohash + ":" + filename)) + ":" + kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw("" + System.currentTimeMillis())); + String access = kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(otherpeer + ":" + filename)) + ":" + kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw("" + System.currentTimeMillis())); prop.put("response", "ok"); prop.put("process_access", access); prop.put("process_address", yacyCore.seedDB.mySeed().getPublicAddress()); @@ -118,7 +111,7 @@ public final class transfer { prop.put("process_path", ""); // currently empty; the store process will find a path prop.put("process_maxsize", "-1"); // if response is too big we return the size of the file sb.rankingPermissions.put(serverCodings.encodeMD5Hex(kelondroBase64Order.standardCoder.encodeString(access)), filename); - sb.getLog().logFine("RankingTransmission: granted peer " + oseed.hash + ":" + oseed.getName() + " to send CR file " + filename); + sb.getLog().logFine("RankingTransmission: granted peer " + otherpeerName + " to send CR file " + filename); } return prop; } @@ -136,7 +129,7 @@ public final class transfer { if ((grantedFile == null) || (!(grantedFile.equals(filename)))) { // fraud-access of this interface prop.put("response", "denied"); - sb.getLog().logFine("RankingTransmission: denied " + oseed.hash + ":" + oseed.getName() + " to send CR file " + filename + ": wrong access code"); + sb.getLog().logFine("RankingTransmission: denied " + otherpeerName + " to send CR file " + filename + ": wrong access code"); } else { sb.rankingPermissions.remove(accesscode); // not needed any more File path = new File(sb.rankingPath, plasmaRankingDistribution.CR_OTHER); @@ -148,10 +141,10 @@ public final class transfer { String md5t = serverCodings.encodeMD5Hex(file); if (md5t.equals(md5)) { prop.put("response", "ok"); - sb.getLog().logFine("RankingTransmission: received from peer " + oseed.hash + ":" + oseed.getName() + " CR file " + filename); + sb.getLog().logFine("RankingTransmission: received from peer " + otherpeerName + " CR file " + filename); } else { prop.put("response", "transfer failure"); - sb.getLog().logFine("RankingTransmission: transfer failure from peer " + oseed.hash + ":" + oseed.getName() + " for CR file " + filename); + sb.getLog().logFine("RankingTransmission: transfer failure from peer " + otherpeerName + " for CR file " + filename); } }else{ //exploit? @@ -167,7 +160,7 @@ public final class transfer { } // wrong access - sb.getLog().logFine("RankingTransmission: rejected unknown process " + process + ":" + purpose + " from peer " + oseed.hash + ":" + oseed.getName()); + sb.getLog().logFine("RankingTransmission: rejected unknown process " + process + ":" + purpose + " from peer " + otherpeerName); return prop; } diff --git a/htroot/yacy/transferRWI.java b/htroot/yacy/transferRWI.java index a1632fdc9..03c16fbf0 100644 --- a/htroot/yacy/transferRWI.java +++ b/htroot/yacy/transferRWI.java @@ -66,59 +66,46 @@ import de.anomic.yacy.yacySeed; public final class transferRWI { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws InterruptedException { + // return variable that accumulates replacements + final plasmaSwitchboard sb = (plasmaSwitchboard) env; final serverObjects prop = new serverObjects(); - - if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) { - return prop; - } - + if ((post == null) || (env == null)) return prop; + if (!yacyNetwork.authentifyRequest(post, env)) return prop; if (!post.containsKey("wordc")) return prop; if (!post.containsKey("entryc")) return prop; - + // request values - final String ohash = post.get("iam", ""); // seed hash of requester - final String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability -// final String key = (String) post.get("key", ""); // transmission key - final int wordc = post.getInt("wordc", 0); // number of different words - final int entryc = post.getInt("entryc", 0); // number of entries in indexes - byte[] indexes = post.get("indexes", "").getBytes(); // the indexes, as list of word entries - - final plasmaSwitchboard sb = (plasmaSwitchboard) env; + final String iam = post.get("iam", ""); // seed hash of requester + final String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability +// final String key = (String) post.get("key", ""); // transmission key + final int wordc = post.getInt("wordc", 0); // number of different words + final int entryc = post.getInt("entryc", 0); // number of entries in indexes + byte[] indexes = post.get("indexes", "").getBytes(); // the indexes, as list of word entries boolean granted = sb.getConfig("allowReceiveIndex", "false").equals("true"); boolean blockBlacklist = sb.getConfig("indexReceiveBlockBlacklist", "false").equals("true"); boolean checkLimit = sb.getConfigBool("indexDistribution.transferRWIReceiptLimitEnabled", true); final long cachelimit = sb.getConfigLong("indexDistribution.dhtReceiptLimit", 10000); - - final yacySeed oseed = yacyCore.seedDB.get(ohash); - if (oseed == null) { - prop.put("unknownURL", ""); - prop.put("result", "busy"); - prop.put("pause", "120000"); - return prop; - } else { - oseed.setFlagDirectConnect(true); - oseed.setLastSeenUTC(); - } - final String oname = ohash + ":" + oseed.getName() + "/" + oseed.getVersion(); - + final yacySeed otherPeer = yacyCore.seedDB.get(iam); + final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion())); + // response values String result = "ok"; StringBuffer unknownURLs = new StringBuffer(); int pause = 10000; - + if ((youare == null) || (!youare.equals(yacyCore.seedDB.mySeed().hash))) { - sb.getLog().logInfo("Rejecting RWIs from peer " + oname + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash); + sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash); result = "wrong_target"; pause = 0; } else if ((!granted) || (sb.isRobinsonMode())) { // we dont want to receive indexes - sb.getLog().logInfo("Rejecting RWIs from peer " + oname + ". Not granted."); + sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Not granted."); result = "not_granted"; pause = 0; } else if (checkLimit && sb.wordIndex.dhtInCacheSize() > cachelimit) { // we are too busy to receive indexes - sb.getLog().logInfo("Rejecting RWIs from peer " + oname + ". We are too busy (buffersize=" + sb.wordIndex.dhtInCacheSize() + ")."); + sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". We are too busy (buffersize=" + sb.wordIndex.dhtInCacheSize() + ")."); granted = false; // don't accept more words if there are too many words to flush result = "busy"; pause = 60000; @@ -131,7 +118,7 @@ public final class transferRWI { } */ else { // we want and can receive indexes // log value status (currently added to find outOfMemory error - sb.getLog().logFine("Processing " + indexes.length + " bytes / " + wordc + " words / " + entryc + " entries from " + oname); + sb.getLog().logFine("Processing " + indexes.length + " bytes / " + wordc + " words / " + entryc + " entries from " + otherPeerName); final long startProcess = System.currentTimeMillis(); // decode request @@ -174,7 +161,7 @@ public final class transferRWI { // block blacklisted entries if ((blockBlacklist) && (plasmaSwitchboard.urlBlacklist.hashInBlacklistedCache(plasmaURLPattern.BLACKLIST_DHT, urlHash))) { int deleted = sb.wordIndex.tryRemoveURLs(urlHash); - yacyCore.log.logFine("transferRWI: blocked blacklisted URLHash '" + urlHash + "' from peer " + oname + "; deleted " + deleted + " URL entries from RWIs"); + yacyCore.log.logFine("transferRWI: blocked blacklisted URLHash '" + urlHash + "' from peer " + otherPeerName + "; deleted " + deleted + " URL entries from RWIs"); blocked++; continue; } @@ -208,10 +195,10 @@ public final class transferRWI { } if (unknownURLs.length() > 0) { unknownURLs.delete(0, 1); } if ((wordhashes.length == 0) || (received == 0)) { - sb.getLog().logInfo("Received 0 RWIs from " + oname + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs, blocked " + blocked + " RWIs"); + sb.getLog().logInfo("Received 0 RWIs from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs, blocked " + blocked + " RWIs"); } else { final double avdist = (yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, wordhashes[0]) + yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, wordhashes[received - 1])) / 2.0; - sb.getLog().logInfo("Received " + received + " Entries " + wordc + " Words [" + wordhashes[0] + " .. " + wordhashes[received - 1] + "]/" + avdist + " from " + oname + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + "/" + receivedURL + " URLs, blocked " + blocked + " RWIs"); + sb.getLog().logInfo("Received " + received + " Entries " + wordc + " Words [" + wordhashes[0] + " .. " + wordhashes[received - 1] + "]/" + avdist + " from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + "/" + receivedURL + " URLs, blocked " + blocked + " RWIs"); } result = "ok"; diff --git a/htroot/yacy/transferURL.java b/htroot/yacy/transferURL.java index 2d482de5e..6708b9e8e 100644 --- a/htroot/yacy/transferURL.java +++ b/htroot/yacy/transferURL.java @@ -62,48 +62,38 @@ import de.anomic.yacy.yacySeed; public final class transferURL { + public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws InterruptedException { - // return variable that accumulates replacements - final serverObjects prop = new serverObjects(); - - if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) { - return prop; - } - long start = System.currentTimeMillis(); long freshdate = 0; try {freshdate = serverDate.parseShortDay("20061101").getTime();} catch (ParseException e1) {} + + // return variable that accumulates replacements + final plasmaSwitchboard sb = (plasmaSwitchboard) env; + final serverObjects prop = new serverObjects(); + if ((post == null) || (env == null)) return prop; + if (!yacyNetwork.authentifyRequest(post, env)) return prop; // request values - final String ohash = post.get("iam", ""); // seed hash of requester - final String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability -// final String key = post.get("key", ""); // transmission key - final int urlc = post.getInt("urlc", 0); // number of transported urls - - final plasmaSwitchboard sb = (plasmaSwitchboard) env; + final String iam = post.get("iam", ""); // seed hash of requester + final String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability +// final String key = post.get("key", ""); // transmission key + final int urlc = post.getInt("urlc", 0); // number of transported urls final boolean granted = sb.getConfig("allowReceiveIndex", "false").equals("true"); final boolean blockBlacklist = sb.getConfig("indexReceiveBlockBlacklist", "false").equals("true"); - final yacySeed oseed = yacyCore.seedDB.get(ohash); - if (oseed == null) { - prop.put("result", "error_not_granted"); - prop.put("pause", "120000"); - return prop; - } else { - oseed.setFlagDirectConnect(true); - oseed.setLastSeenUTC(); - } - final String oname = ohash + ":" + oseed.getName() + "/" + oseed.getVersion(); - // response values String result = ""; String doublevalues = "0"; + final yacySeed otherPeer = yacyCore.seedDB.get(iam); + final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion())); + if ((youare == null) || (!youare.equals(yacyCore.seedDB.mySeed().hash))) { - sb.getLog().logInfo("Rejecting URLs from peer " + oname + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash); + sb.getLog().logInfo("Rejecting URLs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash); result = "wrong_target"; } else if ((!granted) || (sb.isRobinsonMode())) { - sb.getLog().logInfo("Rejecting URLs from peer " + oname + ". Not granted."); + sb.getLog().logInfo("Rejecting URLs from peer " + otherPeerName + ". Not granted."); result = "error_not_granted"; } else { int received = 0; @@ -118,7 +108,7 @@ public final class transferURL { // read new lurl-entry urls = (String) post.get("url" + i); if (urls == null) { - yacyCore.log.logFine("transferURL: got null URL-string from peer " + oname); + yacyCore.log.logFine("transferURL: got null URL-string from peer " + otherPeerName); blocked++; continue; } @@ -126,7 +116,7 @@ public final class transferURL { // parse new lurl-entry lEntry = sb.wordIndex.loadedURL.newEntry(urls); if (lEntry == null) { - yacyCore.log.logWarning("transferURL: received invalid URL (entry null) from peer " + oname + "\n\tURL Property: " + urls); + yacyCore.log.logWarning("transferURL: received invalid URL (entry null) from peer " + otherPeerName + "\n\tURL Property: " + urls); blocked++; continue; } @@ -134,14 +124,14 @@ public final class transferURL { // check if entry is well-formed indexURLEntry.Components comp = lEntry.comp(); if (comp.url() == null) { - yacyCore.log.logWarning("transferURL: received invalid URL from peer " + oname + "\n\tURL Property: " + urls); + yacyCore.log.logWarning("transferURL: received invalid URL from peer " + otherPeerName + "\n\tURL Property: " + urls); blocked++; continue; } // check whether entry is too old if (lEntry.freshdate().getTime() <= freshdate) { - yacyCore.log.logFine("transerURL: received too old URL from peer " + oname + ": " + lEntry.freshdate()); + yacyCore.log.logFine("transerURL: received too old URL from peer " + otherPeerName + ": " + lEntry.freshdate()); blocked++; continue; } @@ -149,7 +139,7 @@ public final class transferURL { // check if the entry is blacklisted if ((blockBlacklist) && (plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_DHT, comp.url()))) { int deleted = sb.wordIndex.tryRemoveURLs(lEntry.hash()); - yacyCore.log.logFine("transferURL: blocked blacklisted URL '" + comp.url().toNormalform(false, true) + "' from peer " + oname + "; deleted " + deleted + " URL entries from RWIs"); + yacyCore.log.logFine("transferURL: blocked blacklisted URL '" + comp.url().toNormalform(false, true) + "' from peer " + otherPeerName + "; deleted " + deleted + " URL entries from RWIs"); lEntry = null; blocked++; continue; @@ -157,7 +147,7 @@ public final class transferURL { // check if the entry is in our network domain if (!sb.acceptURL(comp.url())) { - yacyCore.log.logFine("transferURL: blocked URL outside of our domain '" + comp.url().toNormalform(false, true) + "' from peer " + oname); + yacyCore.log.logFine("transferURL: blocked URL outside of our domain '" + comp.url().toNormalform(false, true) + "' from peer " + otherPeerName); lEntry = null; blocked++; continue; @@ -166,8 +156,8 @@ public final class transferURL { // write entry to database try { sb.wordIndex.loadedURL.store(lEntry); - sb.wordIndex.loadedURL.stack(lEntry, ohash, ohash, 3); - yacyCore.log.logFine("transferURL: received URL '" + comp.url().toNormalform(false, true) + "' from peer " + oname); + sb.wordIndex.loadedURL.stack(lEntry, iam, iam, 3); + yacyCore.log.logFine("transferURL: received URL '" + comp.url().toNormalform(false, true) + "' from peer " + otherPeerName); received++; } catch (IOException e) { e.printStackTrace(); @@ -179,8 +169,8 @@ public final class transferURL { // return rewrite properties final int more = sb.wordIndex.loadedURL.size() - sizeBefore; doublevalues = Integer.toString(received - more); - sb.getLog().logInfo("Received " + received + " URLs from peer " + oname + " in " + (System.currentTimeMillis() - start) + " ms, Blocked " + blocked + " URLs"); - if ((received - more) > 0) sb.getLog().logSevere("Received " + doublevalues + " double URLs from peer " + oname); + sb.getLog().logInfo("Received " + received + " URLs from peer " + otherPeerName + " in " + (System.currentTimeMillis() - start) + " ms, Blocked " + blocked + " URLs"); + if ((received - more) > 0) sb.getLog().logSevere("Received " + doublevalues + " double URLs from peer " + otherPeerName); result = "ok"; } diff --git a/source/de/anomic/yacy/yacyCore.java b/source/de/anomic/yacy/yacyCore.java index bf00a05ec..e5f1ed1a8 100644 --- a/source/de/anomic/yacy/yacyCore.java +++ b/source/de/anomic/yacy/yacyCore.java @@ -81,21 +81,23 @@ public class yacyCore { // statics public static final ThreadGroup publishThreadGroup = new ThreadGroup("publishThreadGroup"); - public static yacySeedDB seedDB; - public static yacyNewsPool newsPool; + public static yacySeedDB seedDB = null; + public static yacyNewsPool newsPool = null; public static final HashMap seedUploadMethods = new HashMap(); - public static yacyPeerActions peerActions; - public static yacyDHTAction dhtAgent; + public static yacyPeerActions peerActions = null; + public static yacyDHTAction dhtAgent = null; public static serverLog log; - public static long lastOnlineTime; - /** pseudo-random key derived from a time-interval while YaCy startup. */ - public static long speedKey; + public static long lastOnlineTime = 0; + /** pseudo-random key derived from a time-interval while YaCy startup*/ + public static long speedKey = 0; public static File yacyDBPath; public static final Map amIAccessibleDB = Collections.synchronizedMap(new HashMap()); // Holds PeerHash / yacyAccessible Relations // constants for PeerPing behaviour - private static final int PING_INITIAL = 16; - private static final int PING_MIN_LASTSEEN = 240000; // in milliseconds - private static final int PING_MIN_PEERSEEN = 2; // min. accessible to force senior + private static final int PING_INITIAL = 10; + private static final int PING_MAX_RUNNING = 3; + private static final int PING_MIN_RUNNING = 1; + private static final int PING_MIN_DBSIZE = 5; + private static final int PING_MIN_PEERSEEN = 1; // min. accessible to force senior private static final long PING_MAX_DBAGE = 15 * 60 * 1000; // in milliseconds // public static yacyShare shareManager = null; @@ -369,33 +371,45 @@ public class yacyCore { int attempts = seedDB.sizeConnected(); // getting a list of peers to contact - if (seedDB.mySeed().isVirgin()) { - attempts = Math.min(attempts, PING_INITIAL); - final Map ch = plasmaSwitchboard.getSwitchboard().clusterhashes; + if (seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN).equals(yacySeed.PEERTYPE_VIRGIN)) { + if (attempts > PING_INITIAL) { attempts = PING_INITIAL; } + Map ch = plasmaSwitchboard.getSwitchboard().clusterhashes; seeds = seedDB.seedsByAge(true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection // add also all peers from cluster if this is a public robinson cluster if (plasmaSwitchboard.getSwitchboard().clusterhashes != null) { - final Iterator> i = ch.entrySet().iterator(); + Iterator> i = ch.entrySet().iterator(); String hash; Map.Entry entry; yacySeed seed; while (i.hasNext()) { entry = i.next(); hash = entry.getKey(); - seed = seeds.get(hash); + seed = (yacySeed) seeds.get(hash); if (seed == null) { seed = seedDB.get(hash); - if (seed == null) { continue; } + if (seed == null) continue; } - seed.setAlternativeAddress(entry.getValue()); + seed.setAlternativeAddress((String) entry.getValue()); seeds.put(hash, seed); - } + } } } else { - seeds = seedDB.getOldestSeeds(attempts / 5, PING_MIN_LASTSEEN); + int diff = PING_MIN_DBSIZE - amIAccessibleDB.size(); + if (diff > PING_MIN_RUNNING) { + diff = Math.min(diff, PING_MAX_RUNNING); + if (attempts > diff) { attempts = diff; } + } else { + if (attempts > PING_MIN_RUNNING) { attempts = PING_MIN_RUNNING; } + } + seeds = seedDB.seedsByAge(false, attempts); // best for seed list maintenance/cleaning } - if (seeds == null || seeds.size() == 0) { return 0; } - attempts = seeds.size(); + + if ((seeds == null) || seeds.size() == 0) { return 0; } + if (seeds.size() < attempts) { attempts = seeds.size(); } + + // This will try to get Peers that are not currently in amIAccessibleDB + Iterator si = seeds.values().iterator(); + yacySeed seed; // include a YaCyNews record to my seed try { @@ -420,14 +434,10 @@ public class yacyCore { final List syncList = Collections.synchronizedList(new LinkedList()); // memory for threads final serverSemaphore sync = new serverSemaphore(attempts); - // This will try to get Peers that are not currently in amIAccessibleDB - Iterator si = seeds.values().iterator(); - yacySeed seed; - // going through the peer list and starting a new publisher thread for each peer int i = 0; - while (si.hasNext()) { - seed = si.next(); + while (si. hasNext()) { + seed = (yacySeed) si.next(); if (seed == null) { sync.P(); continue; @@ -436,8 +446,8 @@ public class yacyCore { final String address = seed.getClusterAddress(); log.logFine("HELLO #" + i + " to peer '" + seed.get(yacySeed.NAME, "") + "' at " + address); // debug - final String seederror = seed.isProper(); - if (address == null || seederror != null) { + String seederror = seed.isProper(); + if ((address == null) || (seederror != null)) { // we don't like that address, delete it peerActions.peerDeparture(seed, "peer ping to peer resulted in address = " + address + "; seederror = " + seederror); sync.P(); @@ -480,9 +490,9 @@ public class yacyCore { final int dbSize; synchronized (amIAccessibleDB) { dbSize = amIAccessibleDB.size(); - final Iterator ai = amIAccessibleDB.keySet().iterator(); + Iterator ai = amIAccessibleDB.keySet().iterator(); while (ai.hasNext()) { - final yacyAccessible ya = amIAccessibleDB.get(ai.next()); + yacyAccessible ya = (yacyAccessible) amIAccessibleDB.get(ai.next()); if (ya.lastUpdated < cutofftime) { ai.remove(); } else { @@ -498,11 +508,11 @@ public class yacyCore { log.logInfo("PeerPing: I am accessible for " + accessible + " peer(s), not accessible for " + notaccessible + " peer(s)."); - if (accessible + notaccessible > 0) { + if ((accessible + notaccessible) > 0) { final String newPeerType; // At least one other Peer told us our type - if (accessible >= PING_MIN_PEERSEEN || - accessible >= notaccessible) { + if ((accessible >= PING_MIN_PEERSEEN) || + (accessible >= notaccessible)) { // We can be reached from a majority of other Peers if (yacyCore.seedDB.mySeed().isPrincipal()) { newPeerType = yacySeed.PEERTYPE_PRINCIPAL; @@ -513,11 +523,11 @@ public class yacyCore { // We cannot be reached from the outside newPeerType = yacySeed.PEERTYPE_JUNIOR; } - if (yacyCore.seedDB.mySeed().isType(newPeerType)) { + if (yacyCore.seedDB.mySeed().orVirgin().equals(newPeerType)) { log.logInfo("PeerPing: myType is " + yacyCore.seedDB.mySeed().orVirgin()); } else { log.logInfo("PeerPing: changing myType from '" + yacyCore.seedDB.mySeed().orVirgin() + "' to '" + newPeerType + "'"); - yacyCore.seedDB.mySeed().setType(newPeerType); + yacyCore.seedDB.mySeed().put(yacySeed.PEERTYPE, newPeerType); } } else { log.logInfo("PeerPing: No data, staying at myType: " + yacyCore.seedDB.mySeed().orVirgin()); diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index a12fd89e0..f3ba0e0f6 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -90,7 +90,7 @@ public final class yacySeedDB { * these hashes all shall be generated by base64.enhancedCoder */ public static final int commonHashLength = 12; - public static final int dhtActivityMagic = 48; + public static final int dhtActivityMagic = 32; public static final String[] sortFields = new String[] {yacySeed.LCOUNT, yacySeed.ICOUNT, yacySeed.UPTIME, yacySeed.VERSION, yacySeed.LASTSEEN}; public static final String[] longaccFields = new String[] {yacySeed.LCOUNT, yacySeed.ICOUNT, yacySeed.ISPEED}; @@ -378,40 +378,6 @@ public final class yacySeedDB { } } - public HashMap getOldestSeeds(int count, int minage) { - // returns a peerhash/yacySeed relation - try { - final kelondroMScoreCluster seedScore = new kelondroMScoreCluster(); - final Iterator s = seedsConnected(true, false, null, (float) 0.0); - yacySeed ys; - int age; - int searchcount = 1000; - while (s.hasNext() && searchcount-- > 0) { - ys = s.next(); - if (ys == null) { continue; } - age = (int) Math.abs(System.currentTimeMillis() + serverDate.dayMillis - ys.getLastSeenUTC()); - if (age < minage) { continue; } - seedScore.addScore(ys.hash, age); // the higher age, the older is the peer - } - if (seedScore.size() == 0) { return null; } - - // result is now in the score object; create a result vector - final HashMap result = new HashMap(); - final Iterator it = seedScore.scores(false); - searchcount = Math.min(count, seedScore.size()); - int c = 0; - while (c++ < searchcount && it.hasNext()) { - ys = getConnected(it.next()); - if (ys != null && ys.hash != null) { result.put(ys.hash, ys); } - } - return result; - } catch (kelondroException e) { - seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile); - yacyCore.log.logFine("Internal Error at yacySeedDB.seedsByAge: " + e.getMessage(), e); - return null; - } - } - public int sizeConnected() { return seedActiveDB.size(); /*