diff --git a/htroot/yacy/crawlOrder.java b/htroot/yacy/crawlOrder.java
index 162e8c8d1..34d9b2c07 100644
--- a/htroot/yacy/crawlOrder.java
+++ b/htroot/yacy/crawlOrder.java
@@ -57,6 +57,7 @@ import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.tools.crypt;
import de.anomic.yacy.yacyCore;
+import de.anomic.yacy.yacyNetwork;
import de.anomic.yacy.yacySeed;
public final class crawlOrder {
@@ -65,9 +66,9 @@ public final class crawlOrder {
// return variable that accumulates replacements
plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
-
if ((post == null) || (env == null)) return prop;
-
+ if (!yacyNetwork.authentifyRequest(post, env)) return prop;
+
//int proxyPrefetchDepth = Integer.parseInt(env.getConfig("proxyPrefetchDepth", "0"));
//int crawlingdepth = Integer.parseInt(env.getConfig("crawlingDepth", "0"));
@@ -77,6 +78,8 @@ public final class crawlOrder {
String process = post.get("process", ""); // process type
String key = post.get("key", ""); // transmission key
int orderDepth = post.getInt("depth", 0); // crawl depth
+ String unitName = post.get("network.unit.name", yacySeed.DFLT_NETWORK_UNIT); // the network unit
+ if (!unitName.equals(env.getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT))) return null;
// response values
/*
diff --git a/htroot/yacy/crawlReceipt.java b/htroot/yacy/crawlReceipt.java
index 150d562a1..2819011db 100644
--- a/htroot/yacy/crawlReceipt.java
+++ b/htroot/yacy/crawlReceipt.java
@@ -57,6 +57,7 @@ import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
import de.anomic.tools.crypt;
import de.anomic.yacy.yacyCore;
+import de.anomic.yacy.yacyNetwork;
import de.anomic.yacy.yacySeed;
public final class crawlReceipt {
@@ -73,6 +74,7 @@ public final class crawlReceipt {
serverLog log = switchboard.getLog();
if ((post == null) || (env == null)) return prop;
+ if (!yacyNetwork.authentifyRequest(post, env)) return prop;
//int proxyPrefetchDepth = Integer.parseInt(env.getConfig("proxyPrefetchDepth", "0"));
//int crawlingDepth = Integer.parseInt(env.getConfig("crawlingDepth", "0"));
diff --git a/htroot/yacy/hello.java b/htroot/yacy/hello.java
index 416df6992..31f892147 100644
--- a/htroot/yacy/hello.java
+++ b/htroot/yacy/hello.java
@@ -57,21 +57,22 @@ import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyClient;
import de.anomic.yacy.yacyCore;
+import de.anomic.yacy.yacyNetwork;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyVersion;
public final class hello {
- public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch ss) throws InterruptedException {
- if (post == null || ss == null || yacyCore.seedDB == null || yacyCore.seedDB.mySeed == null) { return null; }
+ public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws InterruptedException {
+ if (post == null || env == null || yacyCore.seedDB == null || yacyCore.seedDB.mySeed == null) { return null; }
- plasmaSwitchboard sb = (plasmaSwitchboard) ss;
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
- if (prop == null) { return null; }
+ if ((post == null) || (env == null)) return prop;
+ if (!yacyNetwork.authentifyRequest(post, env)) return prop;
// final String iam = (String) post.get("iam", ""); // complete seed of the requesting peer
-// final String pattern = (String) post.get("pattern", ""); //
// final String mytime = (String) post.get(MYTIME, ""); //
final String key = post.get("key", ""); // transmission key for response
final String seed = post.get("seed", "");
diff --git a/htroot/yacy/list.java b/htroot/yacy/list.java
index 99e4955cd..818729f6c 100644
--- a/htroot/yacy/list.java
+++ b/htroot/yacy/list.java
@@ -63,20 +63,23 @@ import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacyCore;
+import de.anomic.yacy.yacyNetwork;
import de.anomic.yacy.yacySeed;
public final class list {
- public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch ss) {
- if (post == null || ss == null)
- throw new NullPointerException("post: " + post + ", sb: " + ss);
- plasmaSwitchboard sb = (plasmaSwitchboard) ss;
+ public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
+ if (post == null || env == null)
+ throw new NullPointerException("post: " + post + ", sb: " + env);
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
+ if ((post == null) || (env == null)) return prop;
+ if (!yacyNetwork.authentifyRequest(post, env)) return prop;
final String col = post.get("col", "");
- final File listsPath = new File(ss.getRootPath(),ss.getConfig("listsPath", "DATA/LISTS"));
+ final File listsPath = new File(env.getRootPath(),env.getConfig("listsPath", "DATA/LISTS"));
String otherPeerName = null;
if (post.containsKey("iam")) {
@@ -93,7 +96,7 @@ public final class list {
if (col.equals("black")) {
final StringBuffer out = new StringBuffer();
- final String filenames=ss.getConfig("BlackLists.Shared", "");
+ final String filenames=env.getConfig("BlackLists.Shared", "");
final String[] filenamesarray = filenames.split(",");
if(filenamesarray.length > 0){
@@ -109,7 +112,7 @@ public final class list {
}
// start contrib by [FB]
else if (col.length() == 0 && post.get("list", "").equals("queueUrls")) {
- final URLFetcherStack db = CrawlURLFetchStack_p.getURLFetcherStack(ss);
+ final URLFetcherStack db = CrawlURLFetchStack_p.getURLFetcherStack(env);
final String display = post.get("display", "list");
if (display.equals("list")) {
// list urls from remote crawler queue for other peers
diff --git a/htroot/yacy/message.java b/htroot/yacy/message.java
index 380d8bfab..8ef8923e0 100644
--- a/htroot/yacy/message.java
+++ b/htroot/yacy/message.java
@@ -62,6 +62,7 @@ import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
import de.anomic.tools.crypt;
import de.anomic.yacy.yacyCore;
+import de.anomic.yacy.yacyNetwork;
import de.anomic.yacy.yacySeed;
public final class message {
@@ -71,13 +72,14 @@ public final class message {
return SimpleFormatter.format(date);
}
- public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch ss) {
- if (post == null || ss == null) { return null; }
+ public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
+ if (post == null || env == null) { return null; }
// return variable that accumulates replacements
- plasmaSwitchboard sb = (plasmaSwitchboard) ss;
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
- if (prop == null || sb == null) { return null; }
+ if ((post == null) || (env == null)) return prop;
+ if (!yacyNetwork.authentifyRequest(post, env)) return prop;
String process = post.get("process", "permission");
String key = post.get("key", "");
@@ -153,7 +155,7 @@ public final class message {
yacyCore.seedDB.mySeed.getName(), yacyCore.seedDB.mySeed.hash,
subject, mb));
- messageForwardingViaEmail(ss, msgEntry);
+ messageForwardingViaEmail(env, msgEntry);
// finally write notification
File notifierSource = new File(sb.getRootPath(), sb.getConfig("htRootPath","htroot") + "/env/grafics/message.gif");
diff --git a/htroot/yacy/profile.java b/htroot/yacy/profile.java
index d99a64621..4af2135b8 100644
--- a/htroot/yacy/profile.java
+++ b/htroot/yacy/profile.java
@@ -59,14 +59,16 @@ import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
+import de.anomic.yacy.yacyNetwork;
public final class profile {
- public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch ss) {
+ public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
// return variable that accumulates replacements
serverObjects prop = new serverObjects();
- plasmaSwitchboard sb = (plasmaSwitchboard) ss;
- if (prop == null) { return null; }
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
+ if ((post == null) || (env == null)) return prop;
+ if (!yacyNetwork.authentifyRequest(post, env)) return prop;
if ((sb.isRobinsonMode()) &&
(!sb.isPublicRobinson()) &&
diff --git a/htroot/yacy/query.java b/htroot/yacy/query.java
index a57f07248..a4b9ddfdc 100644
--- a/htroot/yacy/query.java
+++ b/htroot/yacy/query.java
@@ -53,6 +53,7 @@ import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
+import de.anomic.yacy.yacyNetwork;
public final class query {
@@ -61,8 +62,10 @@ public final class query {
// return variable that accumulates replacements
final plasmaSwitchboard sb = (plasmaSwitchboard) ss;
- if (sb == null) { return null; }
final serverObjects prop = new serverObjects();
+ if ((post == null) || (ss == null)) return prop;
+ if (!yacyNetwork.authentifyRequest(post, ss)) return prop;
+
if ((sb.isRobinsonMode()) &&
(!sb.isPublicRobinson()) &&
diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java
index 0bec68676..1e90cdf59 100644
--- a/htroot/yacy/search.java
+++ b/htroot/yacy/search.java
@@ -72,16 +72,18 @@ import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyDHTAction;
+import de.anomic.yacy.yacyNetwork;
import de.anomic.yacy.yacySeed;
import de.anomic.tools.crypt;
public final class search {
- public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch ss) {
- if (post == null || ss == null) { return null; }
-
+ public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
// return variable that accumulates replacements
- final plasmaSwitchboard sb = (plasmaSwitchboard) ss;
+ final plasmaSwitchboard sb = (plasmaSwitchboard) env;
+ serverObjects prop = new serverObjects();
+ if ((post == null) || (env == null)) return prop;
+ if (!yacyNetwork.authentifyRequest(post, env)) return prop;
//System.out.println("yacy: search received request = " + post.toString());
@@ -113,9 +115,7 @@ public final class search {
// http://localhost:8080/yacy/search.html?query=gh8DKIhGKXws (search for book)
// http://localhost:8080/yacy/search.html?query=4galTpdpDM5Qgh8DKIhGKXws&abstracts=auto (search for linux and book, generate abstract automatically)
// http://localhost:8080/yacy/search.html?query=&abstracts=4galTpdpDM5Q (only abstracts for linux)
-
- serverObjects prop = new serverObjects();
-
+
if ((sb.isRobinsonMode()) &&
(!((sb.isPublicRobinson()) ||
(sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))))) {
diff --git a/htroot/yacy/transfer.java b/htroot/yacy/transfer.java
index 3371ca407..91c42bff3 100644
--- a/htroot/yacy/transfer.java
+++ b/htroot/yacy/transfer.java
@@ -55,15 +55,17 @@ import de.anomic.server.serverFileUtils;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
+import de.anomic.yacy.yacyNetwork;
import de.anomic.yacy.yacySeed;
public final class transfer {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
- if (post == null || env == null) return null;
plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
-
+ if ((post == null) || (env == null)) return prop;
+ if (!yacyNetwork.authentifyRequest(post, env)) return prop;
+
String process = post.get("process", ""); // permission or store
//String key = post.get("key", ""); // a transmission key from the client
String otherpeer = post.get("iam", ""); // identification of the client (a peer-hash)
diff --git a/htroot/yacy/transferRWI.java b/htroot/yacy/transferRWI.java
index 12d26f72b..1139c618b 100644
--- a/htroot/yacy/transferRWI.java
+++ b/htroot/yacy/transferRWI.java
@@ -60,17 +60,18 @@ import de.anomic.server.serverSwitch;
import de.anomic.tools.nxTools;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyDHTAction;
+import de.anomic.yacy.yacyNetwork;
import de.anomic.yacy.yacySeed;
public final class transferRWI {
- public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch ss) throws InterruptedException {
- if (post == null || ss == null) { return null; }
-
+ public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws InterruptedException {
+
// return variable that accumulates replacements
- final plasmaSwitchboard sb = (plasmaSwitchboard) ss;
+ final plasmaSwitchboard sb = (plasmaSwitchboard) env;
final serverObjects prop = new serverObjects();
- if (prop == null || sb == null) { return null; }
+ if ((post == null) || (env == null)) return prop;
+ if (!yacyNetwork.authentifyRequest(post, env)) return prop;
// request values
final String iam = post.get("iam", ""); // seed hash of requester
diff --git a/htroot/yacy/transferURL.java b/htroot/yacy/transferURL.java
index a26822bcb..ac886b62c 100644
--- a/htroot/yacy/transferURL.java
+++ b/htroot/yacy/transferURL.java
@@ -57,22 +57,22 @@ import de.anomic.server.serverCore;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
+import de.anomic.yacy.yacyNetwork;
import de.anomic.yacy.yacySeed;
public final class transferURL {
- public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch ss) throws InterruptedException {
- if (post == null || ss == null) { return null; }
-
+ public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws InterruptedException {
long start = System.currentTimeMillis();
long freshdate = 0;
try {freshdate = plasmaURL.shortDayFormatter.parse("20061101").getTime();} catch (ParseException e1) {}
// return variable that accumulates replacements
- final plasmaSwitchboard sb = (plasmaSwitchboard) ss;
+ final plasmaSwitchboard sb = (plasmaSwitchboard) env;
final serverObjects prop = new serverObjects();
- if (prop == null || sb == null) { return null; }
+ if ((post == null) || (env == null)) return prop;
+ if (!yacyNetwork.authentifyRequest(post, env)) return prop;
// request values
final String iam = post.get("iam", ""); // seed hash of requester
diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java
index dd4cadd21..473e95bb6 100644
--- a/source/de/anomic/yacy/yacyClient.java
+++ b/source/de/anomic/yacy/yacyClient.java
@@ -46,12 +46,12 @@ package de.anomic.yacy;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
-import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
+import de.anomic.http.httpRemoteProxyConfig;
import de.anomic.http.httpc;
import de.anomic.index.indexContainer;
import de.anomic.index.indexRWIEntry;
@@ -96,46 +96,23 @@ public final class yacyClient {
// but they appear to be another peer by comparisment of the other peer's hash
// this works of course only if we know the other peer's hash.
- final String key = crypt.randomSalt();
HashMap result = null;
+ String salt;
try {
- /*
- URL url = new URL("http://" + address + "/yacy/hello.html?iam=" +
- yacyCore.seedCache.mySeed.hash +
- "&pattern=&count=20" +
- "&key=" + key + "&seed=" + yacyCore.seedCache.mySeed.genSeedStr(key));
- yacyCore.log.logDebug("HELLO to URL " + url.toString());
- result = nxTools.table(httpc.wget(url,
- 10000, null, null, yacyCore.seedCache.sb.remoteProxyHost, yacyCore.seedCache.sb.remoteProxyPort));
- */
-
- // building URL
- final URL url = new URL("http://" + address + "/yacy/hello.html");
-
- // should we use the proxy?
- boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
-
- // adding all needed parameters
- final serverObjects obj = new serverObjects(6);
- obj.put("iam", yacyCore.seedDB.mySeed.hash);
- obj.put("pattern", "");
- obj.put("count", "20");
- obj.put("key", key);
- obj.put("mytime", yacyCore.universalDateShortString(new Date()));
- obj.put("myUTC", System.currentTimeMillis());
- obj.put("seed", yacyCore.seedDB.mySeed.genSeedStr(key));
- obj.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
-
- // sending request
+ // generate request
+ final serverObjects obj = new serverObjects();
+ salt = yacyNetwork.enrichRequestPost(obj, plasmaSwitchboard.getSwitchboard(), null);
+ obj.put("count", "20");
+ obj.put("seed", yacyCore.seedDB.mySeed.genSeedStr(salt));
+
+ // send request
result = nxTools.table(
- httpc.wput(url,
+ httpc.wput(new URL("http://" + address + "/yacy/hello.html"),
yacySeed.b64Hash2hexHash(otherHash) + ".yacyh",
12000,
null,
null,
- (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
+ proxyConfig(),
obj,
null
), "UTF-8"
@@ -164,7 +141,7 @@ public final class yacyClient {
if (seed.length() > yacySeed.maxsize) {
yacyCore.log.logInfo("hello/client 0: rejected contacting seed; too large (" + seed.length() + " > " + yacySeed.maxsize + ")");
} else {
- otherPeer = yacySeed.genRemoteSeed(seed, key, true);
+ otherPeer = yacySeed.genRemoteSeed(seed, salt, true);
if (otherPeer == null || !otherPeer.hash.equals(otherHash)) {
yacyCore.log.logFine("yacyClient.publishMySeed: consistency error: other peer '" + ((otherPeer==null)?"unknown":otherPeer.getName()) + "' wrong");
return -1; // no success
@@ -250,45 +227,36 @@ public final class yacyClient {
if (seedStr.length() > yacySeed.maxsize) {
yacyCore.log.logInfo("hello/client: rejected contacting seed; too large (" + seedStr.length() + " > " + yacySeed.maxsize + ")");
} else {
- if (yacyCore.peerActions.peerArrival(yacySeed.genRemoteSeed(seedStr, key, true), (i == 1))) count++;
+ if (yacyCore.peerActions.peerArrival(yacySeed.genRemoteSeed(seedStr, salt, true), (i == 1))) count++;
}
}
return count;
}
public static yacySeed querySeed(yacySeed target, String seedHash) {
- final String key = crypt.randomSalt();
- try {
- // should we use the proxy?
- boolean useProxy = (plasmaSwitchboard.getSwitchboard().remoteProxyConfig != null) &&
- (plasmaSwitchboard.getSwitchboard().remoteProxyConfig.useProxy()) &&
- (plasmaSwitchboard.getSwitchboard().remoteProxyConfig.useProxy4Yacy());
+ // prepare request
+ final serverObjects post = new serverObjects();
+ String salt = yacyNetwork.enrichRequestPost(post, plasmaSwitchboard.getSwitchboard(), target.hash);
+ post.put("object", "seed");
+ post.put("env", seedHash);
- // sending request
+ // send request
+ try {
final HashMap result = nxTools.table(
- httpc.wget(
- new URL("http://" + target.getClusterAddress() +
- "/yacy/query.html" +
- "?iam=" + yacyCore.seedDB.mySeed.hash +
- "&youare=" + target.hash +
- "&key=" + key +
- "&object=seed" +
- "&env=" + seedHash +
- "&network.unit.name=" + plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT)
- ),
- target.getHexHash() + ".yacyh",
- 8000,
- null,
- null,
- (useProxy)? plasmaSwitchboard.getSwitchboard().remoteProxyConfig:null,
- null,
- null
- )
- , "UTF-8");
+ httpc.wput(new URL("http://" + target.getClusterAddress() + "/yacy/query.html"),
+ target.getHexHash() + ".yacyh",
+ 8000,
+ null,
+ null,
+ proxyConfig(),
+ post,
+ null
+ ), "UTF-8"
+ );
if (result == null || result.size() == 0) { return null; }
//final Date remoteTime = yacyCore.parseUniversalDate((String) result.get(yacySeed.MYTIME)); // read remote time
- return yacySeed.genRemoteSeed((String) result.get("response"), key, true);
+ return yacySeed.genRemoteSeed((String) result.get("response"), salt, true);
} catch (Exception e) {
yacyCore.log.logSevere("yacyClient.querySeed error:" + e.getMessage());
return null;
@@ -296,34 +264,26 @@ public final class yacyClient {
}
public static int queryRWICount(yacySeed target, String wordHash) {
- try {
- // should we use the proxy?
- boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
+ // prepare request
+ final serverObjects post = new serverObjects();
+ yacyNetwork.enrichRequestPost(post, plasmaSwitchboard.getSwitchboard(), target.hash);
+ post.put("object", "rwicount");
+ post.put("ttl", "0");
+ post.put("env", wordHash);
- // sending request
+ // send request
+ try {
final HashMap result = nxTools.table(
- httpc.wget(
- new URL("http://" + target.getClusterAddress() +
- "/yacy/query.html" +
- "?iam=" + yacyCore.seedDB.mySeed.hash +
- "&youare=" + target.hash +
- "&key=" +
- "&object=rwicount" +
- "&env=" + wordHash +
- "&ttl=0" +
- "&network.unit.name=" + plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT)
- ),
- target.getHexHash() + ".yacyh",
- 10000,
- null,
- null,
- (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
- null,
- null
- )
- , "UTF-8");
+ httpc.wput(new URL("http://" + target.getClusterAddress() + "/yacy/query.html"),
+ target.getHexHash() + ".yacyh",
+ 8000,
+ null,
+ null,
+ proxyConfig(),
+ post,
+ null
+ ), "UTF-8"
+ );
if (result == null || result.size() == 0) { return -1; }
return Integer.parseInt((String) result.get("response"));
@@ -337,39 +297,27 @@ public final class yacyClient {
if (target == null) { return -1; }
if (yacyCore.seedDB.mySeed == null) return -1;
- // should we use the proxy?
- boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
-
- // building url
- final String querystr =
- "http://" + target.getClusterAddress() +
- "/yacy/query.html" +
- "?iam=" + yacyCore.seedDB.mySeed.hash +
- "&youare=" + target.hash +
- "&key=" +
- "&object=lurlcount" +
- "&env=" +
- "&ttl=0" +
- "&network.unit.name=" + plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT);
+ // prepare request
+ final serverObjects post = new serverObjects();
+ yacyNetwork.enrichRequestPost(post, plasmaSwitchboard.getSwitchboard(), target.hash);
+ post.put("object", "lurlcount");
+ post.put("ttl", "0");
+ post.put("env", "");
- // seinding request
+ // send request
try {
- final HashMap result = nxTools.table(
- httpc.wget(
- new URL(querystr),
- target.getHexHash() + ".yacyh",
- 6000,
- null,
- null,
- (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
- null,
- null
- )
- , "UTF-8");
+ final HashMap result = nxTools.table(
+ httpc.wput(new URL("http://" + target.getClusterAddress() + "/yacy/query.html"),
+ target.getHexHash() + ".yacyh",
+ 6000,
+ null,
+ null,
+ proxyConfig(),
+ post,
+ null
+ ), "UTF-8"
+ );
-// yacyCore.log("DEBUG QUERY: query=" + querystr + "; result = " + result.toString());
if ((result == null) || (result.size() == 0)) return -1;
final String resp = (String) result.get("response");
if (resp == null) {
@@ -394,7 +342,7 @@ public final class yacyClient {
int maxDistance,
boolean global,
int partitions,
- yacySeed targetPeer,
+ yacySeed target,
plasmaCrawlLURL urlManager,
plasmaWordIndex wordIndex,
indexContainer containerCache,
@@ -421,236 +369,223 @@ public final class yacyClient {
// partitions : number of remote peers that are asked (for evaluation of QPM)
// duetime : maximum time that a peer should spent to create a result
- // request result
- final String key = crypt.randomSalt();
- try {
- // should we use the proxy?
- boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
-
- // building url
- final String url = "http://" + targetPeer.getClusterAddress() + "/yacy/search.html";
-
- // adding all needed parameters
- /*
- String url = "http://" + targetPeer.getAddress() +
- "/yacy/search.html?myseed=" + yacyCore.seedCache.mySeed.genSeedStr(key) +
- "&youare=" + targetPeer.hash + "&key=" + key +
- "&myseed=" + yacyCore.seedCache.mySeed.genSeedStr(key) +
- "&count=" + count + "&resource=" + ((global) ? "global" : "local") +
- "&query=" + wordhashes;
- */
- final serverObjects obj = new serverObjects(20);
- long duetime = timingProfile.duetime();
- obj.put("myseed", yacyCore.seedDB.mySeed.genSeedStr(key));
- obj.put("youare", targetPeer.hash);
- obj.put("key", key);
- obj.put("count", timingProfile.getTargetCount(plasmaSearchTimingProfile.PROCESS_POSTSORT));
- obj.put("resource", ((global) ? "global" : "local"));
- obj.put("partitions", partitions);
- obj.put("query", wordhashes);
- obj.put("exclude", excludehashes);
- obj.put("urls", urlhashes);
- obj.put("prefer", prefer);
- obj.put("filter", filter);
- obj.put("ttl", "0");
- obj.put("duetime", Long.toString(duetime));
- obj.put("timing", crypt.simpleEncode(timingProfile.targetToString())); // new duetimes splitted by specific search tasks
- obj.put("maxdist", maxDistance);
- obj.put("profile", crypt.simpleEncode(rankingProfile.toExternalString()));
- obj.put("constraint", constraint.exportB64());
- obj.put("mytime", yacyCore.universalDateShortString(new Date()));
- obj.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
- if (abstractCache != null) obj.put("abstracts", "auto");
-
- //yacyCore.log.logDebug("yacyClient.search url=" + url);
- final long timestamp = System.currentTimeMillis();
+ // prepare request
+ final serverObjects post = new serverObjects();
+ final String salt = yacyNetwork.enrichRequestPost(post, plasmaSwitchboard.getSwitchboard(), target.hash);
+ long duetime = timingProfile.duetime();
+ post.put("myseed", yacyCore.seedDB.mySeed.genSeedStr(salt));
+ post.put("count", timingProfile.getTargetCount(plasmaSearchTimingProfile.PROCESS_POSTSORT));
+ post.put("resource", ((global) ? "global" : "local"));
+ post.put("partitions", partitions);
+ post.put("query", wordhashes);
+ post.put("exclude", excludehashes);
+ post.put("urls", urlhashes);
+ post.put("prefer", prefer);
+ post.put("filter", filter);
+ post.put("ttl", "0");
+ post.put("duetime", Long.toString(duetime));
+ post.put("timing", crypt.simpleEncode(timingProfile.targetToString())); // new duetimes splitted by specific search tasks
+ post.put("maxdist", maxDistance);
+ post.put("profile", crypt.simpleEncode(rankingProfile.toExternalString()));
+ post.put("constraint", constraint.exportB64());
+ if (abstractCache != null) post.put("abstracts", "auto");
+ final long timestamp = System.currentTimeMillis();
- // sending request
- HashMap result = null;
- try {
- result = nxTools.table(
- httpc.wput(
- new URL(url),
- targetPeer.getHexHash() + ".yacyh",
- 300000,
- null,
- null,
- (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
- obj,
- null
+ // send request
+ HashMap result = null;
+ try {
+ result = nxTools.table(
+ httpc.wput(new URL("http://" + target.getClusterAddress() + "/yacy/search.html"),
+ target.getHexHash() + ".yacyh",
+ 60000,
+ null,
+ null,
+ proxyConfig(),
+ post,
+ null
), "UTF-8"
- );
- } catch (IOException e) {
- yacyCore.log.logFine("SEARCH failed FROM " + targetPeer.hash + ":" + targetPeer.getName() + " (" + e.getMessage() + "), score=" + targetPeer.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes.substring(0, 12)));
- yacyCore.peerActions.peerDeparture(targetPeer);
- return null;
- }
+ );
+ } catch (IOException e) {
+ yacyCore.log.logFine("SEARCH failed FROM " + target.hash + ":" + target.getName() + " (" + e.getMessage() + "), score=" + target.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(target.hash, wordhashes.substring(0, 12)));
+ yacyCore.peerActions.peerDeparture(target);
+ return null;
+ }
- if (result.size() == 0) {
- yacyCore.log.logFine("SEARCH failed FROM " + targetPeer.hash + ":" + targetPeer.getName() + " (zero response), score=" + targetPeer.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes.substring(0, 12)));
- return null;
- }
-
- // compute all computation times
- final long totalrequesttime = System.currentTimeMillis() - timestamp;
- String returnProfile = (String) result.get("profile");
- if (returnProfile != null) timingProfile.putYield(returnProfile);
-
- /*
- HashMap result = nxTools.table(httpc.wget(new URL(url),
- 300000, null, null, yacyCore.seedCache.remoteProxyHost, yacyCore.seedCache.remoteProxyPort));
- */
- // OUTPUT:
- // version : application version of responder
- // uptime : uptime in seconds of responder
- // total : number of total available LURL's for this search
- // count : number of returned LURL's for this search
- // resource<n> : LURL of search
- // fwhop : hops (depth) of forwards that had been performed to construct this result
- // fwsrc : peers that helped to construct this result
- // fwrec : peers that would have helped to construct this result (recommendations)
- // searchtime : time that the peer actually spent to create the result
- // references : references (search hints) that was calculated during search
+ if ((result == null) || (result.size() == 0)) {
+ yacyCore.log.logFine("SEARCH failed FROM "
+ + target.hash
+ + ":"
+ + target.getName()
+ + " (zero response), score="
+ + target.selectscore
+ + ", DHTdist="
+ + yacyDHTAction.dhtDistance(target.hash, wordhashes
+ .substring(0, 12)));
+ return null;
+ }
- // now create a plasmaIndex out of this result
- //System.out.println("yacyClient: " + ((urlhashes.length() == 0) ? "primary" : "secondary")+ " search result = " + result.toString()); // debug
- final int results = Integer.parseInt((String) result.get("count"));
- //System.out.println("***result count " + results);
-
- // create containers
- final int words = wordhashes.length() / yacySeedDB.commonHashLength;
- indexContainer[] container = new indexContainer[words];
- for (int i = 0; i < words; i++) {
- container[i] = wordIndex.emptyContainer(wordhashes.substring(i * yacySeedDB.commonHashLength, (i + 1) * yacySeedDB.commonHashLength));
- }
+ // compute all computation times
+ final long totalrequesttime = System.currentTimeMillis() - timestamp;
+ String returnProfile = (String) result.get("profile");
+ if (returnProfile != null) timingProfile.putYield(returnProfile);
+
+ // OUTPUT:
+ // version : application version of responder
+ // uptime : uptime in seconds of responder
+ // total : number of total available LURL's for this search
+ // count : number of returned LURL's for this search
+ // resource<n> : LURL of search
+ // fwhop : hops (depth) of forwards that had been performed to construct this result
+ // fwsrc : peers that helped to construct this result
+ // fwrec : peers that would have helped to construct this result (recommendations)
+ // searchtime : time that the peer actually spent to create the result
+ // references : references (search hints) that was calculated during search
+
+ // now create a plasmaIndex out of this result
+ // System.out.println("yacyClient: " + ((urlhashes.length() == 0) ? "primary" : "secondary")+ " search result = " + result.toString()); // debug
+
+ final int results = Integer.parseInt((String) result.get("count"));
+ // System.out.println("***result count " + results);
- // insert results to containers
- indexURLEntry urlEntry;
- String[] urls = new String[results];
- for (int n = 0; n < results; n++) {
- // get one single search result
- urlEntry = urlManager.newEntry((String) result.get("resource" + n));
- if (urlEntry == null) continue;
- assert (urlEntry.hash().length() == 12) : "urlEntry.hash() = " + urlEntry.hash();
- if (urlEntry.hash().length() != 12) continue; // bad url hash
- indexURLEntry.Components comp = urlEntry.comp();
- if (blacklist.isListed(plasmaURLPattern.BLACKLIST_SEARCH, comp.url())) {
- yacyCore.log.logInfo("remote search (client): filtered blacklisted url " + comp.url() + " from peer " + targetPeer.getName());
- continue; // block with backlist
- }
-
- // save the url entry
- indexRWIEntry entry;
- if (urlEntry.word() == null) {
- yacyCore.log.logWarning("remote search (client): no word attached from peer " + targetPeer.getName() + ", version " + targetPeer.getVersion());
- continue; // no word attached
- }
-
- // the search-result-url transports all the attributes of word indexes
- entry = urlEntry.word();
- if (!(entry.urlHash().equals(urlEntry.hash()))) {
- yacyCore.log.logInfo("remote search (client): url-hash " + urlEntry.hash() + " does not belong to word-attached-hash " + entry.urlHash() + "; url = " + comp.url() + " from peer " + targetPeer.getName());
- continue; // spammed
- }
+ // create containers
+ final int words = wordhashes.length() / yacySeedDB.commonHashLength;
+ indexContainer[] container = new indexContainer[words];
+ for (int i = 0; i < words; i++) {
+ container[i] = wordIndex.emptyContainer(wordhashes.substring(i * yacySeedDB.commonHashLength, (i + 1) * yacySeedDB.commonHashLength));
+ }
- // passed all checks, store url
- urlManager.store(urlEntry);
- urlManager.stack(urlEntry, yacyCore.seedDB.mySeed.hash, targetPeer.hash, 2);
+ // insert results to containers
+ indexURLEntry urlEntry;
+ String[] urls = new String[results];
+ for (int n = 0; n < results; n++) {
+ // get one single search result
+ urlEntry = urlManager.newEntry((String) result.get("resource" + n));
+ if (urlEntry == null) continue;
+ assert (urlEntry.hash().length() == 12) : "urlEntry.hash() = " + urlEntry.hash();
+ if (urlEntry.hash().length() != 12) continue; // bad url hash
+ indexURLEntry.Components comp = urlEntry.comp();
+ if (blacklist.isListed(plasmaURLPattern.BLACKLIST_SEARCH, comp.url())) {
+ yacyCore.log.logInfo("remote search (client): filtered blacklisted url " + comp.url() + " from peer " + target.getName());
+ continue; // block with backlist
+ }
- if (urlEntry.snippet() != null) {
- // we don't store the snippets along the url entry, because they are search-specific.
- // instead, they are placed in a snipped-search cache.
- //System.out.println("--- RECEIVED SNIPPET '" + link.snippet() + "'");
- snippets.storeToCache(wordhashes, urlEntry.hash(), urlEntry.snippet());
- }
- // add the url entry to the word indexes
- for (int m = 0; m < words; m++) {
- container[m].add(entry, System.currentTimeMillis());
- }
- // store url hash for statistics
- urls[n] = urlEntry.hash();
- }
+ // save the url entry
+ indexRWIEntry entry;
+ if (urlEntry.word() == null) {
+ yacyCore.log.logWarning("remote search (client): no word attached from peer " + target.getName() + ", version " + target.getVersion());
+ continue; // no word attached
+ }
- // insert the containers to the index
- for (int m = 0; m < words; m++) { containerCache.addAllUnique(container[m]); }
+ // the search-result-url transports all the attributes of word
+ // indexes
+ entry = urlEntry.word();
+ if (!(entry.urlHash().equals(urlEntry.hash()))) {
+ yacyCore.log.logInfo("remote search (client): url-hash " + urlEntry.hash() + " does not belong to word-attached-hash " + entry.urlHash() + "; url = " + comp.url() + " from peer " + target.getName());
+ continue; // spammed
+ }
- // read index abstract
- if (abstractCache != null) {
- Iterator i = result.entrySet().iterator();
- Map.Entry entry;
- TreeMap singleAbstract;
- String wordhash;
- serverByteBuffer ci;
- while (i.hasNext()) {
- entry = (Map.Entry) i.next();
- if (((String) entry.getKey()).startsWith("indexabstract.")) {
- wordhash = ((String) entry.getKey()).substring(14);
- synchronized (abstractCache) {
- singleAbstract = (TreeMap) abstractCache.get(wordhash); // a mapping from url-hashes to a string of peer-hashes
- if (singleAbstract == null) singleAbstract = new TreeMap();
- ci = new serverByteBuffer(((String) entry.getValue()).getBytes());
- System.out.println("DEBUG-ABSTRACTFETCH: for word hash " + wordhash + " received " + ci.toString());
- plasmaURL.decompressIndex(singleAbstract, ci, targetPeer.hash);
- abstractCache.put(wordhash, singleAbstract);
- }
- }
- }
- }
- // generate statistics
- long searchtime;
- try {
- searchtime = Integer.parseInt((String) result.get("searchtime"));
- } catch (NumberFormatException e) {
- searchtime = totalrequesttime;
- }
- yacyCore.log.logFine("SEARCH " + results + " URLS FROM " + targetPeer.hash + ":" + targetPeer.getName() + ", score=" + targetPeer.selectscore + ", DHTdist=" + ((wordhashes.length() < 12) ? "void" : Double.toString(yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes.substring(0, 12)))) + ", duetime=" + duetime + ", searchtime=" + searchtime + ", netdelay=" + (totalrequesttime - searchtime) + ", references=" + result.get("references"));
- return urls;
- } catch (Exception e) {
- yacyCore.log.logSevere("yacyClient.search error: '" + targetPeer.get(yacySeed.NAME, "anonymous") + "' failed - " + e);
- e.printStackTrace();
- return null;
- }
- }
+ // passed all checks, store url
+ try {
+ urlManager.store(urlEntry);
+ urlManager.stack(urlEntry, yacyCore.seedDB.mySeed.hash, target.hash, 2);
+ } catch (IOException e) {
+ yacyCore.log.logSevere("could not store search result", e);
+ continue; // db-error
+ }
+
+ if (urlEntry.snippet() != null) {
+ // we don't store the snippets along the url entry, because they
+ // are search-specific.
+ // instead, they are placed in a snipped-search cache.
+ // System.out.println("--- RECEIVED SNIPPET '" + link.snippet()
+ // + "'");
+ snippets.storeToCache(wordhashes, urlEntry.hash(), urlEntry.snippet());
+ }
+ // add the url entry to the word indexes
+ for (int m = 0; m < words; m++) {
+ container[m].add(entry, System.currentTimeMillis());
+ }
+ // store url hash for statistics
+ urls[n] = urlEntry.hash();
+ }
+
+ // insert the containers to the index
+ for (int m = 0; m < words; m++) {
+ containerCache.addAllUnique(container[m]);
+ }
+
+ // read index abstract
+ if (abstractCache != null) {
+ Iterator i = result.entrySet().iterator();
+ Map.Entry entry;
+ TreeMap singleAbstract;
+ String wordhash;
+ serverByteBuffer ci;
+ while (i.hasNext()) {
+ entry = (Map.Entry) i.next();
+ if (((String) entry.getKey()).startsWith("indexabstract.")) {
+ wordhash = ((String) entry.getKey()).substring(14);
+ synchronized (abstractCache) {
+ singleAbstract = (TreeMap) abstractCache.get(wordhash); // a mapping from url-hashes to a string of peer-hashes
+ if (singleAbstract == null) singleAbstract = new TreeMap();
+ ci = new serverByteBuffer(((String) entry.getValue()).getBytes());
+ System.out.println("DEBUG-ABSTRACTFETCH: for word hash " + wordhash + " received " + ci.toString());
+ plasmaURL.decompressIndex(singleAbstract, ci, target.hash);
+ abstractCache.put(wordhash, singleAbstract);
+ }
+ }
+ }
+ }
+
+ // generate statistics
+ long searchtime;
+ try {
+ searchtime = Integer.parseInt((String) result.get("searchtime"));
+ } catch (NumberFormatException e) {
+ searchtime = totalrequesttime;
+ }
+ yacyCore.log.logFine("SEARCH "
+ + results
+ + " URLS FROM "
+ + target.hash
+ + ":"
+ + target.getName()
+ + ", score="
+ + target.selectscore
+ + ", DHTdist="
+ + ((wordhashes.length() < 12) ? "void" : Double
+ .toString(yacyDHTAction.dhtDistance(target.hash,
+ wordhashes.substring(0, 12)))) + ", duetime="
+ + duetime + ", searchtime=" + searchtime + ", netdelay="
+ + (totalrequesttime - searchtime) + ", references="
+ + result.get("references"));
+ return urls;
+ }
public static HashMap permissionMessage(String targetHash) {
// ask for allowed message size and attachement size
// if this replies null, the peer does not answer
if (yacyCore.seedDB == null || yacyCore.seedDB.mySeed == null) { return null; }
+
+ // prepare request
+ final serverObjects post = new serverObjects();
+ yacyNetwork.enrichRequestPost(post, plasmaSwitchboard.getSwitchboard(), targetHash);
+ post.put("process", "permission");
- // should we use the proxy?
- boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
-
- // adding all needed parameters
- final String key = crypt.randomSalt();
- final serverObjects post = new serverObjects(5);
- post.put("key", key);
- post.put("process", "permission");
- post.put("iam", yacyCore.seedDB.mySeed.hash);
- post.put("youare", targetHash);
- post.put("mytime", yacyCore.universalDateShortString(new Date()));
- post.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
-
- // get target address
- String address = targetAddress(targetHash);
-
- // sending request
+ // send request
try {
- return nxTools.table(
- httpc.wput(
- new URL("http://" + address + "/yacy/message.html"),
- yacySeed.b64Hash2hexHash(targetHash) + ".yacyh",
- 8000,
- null,
- null,
- (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
- post,
- null
- ), "UTF-8"
+ final HashMap result = nxTools.table(
+ httpc.wput(new URL("http://" + targetAddress(targetHash) + "/yacy/message.html"),
+ yacySeed.b64Hash2hexHash(targetHash)+ ".yacyh",
+ 8000,
+ null,
+ null,
+ proxyConfig(),
+ post,
+ null
+ ), "UTF-8"
);
+ return result;
} catch (Exception e) {
// most probably a network time-out exception
yacyCore.log.logSevere("yacyClient.permissionMessage error:" + e.getMessage());
@@ -660,43 +595,33 @@ public final class yacyClient {
public static HashMap postMessage(String targetHash, String subject, byte[] message) {
// this post a message to the remote message board
+
+ // prepare request
+ final serverObjects post = new serverObjects();
+ final String salt = yacyNetwork.enrichRequestPost(post, plasmaSwitchboard.getSwitchboard(), targetHash);
+ post.put("process", "post");
+ post.put("myseed", yacyCore.seedDB.mySeed.genSeedStr(salt));
+ post.put("subject", subject);
+ try {
+ post.put("message", new String(message, "UTF-8"));
+ } catch (UnsupportedEncodingException e) {
+ post.put("message", new String(message));
+ }
- // should we use the proxy?
- boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
-
- // adding all needed parameters
- final String key = crypt.randomSalt();
- final serverObjects post = new serverObjects(7);
- post.put("key", key);
- post.put("process", "post");
- post.put("myseed", yacyCore.seedDB.mySeed.genSeedStr(key));
- post.put("youare", targetHash);
- post.put("subject", subject);
- post.put("mytime", yacyCore.universalDateShortString(new Date()));
- post.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
- try {
- post.put("message", new String(message, "UTF-8"));
- } catch (UnsupportedEncodingException e) {
- post.put("message", new String(message));
- }
- // get target address
- String address = targetAddress(targetHash);
-
- // sending request
+ // send request
try {
- return nxTools.table(
- httpc.wput(
- new URL("http://" + address + "/yacy/message.html"),
- yacySeed.b64Hash2hexHash(targetHash) + ".yacyh",
- 20000,
- null,
- null,
- (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
- post,
- null
- ), "UTF-8");
+ final HashMap result = nxTools.table(
+ httpc.wput(new URL("http://" + targetAddress(targetHash) + "/yacy/message.html"),
+ yacySeed.b64Hash2hexHash(targetHash)+ ".yacyh",
+ 20000,
+ null,
+ null,
+ proxyConfig(),
+ post,
+ null
+ ), "UTF-8"
+ );
+ return result;
} catch (Exception e) {
yacyCore.log.logSevere("yacyClient.postMessage error:" + e.getMessage());
return null;
@@ -718,39 +643,31 @@ public final class yacyClient {
}
public static HashMap transferPermission(String targetAddress, long filesize, String filename) {
- // should we use the proxy?
- boolean useProxy = (yacyCore.seedDB != null) &&
- (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
-
- // add all needed parameters
- final String key = crypt.randomSalt();
+
+ // prepare request
final serverObjects post = new serverObjects();
- post.put("key", key);
+ yacyNetwork.enrichRequestPost(post, plasmaSwitchboard.getSwitchboard(), null);
post.put("process", "permission");
- post.put("iam", yacyCore.seedDB.mySeed.hash);
post.put("purpose", "crcon");
post.put("filename", filename);
post.put("filesize", Long.toString(filesize));
post.put("can-send-protocol", "http");
- post.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
// send request
try {
final URL url = new URL("http://" + targetAddress + "/yacy/transfer.html");
- return nxTools.table(
- httpc.wput(
- url,
- url.getHost(), // yacyCore.seedDB.mySeed.getHexHash() + ".yacyh",
- 8000,
- null,
- null,
- (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
- post,
- null
- ), "UTF-8"
+ final HashMap result = nxTools.table(
+ httpc.wput(url,
+ url.getHost(),
+ 6000,
+ null,
+ null,
+ proxyConfig(),
+ post,
+ null
+ ), "UTF-8"
);
+ return result;
} catch (Exception e) {
// most probably a network time-out exception
yacyCore.log.logSevere("yacyClient.permissionTransfer error:" + e.getMessage());
@@ -759,41 +676,34 @@ public final class yacyClient {
}
public static HashMap transferStore(String targetAddress, String access, String filename, byte[] file) {
- // should we use the proxy?
- boolean useProxy = (yacyCore.seedDB != null) &&
- (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
- // adding all needed parameters
- final String key = crypt.randomSalt();
- final serverObjects post = new serverObjects(7);
- post.put("key", key);
+ // prepare request
+ final serverObjects post = new serverObjects();
+ yacyNetwork.enrichRequestPost(post, plasmaSwitchboard.getSwitchboard(), null);
post.put("process", "store");
- post.put("iam", yacyCore.seedDB.mySeed.hash);
post.put("purpose", "crcon");
post.put("filename", filename);
post.put("filesize", Long.toString(file.length));
post.put("md5", serverCodings.encodeMD5Hex(file));
post.put("access", access);
- post.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
HashMap files = new HashMap();
files.put("filename", file);
- // sending request
+ // send request
try {
final URL url = new URL("http://" + targetAddress + "/yacy/transfer.html");
- return nxTools.table(
- httpc.wput(
- url,
- url.getHost(), // yacyCore.seedDB.mySeed.getHexHash() + ".yacyh",
- 20000,
- null,
- null,
- (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
- post,
- files
- ), "UTF-8");
+ final HashMap result = nxTools.table(
+ httpc.wput(url,
+ url.getHost(),
+ 20000,
+ null,
+ null,
+ proxyConfig(),
+ post,
+ files
+ ), "UTF-8"
+ );
+ return result;
} catch (Exception e) {
yacyCore.log.logSevere("yacyClient.postMessage error:" + e.getMessage());
return null;
@@ -829,25 +739,15 @@ public final class yacyClient {
return crawlOrder(targetSeed, new URL[]{url}, new URL[]{referrer}, timeout);
}
- public static HashMap crawlOrder(yacySeed targetSeed, URL[] url, URL[] referrer, int timeout) {
- // this post a message to the remote message board
- if (targetSeed == null) { return null; }
- if (yacyCore.seedDB.mySeed == null) { return null; }
- if (yacyCore.seedDB.mySeed == targetSeed) { return null; }
-
- // should we use the proxy?
- boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
+ public static HashMap crawlOrder(yacySeed target, URL[] url, URL[] referrer, int timeout) {
+ assert (target != null);
+ assert (yacyCore.seedDB.mySeed != null);
+ assert (yacyCore.seedDB.mySeed != target);
- // construct request
- final String key = crypt.randomSalt();
- final serverObjects post = new serverObjects(9);
- post.put("key", key);
+ // prepare request
+ final serverObjects post = new serverObjects();
+ yacyNetwork.enrichRequestPost(post, plasmaSwitchboard.getSwitchboard(), target.hash);
post.put("process", "crawl");
- post.put("iam", yacyCore.seedDB.mySeed.hash);
- post.put("youare", targetSeed.hash);
- post.put("mytime", yacyCore.universalDateShortString(new Date()));
if (url.length == 1) {
post.put("url", crypt.simpleEncode(url[0].toString()));
post.put("referrer", crypt.simpleEncode((referrer[0] == null) ? "" : referrer[0].toString()));
@@ -859,29 +759,28 @@ public final class yacyClient {
}
post.put("depth", "0");
post.put("ttl", "0");
- post.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
// determining target address
- final String address = targetSeed.getClusterAddress();
+ final String address = target.getClusterAddress();
if (address == null) { return null; }
-
- // sending request
+
+ // send request
try {
- return nxTools.table(
- httpc.wput(
- new URL("http://" + address + "/yacy/crawlOrder.html"),
- targetSeed.getHexHash() + ".yacyh",
- timeout,
- null,
- null,
- (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
- post,
- null
+ final HashMap result = nxTools.table(
+ httpc.wput(new URL("http://" + address + "/yacy/crawlOrder.html"),
+ target.getHexHash() + ".yacyh",
+ timeout,
+ null,
+ null,
+ proxyConfig(),
+ post,
+ null
), "UTF-8"
);
+ return result;
} catch (Exception e) {
// most probably a network time-out exception
- yacyCore.log.logSevere("yacyClient.crawlOrder error: peer=" + targetSeed.getName() + ", error=" + e.getMessage());
+ yacyCore.log.logSevere("yacyClient.crawlOrder error: peer=" + target.getName() + ", error=" + e.getMessage());
return null;
}
}
@@ -893,10 +792,10 @@ public final class yacyClient {
-er crawlt, Ergebnis erscheint aber unter falschem initiator
*/
- public static HashMap crawlReceipt(yacySeed targetSeed, String process, String result, String reason, indexURLEntry entry, String wordhashes) {
- if (targetSeed == null) { return null; }
- if (yacyCore.seedDB.mySeed == null) { return null; }
- if (yacyCore.seedDB.mySeed == targetSeed) { return null; }
+ public static HashMap crawlReceipt(yacySeed target, String process, String result, String reason, indexURLEntry entry, String wordhashes) {
+ assert (target != null);
+ assert (yacyCore.seedDB.mySeed != null);
+ assert (yacyCore.seedDB.mySeed != target);
/*
the result can have one of the following values:
@@ -918,43 +817,33 @@ public final class yacyClient {
*/
- // should we use the proxy?
- boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
-
- // construct request
- final String key = crypt.randomSalt();
-
+ // prepare request
+ final serverObjects post = new serverObjects();
+ String salt = yacyNetwork.enrichRequestPost(post, plasmaSwitchboard.getSwitchboard(), target.hash);
+ post.put("process", process);
+ post.put("urlhash", ((entry == null) ? "" : entry.hash()));
+ post.put("result", result);
+ post.put("reason", reason);
+ post.put("wordh", wordhashes);
+ post.put("lurlEntry", ((entry == null) ? "" : crypt.simpleEncode(entry.toString(), salt)));
+
// determining target address
- String address = targetSeed.getClusterAddress();
+ final String address = target.getClusterAddress();
if (address == null) { return null; }
-
- // sending request
+
+ // send request
try {
return nxTools.table(
- httpc.wget(
- new URL("http://" + address + "/yacy/crawlReceipt.html" +
- "?iam=" + yacyCore.seedDB.mySeed.hash +
- "&youare=" + targetSeed.hash +
- "&process=" + process +
- "&key=" + key +
- "&urlhash=" + ((entry == null) ? "" : entry.hash()) +
- "&result=" + result +
- "&reason=" + reason +
- "&wordh=" + wordhashes +
- "&lurlEntry=" + ((entry == null) ? "" : crypt.simpleEncode(entry.toString(), key)) +
- "&network.unit.name=" + plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT)
- ),
- targetSeed.getHexHash() + ".yacyh",
- 60000,
- null,
- null,
- (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
- null,
- null
- )
- , "UTF-8");
+ httpc.wput(new URL("http://" + address + "/yacy/crawlReceipt.html"),
+ target.getHexHash() + ".yacyh",
+ 60000,
+ null,
+ null,
+ proxyConfig(),
+ post,
+ null
+ ), "UTF-8"
+ );
} catch (Exception e) {
// most probably a network time-out exception
yacyCore.log.logSevere("yacyClient.crawlReceipt error:" + e.getMessage());
@@ -1055,25 +944,16 @@ public final class yacyClient {
private static HashMap transferRWI(yacySeed targetSeed, indexContainer[] indexes, boolean gzipBody, int timeout) {
final String address = targetSeed.getPublicAddress();
if (address == null) { return null; }
-
- // should we use the proxy?
- boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
-
+
// prepare post values
- final serverObjects post = new serverObjects(7);
- final String key = crypt.randomSalt();
+ final serverObjects post = new serverObjects();
+ yacyNetwork.enrichRequestPost(post, plasmaSwitchboard.getSwitchboard(), targetSeed.hash);
// enabling gzip compression for post request body
if ((gzipBody) && (targetSeed.getVersion() >= yacyVersion.YACY_SUPPORTS_GZIP_POST_REQUESTS)) {
post.put(httpc.GZIP_POST_BODY,"true");
}
- post.put("key", key);
- post.put("iam", yacyCore.seedDB.mySeed.hash);
- post.put("youare", targetSeed.hash);
post.put("wordc", Integer.toString(indexes.length));
- post.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
int indexcount = 0;
final StringBuffer entrypost = new StringBuffer(indexes.length*73);
@@ -1108,7 +988,7 @@ public final class yacyClient {
timeout,
null,
null,
- (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
+ proxyConfig(),
post,
null
), "UTF-8");
@@ -1131,26 +1011,16 @@ public final class yacyClient {
// this post a message to the remote message board
final String address = targetSeed.getPublicAddress();
if (address == null) { return null; }
-
- // should we use the proxy?
- boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
-
+
// prepare post values
final serverObjects post = new serverObjects(5+urls.length);
- final String key = crypt.randomSalt();
+ yacyNetwork.enrichRequestPost(post, plasmaSwitchboard.getSwitchboard(), targetSeed.hash);
// enabling gzip compression for post request body
if ((gzipBody) && (targetSeed.getVersion() >= yacyVersion.YACY_SUPPORTS_GZIP_POST_REQUESTS)) {
post.put(httpc.GZIP_POST_BODY,"true");
}
- post.put("key", key);
- post.put("iam", yacyCore.seedDB.mySeed.hash);
- post.put("youare", targetSeed.hash);
- post.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
-
String resource = "";
int urlc = 0;
int urlPayloadSize = 0;
@@ -1173,7 +1043,7 @@ public final class yacyClient {
timeout,
null,
null,
- (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
+ proxyConfig(),
post,
null
), "UTF-8");
@@ -1193,18 +1063,11 @@ public final class yacyClient {
}
public static HashMap getProfile(yacySeed targetSeed) {
-
- // should we use the proxy?
- boolean useProxy = (yacyCore.seedDB.sb.remoteProxyConfig != null) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
- (yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
-
+
// this post a message to the remote message board
final serverObjects post = new serverObjects(2);
- post.put("iam", yacyCore.seedDB.mySeed.hash);
- post.put("youare", targetSeed.hash);
- post.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
-
+ yacyNetwork.enrichRequestPost(post, plasmaSwitchboard.getSwitchboard(), targetSeed.hash);
+
String address = targetSeed.getClusterAddress();
if (address == null) { address = "localhost:8080"; }
try {
@@ -1215,7 +1078,7 @@ public final class yacyClient {
10000,
null,
null,
- (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
+ proxyConfig(),
post,
null
), "UTF-8");
@@ -1224,6 +1087,11 @@ public final class yacyClient {
return null;
}
}
+
+ private static final httpRemoteProxyConfig proxyConfig() {
+ httpRemoteProxyConfig p = plasmaSwitchboard.getSwitchboard().remoteProxyConfig;
+ return ((p != null) && (p.useProxy()) && (p.useProxy4Yacy())) ? p : null;
+ }
public static void main(String[] args) {
System.out.println("yacyClient Test");
diff --git a/source/de/anomic/yacy/yacyNetwork.java b/source/de/anomic/yacy/yacyNetwork.java
new file mode 100644
index 000000000..b51886e4f
--- /dev/null
+++ b/source/de/anomic/yacy/yacyNetwork.java
@@ -0,0 +1,101 @@
+// yacyNetwork.java
+// ----------------
+// (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
+// first published 04.07.2007 on http://yacy.net
+//
+// This is a part of YaCy, a peer-to-peer based web search engine
+//
+// $LastChangedDate: 2007-07-03 22:55:47 +0000 (Di, 03 Jul 2007) $
+// $LastChangedRevision: 3950 $
+// $LastChangedBy: orbiter $
+//
+// LICENSE
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+package de.anomic.yacy;
+
+import java.util.Date;
+
+import de.anomic.plasma.plasmaSwitchboard;
+import de.anomic.server.serverCodings;
+import de.anomic.server.serverObjects;
+import de.anomic.server.serverSwitch;
+import de.anomic.tools.crypt;
+
+public class yacyNetwork {
+
+ public static final boolean authentifyRequest(serverObjects post, serverSwitch env) {
+ if ((post == null) || (env == null)) return false;
+
+ // identify network
+ String unitName = post.get("network.unit.name", yacySeed.DFLT_NETWORK_UNIT); // the network unit
+ if (!unitName.equals(env.getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT))) {
+ return false;
+ }
+
+ // check authentification method
+ String authentificationControl = env.getConfig("network.unit.protocol.control", "uncontrolled");
+ if (authentificationControl.equals("uncontrolled")) return true;
+ String authentificationMethod = env.getConfig("network.unit.protocol.request.authentification.method", "");
+ if (authentificationMethod.length() == 0) {
+ return false;
+ }
+ if (authentificationMethod.equals("salted-magic-sim")) {
+ // authentify the peer using the md5-magic
+ String salt = post.get("key", "");
+ String iam = post.get("iam", "");
+ String magic = env.getConfig("network.unit.protocol.request.authentification.essentials", "");
+ String md5 = serverCodings.encodeMD5Hex(salt + iam + magic);
+ return post.get("magicmd5", "").equals(md5);
+ }
+
+ // unknown authentification method
+ return false;
+ }
+
+ public static final String enrichRequestPost(serverObjects post, serverSwitch env, String targetHash) {
+ // put in all the essentials for routing and network authentification
+ // generate a session key
+ final String salt = crypt.randomSalt();
+ post.put("key", salt);
+
+ // just standard identification essentials
+ post.put("iam", yacyCore.seedDB.mySeed.hash);
+ if (targetHash != null) post.put("youare", targetHash);
+
+ // time information for synchronization
+ post.put("mytime", yacyCore.universalDateShortString(new Date()));
+ post.put("myUTC", System.currentTimeMillis());
+
+ // network identification
+ post.put("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT));
+
+ // authentification essentials
+ String authentificationControl = env.getConfig("network.unit.protocol.control", "uncontrolled");
+ String authentificationMethod = env.getConfig("network.unit.protocol.request.authentification.method", "");
+ if ((authentificationControl.equals("controlled")) && (authentificationMethod.length() > 0)) {
+ if (authentificationMethod.equals("salted-magic-sim")) {
+ // generate an authentification essential using the salt, the iam-hash and the network magic
+ String magic = env.getConfig("network.unit.protocol.request.authentification.essentials", "");
+ String md5 = serverCodings.encodeMD5Hex(salt + yacyCore.seedDB.mySeed.hash + magic);
+ post.put("magicmd5", md5);
+ }
+ }
+
+ return salt;
+ }
+
+}
diff --git a/yacy.network.unit b/yacy.network.unit
index 53e800a40..d7106b2b4 100644
--- a/yacy.network.unit
+++ b/yacy.network.unit
@@ -69,8 +69,8 @@ network.unit.update.location1 = http://latest.yacy.net
network.unit.update.location2 = http://latest.yacy-forum.net
# properties for in-protocol response authentification:
-#network.unit.protocol.control = uncontrolled
-#network.unit.protocol.request.authentification.method = salted-magic
+network.unit.protocol.control = uncontrolled
+#network.unit.protocol.request.authentification.method = salted-magic-sim
#network.unit.protocol.request.authentification.essentials =
#network.unit.protocol.response.authentification.method = challenge-pwresponse
#network.unit.protocol.response.authentification.essentials =