boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

finals;

Browse Source 
cleaned;
Properties;

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@767 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
borg-0300 20 years ago
parent ba145efb46
commit 1dd7047af5
2 changed files with 332 additions and 323 deletions
Show Stats Download Patch File Download Diff File

287
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -3,7 +3,10 @@
// (C) by Michael Peter Christen; mc@anomic.de // (C) by Michael Peter Christen; mc@anomic.de
// first published on http://www.anomic.de // first published on http://www.anomic.de
// Frankfurt, Germany, 2004 // Frankfurt, Germany, 2004
// last major change: 02.12.2004 //
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
// //
// This program is free software; you can redistribute it and/or modify // This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by // it under the terms of the GNU General Public License as published by
@ -45,10 +48,7 @@ import java.net.URL;
import java.util.Date; import java.util.Date;
import java.util.Enumeration; import java.util.Enumeration;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Vector; import java.util.Vector;
import de.anomic.http.httpc; import de.anomic.http.httpc;
import de.anomic.plasma.plasmaCrawlLURL; import de.anomic.plasma.plasmaCrawlLURL;
import de.anomic.plasma.plasmaSearch; import de.anomic.plasma.plasmaSearch;
@ -62,9 +62,10 @@ import de.anomic.server.serverCore;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.tools.crypt; import de.anomic.tools.crypt;
import de.anomic.tools.nxTools; import de.anomic.tools.nxTools;
import de.anomic.yacy.yacySeed;
public class yacyClient { public class yacyClient {
public static int publishMySeed(String address, String otherHash) { public static int publishMySeed(String address, String otherHash) {
// this is called to enrich the seed information by // this is called to enrich the seed information by
// - own address (if peer is behind a nat/router) // - own address (if peer is behind a nat/router)
@ -84,7 +85,7 @@ public class yacyClient {
// but they appear to be another peer by comparisment of the other peer's hash // but they appear to be another peer by comparisment of the other peer's hash
// this works of course only if we know the other peer's hash. // this works of course only if we know the other peer's hash.
String key = crypt.randomSalt(); final String key = crypt.randomSalt();
HashMap result = null; HashMap result = null;
try { try {
/* /*
@ -96,9 +97,9 @@ public class yacyClient {
result = nxTools.table(httpc.wget(url, result = nxTools.table(httpc.wget(url,
10000, null, null, yacyCore.seedCache.sb.remoteProxyHost, yacyCore.seedCache.sb.remoteProxyPort)); 10000, null, null, yacyCore.seedCache.sb.remoteProxyHost, yacyCore.seedCache.sb.remoteProxyPort));
*/ */
URL url = new URL("http://" + address + "/yacy/hello.html"); final URL url = new URL("http://" + address + "/yacy/hello.html");
serverObjects obj = new serverObjects(); final serverObjects obj = new serverObjects();
obj.put("iam", yacyCore.seedDB.mySeed.hash); obj.put("iam", yacyCore.seedDB.mySeed.hash);
obj.put("pattern", ""); obj.put("pattern", "");
obj.put("count", "20"); obj.put("count", "20");
@ -118,36 +119,36 @@ public class yacyClient {
} }
return -1; return -1;
} }
if ((result == null) || (result.size() < 3)) { if (result == null || result.size() < 3) {
yacyCore.log.logFine("yacyClient.publishMySeed result error: " + yacyCore.log.logFine("yacyClient.publishMySeed result error: " +
((result == null) ? "result null" : ("result=" + result.toString()))); ((result == null) ? "result null" : ("result=" + result.toString())));
return -1; return -1;
} }
Date remoteTime = yacyCore.parseUniversalDate((String) result.get("mytime")); // read remote time final Date remoteTime = yacyCore.parseUniversalDate((String) result.get("mytime")); // read remote time
// check consistency with expectation // check consistency with expectation
yacySeed otherPeer = null; yacySeed otherPeer = null;
float otherPeerVersion = 0; float otherPeerVersion = 0;
if ((otherHash != null ) && (otherHash.length() > 0)) { if (otherHash != null && otherHash.length() > 0) {
otherPeer = yacySeed.genRemoteSeed((String) result.get("seed0"), key, remoteTime); otherPeer = yacySeed.genRemoteSeed((String) result.get("seed0"), key, remoteTime);
if ((otherPeer == null) || (!(otherPeer.hash.equals(otherHash)))) { if (otherPeer == null || !otherPeer.hash.equals(otherHash)) {
yacyCore.log.logFine("yacyClient.publishMySeed: consistency error: other peer '" + ((otherPeer==null)?"unknown":otherPeer.getName()) + "' wrong"); yacyCore.log.logFine("yacyClient.publishMySeed: consistency error: other peer '" + ((otherPeer==null)?"unknown":otherPeer.getName()) + "' wrong");
return -1; // no success return -1; // no success
} }
otherPeerVersion = otherPeer.getVersion(); otherPeerVersion = otherPeer.getVersion();
} }
// set my own seed according to new information // set my own seed according to new information
yacySeed mySeedBkp = (yacySeed) yacyCore.seedDB.mySeed.clone(); final yacySeed mySeedBkp = (yacySeed) yacyCore.seedDB.mySeed.clone();
// we overwrite our own IP number only, if we do not portForwarding // we overwrite our own IP number only, if we do not portForwarding
if (!serverCore.portForwardingEnabled) { if (serverCore.portForwardingEnabled) {
yacyCore.seedDB.mySeed.put("IP", (String) result.get("yourip"));
} else {
yacyCore.seedDB.mySeed.put("IP", serverCore.publicIP()); yacyCore.seedDB.mySeed.put("IP", serverCore.publicIP());
} else {
yacyCore.seedDB.mySeed.put("IP", (String) result.get("yourip"));
} }
/* If we have port forwarding enabled but the other peer uses a too old yacy version /* If we have port forwarding enabled but the other peer uses a too old yacy version
* we can ignore the seed-type that was reported by the peer. * we can ignore the seed-type that was reported by the peer.
* *
@ -155,16 +156,16 @@ public class yacyClient {
* *
* @see serverCore#portForwardingEnabled * @see serverCore#portForwardingEnabled
*/ */
if ((!serverCore.portForwardingEnabled) || (otherPeerVersion >= (float)0.383)) { if (!serverCore.portForwardingEnabled || otherPeerVersion >= (float)0.383) {
String mytype = (String) result.get("yourtype"); String mytype = (String) result.get("yourtype");
if (mytype == null) mytype = yacySeed.PEERTYPE_JUNIOR; if (mytype == null) { mytype = yacySeed.PEERTYPE_JUNIOR; }
if ( if (
(yacyCore.seedDB.mySeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR).equals(yacySeed.PEERTYPE_PRINCIPAL)) && (yacyCore.seedDB.mySeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR).equals(yacySeed.PEERTYPE_PRINCIPAL)) &&
(mytype.equals(yacySeed.PEERTYPE_SENIOR)) (mytype.equals(yacySeed.PEERTYPE_SENIOR))
) { ) {
mytype = yacySeed.PEERTYPE_PRINCIPAL; mytype = yacySeed.PEERTYPE_PRINCIPAL;
} }
/* /*
* If we were reported as junior we have to check if your port forwarding channel is broken * If we were reported as junior we have to check if your port forwarding channel is broken
* If this is true we try to reconnect the sch channel to the remote server now. * If this is true we try to reconnect the sch channel to the remote server now.
@ -172,10 +173,9 @@ public class yacyClient {
if (mytype.equalsIgnoreCase(yacySeed.PEERTYPE_JUNIOR)) { if (mytype.equalsIgnoreCase(yacySeed.PEERTYPE_JUNIOR)) {
yacyCore.log.logInfo("yacyClient.publishMySeed: Peer '" + ((otherPeer==null)?"unknown":otherPeer.getName()) + "' reported us as junior."); yacyCore.log.logInfo("yacyClient.publishMySeed: Peer '" + ((otherPeer==null)?"unknown":otherPeer.getName()) + "' reported us as junior.");
if (serverCore.portForwardingEnabled) { if (serverCore.portForwardingEnabled) {
if ( if (!Thread.currentThread().isInterrupted() &&
(!Thread.currentThread().isInterrupted()) && serverCore.portForwarding != null &&
(serverCore.portForwarding != null) && !serverCore.portForwarding.isConnected()
(!serverCore.portForwarding.isConnected())
) { ) {
yacyCore.log.logWarning("yacyClient.publishMySeed: Broken portForwarding channel detected. Trying to reconnect ..."); yacyCore.log.logWarning("yacyClient.publishMySeed: Broken portForwarding channel detected. Trying to reconnect ...");
try { try {
@ -190,18 +190,18 @@ public class yacyClient {
} }
yacyCore.seedDB.mySeed.put(yacySeed.PEERTYPE, mytype); yacyCore.seedDB.mySeed.put(yacySeed.PEERTYPE, mytype);
} }
String error; final String error = yacyCore.seedDB.mySeed.isProper();
if ((error = yacyCore.seedDB.mySeed.isProper()) != null) { if (error != null) {
yacyCore.seedDB.mySeed = mySeedBkp; yacyCore.seedDB.mySeed = mySeedBkp;
yacyCore.log.logFine("yacyClient.publishMySeed mySeed error - not proper: " + error); yacyCore.log.logFine("yacyClient.publishMySeed mySeed error - not proper: " + error);
return -1; return -1;
} }
// read the seeds that the peer returned and integrate them into own database // read the seeds that the peer returned and integrate them into own database
int i = 0; int i = 0;
String seedStr;
int count = 0; int count = 0;
String seedStr;
while ((seedStr = (String) result.get("seed" + i++)) != null) { while ((seedStr = (String) result.get("seed" + i++)) != null) {
// integrate new seed into own database // integrate new seed into own database
// the first seed, "seed0" is the seed of the responding peer // the first seed, "seed0" is the seed of the responding peer
@ -209,66 +209,65 @@ public class yacyClient {
} }
return count; return count;
} }
public static yacySeed querySeed(yacySeed target, String seedHash) { public static yacySeed querySeed(yacySeed target, String seedHash) {
String key = crypt.randomSalt(); final String key = crypt.randomSalt();
try { try {
HashMap result = nxTools.table(httpc.wget( final HashMap result = nxTools.table(httpc.wget(
new URL("http://" + target.getAddress() + new URL("http://" + target.getAddress() +
"/yacy/query.html?iam=" + yacyCore.seedDB.mySeed.hash + "/yacy/query.html?iam=" + yacyCore.seedDB.mySeed.hash +
"&youare=" + target.hash + "&key=" + key + "&youare=" + target.hash + "&key=" + key +
"&object=seed&env=" + seedHash), "&object=seed&env=" + seedHash),
10000, null, null, yacyCore.seedDB.sb.remoteProxyHost, yacyCore.seedDB.sb.remoteProxyPort)); 10000, null, null, yacyCore.seedDB.sb.remoteProxyHost, yacyCore.seedDB.sb.remoteProxyPort));
if ((result == null) || (result.size() == 0)) return null; if (result == null || result.size() == 0) { return null; }
Date remoteTime = yacyCore.parseUniversalDate((String) result.get("mytime")); // read remote time final Date remoteTime = yacyCore.parseUniversalDate((String) result.get("mytime")); // read remote time
return yacySeed.genRemoteSeed((String) result.get("response"), key, remoteTime); return yacySeed.genRemoteSeed((String) result.get("response"), key, remoteTime);
} catch (Exception e) { } catch (Exception e) {
yacyCore.log.logSevere("yacyClient.querySeed error:" + e.getMessage()); yacyCore.log.logSevere("yacyClient.querySeed error:" + e.getMessage());
return null; return null;
} }
} }
public static int queryRWICount(yacySeed target, String wordHash) { public static int queryRWICount(yacySeed target, String wordHash) {
try { try {
HashMap result = nxTools.table(httpc.wget( final HashMap result = nxTools.table(httpc.wget(
new URL("http://" + target.getAddress() + new URL("http://" + target.getAddress() +
"/yacy/query.html?iam=" + yacyCore.seedDB.mySeed.hash + "/yacy/query.html?iam=" + yacyCore.seedDB.mySeed.hash +
"&youare=" + target.hash + "&key=" + "&youare=" + target.hash + "&key=" +
"&object=rwicount&env=" + wordHash + "&object=rwicount&env=" + wordHash +
"&ttl=0"), "&ttl=0"),
10000, null, null, yacyCore.seedDB.sb.remoteProxyHost, yacyCore.seedDB.sb.remoteProxyPort)); 10000, null, null, yacyCore.seedDB.sb.remoteProxyHost, yacyCore.seedDB.sb.remoteProxyPort));
if ((result == null) || (result.size() == 0)) return -1; if (result == null || result.size() == 0) { return -1; }
return Integer.parseInt((String) result.get("response")); return Integer.parseInt((String) result.get("response"));
} catch (Exception e) { } catch (Exception e) {
yacyCore.log.logSevere("yacyClient.queryRWICount error:" + e.getMessage()); yacyCore.log.logSevere("yacyClient.queryRWICount error:" + e.getMessage());
return -1; return -1;
} }
} }
public static int queryUrlCount(yacySeed target) { public static int queryUrlCount(yacySeed target) {
if (target == null) return -1; if (target == null) { return -1; }
if (yacyCore.seedDB.mySeed == null) return -1; if (yacyCore.seedDB.mySeed == null) return -1;
String querystr = final String querystr =
"http://" + target.getAddress() + "http://" + target.getAddress() +
"/yacy/query.html?iam=" + yacyCore.seedDB.mySeed.hash + "/yacy/query.html?iam=" + yacyCore.seedDB.mySeed.hash +
"&youare=" + target.hash + "&youare=" + target.hash +
"&key=" + "&key=" +
"&object=lurlcount&env=&ttl=0"; "&object=lurlcount&env=&ttl=0";
try { try {
HashMap result = nxTools.table(httpc.wget( final HashMap result = nxTools.table(httpc.wget(
new URL(querystr), 6000, null, null, new URL(querystr), 6000, null, null,
yacyCore.seedDB.sb.remoteProxyHost, yacyCore.seedDB.sb.remoteProxyPort)); yacyCore.seedDB.sb.remoteProxyHost, yacyCore.seedDB.sb.remoteProxyPort));
//yacyCore.log("DEBUG QUERY: query=" + querystr + "; result = " + result.toString()); // yacyCore.log("DEBUG QUERY: query=" + querystr + "; result = " + result.toString());
if ((result == null) || (result.size() == 0)) return -1; if ((result == null) || (result.size() == 0)) return -1;
String resp = (String) result.get("response"); final String resp = (String) result.get("response");
if (resp == null) return -1; else return Integer.parseInt(resp); if (resp == null) { return -1; } else { return Integer.parseInt(resp); }
} catch (Exception e) { } catch (Exception e) {
yacyCore.log.logSevere("yacyClient.queryUrlCount error asking peer '" + target.getName() + "':" + e.toString()); yacyCore.log.logSevere("yacyClient.queryUrlCount error asking peer '" + target.getName() + "':" + e.toString());
return -1; return -1;
} }
} }
public static int search(String wordhashes, int count, boolean global, public static int search(String wordhashes, int count, boolean global,
yacySeed targetPeer, plasmaCrawlLURL urlManager, yacySeed targetPeer, plasmaCrawlLURL urlManager,
plasmaSearch searchManager, plasmaURLPattern blacklist, plasmaSearch searchManager, plasmaURLPattern blacklist,
@ -276,7 +275,7 @@ public class yacyClient {
long duetime) { long duetime) {
// send a search request to peer with remote Hash // send a search request to peer with remote Hash
// this mainly converts the words into word hashes // this mainly converts the words into word hashes
// INPUT: // INPUT:
// iam : complete seed of the requesting peer // iam : complete seed of the requesting peer
// youare : seed hash of the target peer, used for testing network stability // youare : seed hash of the target peer, used for testing network stability
@ -288,11 +287,11 @@ public class yacyClient {
// count : maximum number of wanted results // count : maximum number of wanted results
// global : if "true", then result may consist of answers from other peers // global : if "true", then result may consist of answers from other peers
// duetime : maximum time that a peer should spent to create a result // duetime : maximum time that a peer should spent to create a result
// request result // request result
String key = crypt.randomSalt(); final String key = crypt.randomSalt();
try { try {
String url = "http://" + targetPeer.getAddress() + "/yacy/search.html"; final String url = "http://" + targetPeer.getAddress() + "/yacy/search.html";
/* /*
String url = "http://" + targetPeer.getAddress() + String url = "http://" + targetPeer.getAddress() +
"/yacy/search.html?myseed=" + yacyCore.seedCache.mySeed.genSeedStr(key) + "/yacy/search.html?myseed=" + yacyCore.seedCache.mySeed.genSeedStr(key) +
@ -301,7 +300,7 @@ public class yacyClient {
"&count=" + count + "&resource=" + ((global) ? "global" : "local") + "&count=" + count + "&resource=" + ((global) ? "global" : "local") +
"&query=" + wordhashes; "&query=" + wordhashes;
*/ */
serverObjects obj = new serverObjects(); final serverObjects obj = new serverObjects();
obj.put("myseed", yacyCore.seedDB.mySeed.genSeedStr(key)); obj.put("myseed", yacyCore.seedDB.mySeed.genSeedStr(key));
obj.put("youare", targetPeer.hash); obj.put("youare", targetPeer.hash);
obj.put("key", key); obj.put("key", key);
@ -312,13 +311,13 @@ public class yacyClient {
obj.put("duetime", Long.toString(duetime)); obj.put("duetime", Long.toString(duetime));
obj.put("mytime", yacyCore.universalDateShortString()); obj.put("mytime", yacyCore.universalDateShortString());
//yacyCore.log.logDebug("yacyClient.search url=" + url); //yacyCore.log.logDebug("yacyClient.search url=" + url);
long timestamp = System.currentTimeMillis(); final long timestamp = System.currentTimeMillis();
HashMap result = nxTools.table(httpc.wput(new URL(url), final HashMap result = nxTools.table(httpc.wput(new URL(url),
300000, null, null, 300000, null, null,
yacyCore.seedDB.sb.remoteProxyHost, yacyCore.seedDB.sb.remoteProxyHost,
yacyCore.seedDB.sb.remoteProxyPort, yacyCore.seedDB.sb.remoteProxyPort,
obj)); obj));
long totalrequesttime = System.currentTimeMillis() - timestamp; final long totalrequesttime = System.currentTimeMillis() - timestamp;
/* /*
HashMap result = nxTools.table(httpc.wget(new URL(url), HashMap result = nxTools.table(httpc.wget(new URL(url),
@ -335,29 +334,29 @@ public class yacyClient {
// fwrec : peers that would have helped to construct this result (recommendations) // fwrec : peers that would have helped to construct this result (recommendations)
// searchtime : time that the peer actually spent to create the result // searchtime : time that the peer actually spent to create the result
// references : references (search hints) that was calculated during search // references : references (search hints) that was calculated during search
// now create a plasmaIndex out of this result // now create a plasmaIndex out of this result
//System.out.println("yacyClient: search result = " + result.toString()); // debug //System.out.println("yacyClient: search result = " + result.toString()); // debug
int results = Integer.parseInt((String) result.get("count")); final int results = Integer.parseInt((String) result.get("count"));
//System.out.println("***result count " + results); //System.out.println("***result count " + results);
plasmaCrawlLURL.Entry link; plasmaCrawlLURL.Entry link;
// create containers // create containers
int words = wordhashes.length() / plasmaWordIndexEntry.wordHashLength; final int words = wordhashes.length() / plasmaWordIndexEntry.wordHashLength;
plasmaWordIndexEntryContainer[] container = new plasmaWordIndexEntryContainer[words]; plasmaWordIndexEntryContainer[] container = new plasmaWordIndexEntryContainer[words];
for (int i = 0; i < words; i++) { for (int i = 0; i < words; i++) {
container[i] = new plasmaWordIndexEntryContainer(wordhashes.substring(i * plasmaWordIndexEntry.wordHashLength, (i + 1) * plasmaWordIndexEntry.wordHashLength)); container[i] = new plasmaWordIndexEntryContainer(wordhashes.substring(i * plasmaWordIndexEntry.wordHashLength, (i + 1) * plasmaWordIndexEntry.wordHashLength));
} }
// insert results to containers // insert results to containers
plasmaCrawlLURL.Entry lEntry; plasmaCrawlLURL.Entry lEntry;
for (int n = 0; n < results; n++) { for (int n = 0; n < results; n++) {
// get one single search result // get one single search result
lEntry = urlManager.newEntry((String) result.get("resource" + n), true); lEntry = urlManager.newEntry((String) result.get("resource" + n), true);
if ((lEntry != null) && (blacklist.isListed(lEntry.url().getHost().toLowerCase(), lEntry.url().getPath()))) continue; // block with backlist if (lEntry != null && blacklist.isListed(lEntry.url().getHost().toLowerCase(), lEntry.url().getPath())) { continue; } // block with backlist
link = urlManager.addEntry(lEntry, yacyCore.seedDB.mySeed.hash, targetPeer.hash, 2); link = urlManager.addEntry(lEntry, yacyCore.seedDB.mySeed.hash, targetPeer.hash, 2);
// save the url entry // save the url entry
plasmaWordIndexEntry entry = new plasmaWordIndexEntry(link.hash(), link.wordCount(), 0, 0, 0, final plasmaWordIndexEntry entry = new plasmaWordIndexEntry(link.hash(), link.wordCount(), 0, 0, 0,
plasmaSearch.calcVirtualAge(link.moddate()), link.quality(), plasmaSearch.calcVirtualAge(link.moddate()), link.quality(),
link.language(), link.doctype(), false); link.language(), link.doctype(), false);
if (link.snippet() != null) { if (link.snippet() != null) {
@ -371,9 +370,9 @@ public class yacyClient {
container[m].add(new plasmaWordIndexEntry[]{entry}, System.currentTimeMillis()); container[m].add(new plasmaWordIndexEntry[]{entry}, System.currentTimeMillis());
} }
} }
// finally insert the containers to the index // finally insert the containers to the index
for (int m = 0; m < words; m++) searchManager.addWords(container[m]); for (int m = 0; m < words; m++) { searchManager.addWords(container[m]); }
// generate statistics // generate statistics
long searchtime; long searchtime;
@ -390,13 +389,13 @@ public class yacyClient {
return 0; return 0;
} }
} }
public static HashMap permissionMessage(String targetHash) { public static HashMap permissionMessage(String targetHash) {
// ask for allowed message size and attachement size // ask for allowed message size and attachement size
// if this replies null, the peer does not answer // if this replies null, the peer does not answer
if ((yacyCore.seedDB == null) || (yacyCore.seedDB.mySeed == null)) return null; if (yacyCore.seedDB == null || yacyCore.seedDB.mySeed == null) { return null; }
serverObjects post = new serverObjects(); final serverObjects post = new serverObjects();
String key = crypt.randomSalt(); final String key = crypt.randomSalt();
post.put("key", key); post.put("key", key);
post.put("process", "permission"); post.put("process", "permission");
post.put("iam", yacyCore.seedDB.mySeed.hash); post.put("iam", yacyCore.seedDB.mySeed.hash);
@ -407,27 +406,27 @@ public class yacyClient {
address = yacyCore.seedDB.mySeed.getAddress(); address = yacyCore.seedDB.mySeed.getAddress();
//System.out.println("local address: " + address); //System.out.println("local address: " + address);
} else { } else {
yacySeed targetSeed = yacyCore.seedDB.getConnected(targetHash); final yacySeed targetSeed = yacyCore.seedDB.getConnected(targetHash);
if (targetSeed == null) return null; if (targetSeed == null) { return null; }
address = targetSeed.getAddress(); address = targetSeed.getAddress();
//System.out.println("remote address: " + address); //System.out.println("remote address: " + address);
} }
if (address == null) address = "localhost:8080"; if (address == null) { address = "localhost:8080"; }
try { try {
return nxTools.table(httpc.wput( return nxTools.table(httpc.wput(
new URL("http://" + address + "/yacy/message.html"), new URL("http://" + address + "/yacy/message.html"),
8000, null, null, yacyCore.seedDB.sb.remoteProxyHost, yacyCore.seedDB.sb.remoteProxyPort, post)); 8000, null, null, yacyCore.seedDB.sb.remoteProxyHost, yacyCore.seedDB.sb.remoteProxyPort, post));
} catch (Exception e) { } catch (Exception e) {
// most probably a network time-out exception // most probably a network time-out exception
yacyCore.log.logSevere("yacyClient.permissionMessage error:" + e.getMessage()); yacyCore.log.logSevere("yacyClient.permissionMessage error:" + e.getMessage());
return null; return null;
} }
} }
public static HashMap postMessage(String targetHash, String subject, byte[] message) { public static HashMap postMessage(String targetHash, String subject, byte[] message) {
// this post a message to the remote message board // this post a message to the remote message board
serverObjects post = new serverObjects(); final serverObjects post = new serverObjects();
String key = crypt.randomSalt(); final String key = crypt.randomSalt();
post.put("key", key); post.put("key", key);
post.put("process", "post"); post.put("process", "post");
post.put("myseed", yacyCore.seedDB.mySeed.genSeedStr(key)); post.put("myseed", yacyCore.seedDB.mySeed.genSeedStr(key));
@ -436,14 +435,15 @@ public class yacyClient {
post.put("mytime", yacyCore.universalDateShortString()); post.put("mytime", yacyCore.universalDateShortString());
post.put("message", new String(message)); post.put("message", new String(message));
String address; String address;
if (targetHash.equals(yacyCore.seedDB.mySeed.hash)) if (targetHash.equals(yacyCore.seedDB.mySeed.hash)) {
address = yacyCore.seedDB.mySeed.getAddress(); address = yacyCore.seedDB.mySeed.getAddress();
else } else {
address = yacyCore.seedDB.getConnected(targetHash).getAddress(); address = yacyCore.seedDB.getConnected(targetHash).getAddress();
if (address == null) address = "localhost:8080"; }
if (address == null) { address = "localhost:8080"; }
//System.out.println("DEBUG POST " + address + "/yacy/message.html" + post.toString()); //System.out.println("DEBUG POST " + address + "/yacy/message.html" + post.toString());
try { try {
Vector v = httpc.wput(new URL("http://" + address + "/yacy/message.html"), 20000, null, null, final Vector v = httpc.wput(new URL("http://" + address + "/yacy/message.html"), 20000, null, null,
yacyCore.seedDB.sb.remoteProxyHost, yacyCore.seedDB.sb.remoteProxyPort, post); yacyCore.seedDB.sb.remoteProxyHost, yacyCore.seedDB.sb.remoteProxyPort, post);
//System.out.println("V=" + v.toString()); //System.out.println("V=" + v.toString());
return nxTools.table(v); return nxTools.table(v);
@ -452,16 +452,16 @@ public class yacyClient {
return null; return null;
} }
} }
public static HashMap crawlOrder(yacySeed targetSeed, URL url, URL referrer) { public static HashMap crawlOrder(yacySeed targetSeed, URL url, URL referrer) {
// this post a message to the remote message board // this post a message to the remote message board
if (targetSeed == null) return null; if (targetSeed == null) { return null; }
if (yacyCore.seedDB.mySeed == null) return null; if (yacyCore.seedDB.mySeed == null) { return null; }
if (yacyCore.seedDB.mySeed == targetSeed) return null; if (yacyCore.seedDB.mySeed == targetSeed) { return null; }
// construct request // construct request
serverObjects post = new serverObjects(); final serverObjects post = new serverObjects();
String key = crypt.randomSalt(); final String key = crypt.randomSalt();
post.put("key", key); post.put("key", key);
post.put("process", "crawl"); post.put("process", "crawl");
post.put("iam", yacyCore.seedDB.mySeed.hash); post.put("iam", yacyCore.seedDB.mySeed.hash);
@ -471,9 +471,9 @@ public class yacyClient {
post.put("referrer", crypt.simpleEncode((referrer == null) ? "" : referrer.toString())); post.put("referrer", crypt.simpleEncode((referrer == null) ? "" : referrer.toString()));
post.put("depth", "0"); post.put("depth", "0");
post.put("ttl", "0"); post.put("ttl", "0");
String address = targetSeed.getAddress(); final String address = targetSeed.getAddress();
if (address == null) return null; if (address == null) { return null; }
try { try {
return nxTools.table(httpc.wput( return nxTools.table(httpc.wput(
new URL("http://" + address + "/yacy/crawlOrder.html"), new URL("http://" + address + "/yacy/crawlOrder.html"),
@ -484,19 +484,19 @@ public class yacyClient {
return null; return null;
} }
} }
/* /*
Test: Test:
http://217.234.95.114:5777/yacy/crawlOrder.html?key=abc&iam=S-cjM67KhtcJ&youare=EK31N7RgRqTn&process=crawl&referrer=&depth=0&url=p|http://www.heise.de/newsticker/meldung/53245 http://217.234.95.114:5777/yacy/crawlOrder.html?key=abc&iam=S-cjM67KhtcJ&youare=EK31N7RgRqTn&process=crawl&referrer=&depth=0&url=p|http://www.heise.de/newsticker/meldung/53245
version=0.297 uptime=225 accepted=true reason=ok delay=30 depth=0 version=0.297 uptime=225 accepted=true reason=ok delay=30 depth=0
-er crawlt, Ergebnis erscheint aber unter falschem initiator -er crawlt, Ergebnis erscheint aber unter falschem initiator
*/ */
public static HashMap crawlReceipt(yacySeed targetSeed, String process, String result, String reason, plasmaCrawlLURL.Entry entry, String wordhashes) { public static HashMap crawlReceipt(yacySeed targetSeed, String process, String result, String reason, plasmaCrawlLURL.Entry entry, String wordhashes) {
if (targetSeed == null) return null; if (targetSeed == null) { return null; }
if (yacyCore.seedDB.mySeed == null) return null; if (yacyCore.seedDB.mySeed == null) { return null; }
if (yacyCore.seedDB.mySeed == targetSeed) return null; if (yacyCore.seedDB.mySeed == targetSeed) { return null; }
/* /*
the result can have one of the following values: the result can have one of the following values:
negative cases, no retry negative cases, no retry
@ -516,12 +516,12 @@ public class yacyClient {
stale - the resource was reloaded but not processed because source had no changes stale - the resource was reloaded but not processed because source had no changes
*/ */
// construct request // construct request
String key = crypt.randomSalt(); final String key = crypt.randomSalt();
String address = targetSeed.getAddress(); String address = targetSeed.getAddress();
if (address == null) return null; if (address == null) { return null; }
try { try {
return nxTools.table(httpc.wget( return nxTools.table(httpc.wget(
new URL("http://" + address + "/yacy/crawlReceipt.html?" + new URL("http://" + address + "/yacy/crawlReceipt.html?" +
@ -547,20 +547,20 @@ public class yacyClient {
String user, String password, String user, String password,
httpHeader requestHeader) throws IOException { httpHeader requestHeader) throws IOException {
*/ */
public static String transferIndex(yacySeed targetSeed, plasmaWordIndexEntity[] indexes, HashMap urlCache) { public static String transferIndex(yacySeed targetSeed, plasmaWordIndexEntity[] indexes, HashMap urlCache) {
HashMap in = transferRWI(targetSeed, indexes); HashMap in = transferRWI(targetSeed, indexes);
if (in == null) return "no_connection_1"; if (in == null) { return "no_connection_1"; }
String result = (String) in.get("result"); String result = (String) in.get("result");
if (result == null) return "no_result_1"; if (result == null) { return "no_result_1"; }
if (!(result.equals("ok"))) return result; if (!(result.equals("ok"))) return result;
// in now contains a list of unknown hashes // in now contains a list of unknown hashes
String uhss = (String) in.get("unknownURL"); final String uhss = (String) in.get("unknownURL");
if (uhss == null) return "no_unknownURL_tag_in_response"; if (uhss == null) { return "no_unknownURL_tag_in_response"; }
if (uhss.length() == 0) return null; // all url's known, we are ready here if (uhss.length() == 0) { return null; } // all url's known, we are ready here
String[] uhs = uhss.split(","); final String[] uhs = uhss.split(",");
//System.out.println("DEBUG yacyClient.transferIndex: " + uhs.length + " urls unknown"); // System.out.println("DEBUG yacyClient.transferIndex: " + uhs.length + " urls unknown");
if (uhs.length == 0) return null; // all url's known if (uhs.length == 0) { return null; } // all url's known
// extract the urlCache from the result // extract the urlCache from the result
plasmaCrawlLURL.Entry[] urls = new plasmaCrawlLURL.Entry[uhs.length]; plasmaCrawlLURL.Entry[] urls = new plasmaCrawlLURL.Entry[uhs.length];
for (int i = 0; i < uhs.length; i++) { for (int i = 0; i < uhs.length; i++) {
@ -568,27 +568,27 @@ public class yacyClient {
if (urls[i] == null) System.out.println("DEBUG transferIndex: error with requested url hash '" + uhs[i] + "', unknownURL='" + uhss + "'"); if (urls[i] == null) System.out.println("DEBUG transferIndex: error with requested url hash '" + uhs[i] + "', unknownURL='" + uhss + "'");
} }
in = transferURL(targetSeed, urls); in = transferURL(targetSeed, urls);
if (in == null) return "no_connection_2"; if (in == null) { return "no_connection_2"; }
result = (String) in.get("result"); result = (String) in.get("result");
if (result == null) return "no_result_2"; if (result == null) { return "no_result_2"; }
if (!(result.equals("ok"))) return result; if (!(result.equals("ok"))) { return result; }
int doubleentries = Integer.parseInt((String) in.get("double")); // int doubleentries = Integer.parseInt((String) in.get("double"));
//System.out.println("DEBUG tansferIndex: transferred " + uhs.length + " URL's, double=" + doubleentries); // System.out.println("DEBUG tansferIndex: transferred " + uhs.length + " URL's, double=" + doubleentries);
return null; return null;
} }
private static HashMap transferRWI(yacySeed targetSeed, plasmaWordIndexEntity[] indexes) { private static HashMap transferRWI(yacySeed targetSeed, plasmaWordIndexEntity[] indexes) {
String address = targetSeed.getAddress(); final String address = targetSeed.getAddress();
if (address == null) return null; if (address == null) { return null; }
// prepare post values // prepare post values
serverObjects post = new serverObjects(); final serverObjects post = new serverObjects();
String key = crypt.randomSalt(); final String key = crypt.randomSalt();
post.put("key", key); post.put("key", key);
post.put("iam", yacyCore.seedDB.mySeed.hash); post.put("iam", yacyCore.seedDB.mySeed.hash);
post.put("youare", targetSeed.hash); post.put("youare", targetSeed.hash);
post.put("wordc", Integer.toString(indexes.length)); post.put("wordc", Integer.toString(indexes.length));
int indexcount = 0; int indexcount = 0;
StringBuffer entrypost = new StringBuffer(indexes.length*73); final StringBuffer entrypost = new StringBuffer(indexes.length*73);
Enumeration eenum; Enumeration eenum;
plasmaWordIndexEntry entry; plasmaWordIndexEntry entry;
for (int i = 0; i < indexes.length; i++) { for (int i = 0; i < indexes.length; i++) {
@ -601,40 +601,40 @@ public class yacyClient {
indexcount++; indexcount++;
} }
} }
if (indexcount == 0) { if (indexcount == 0) {
// nothing to do but everything ok // nothing to do but everything ok
HashMap result = new HashMap(); final HashMap result = new HashMap();
result.put("result", "ok"); result.put("result", "ok");
result.put("unknownURL", ""); result.put("unknownURL", "");
return result; return result;
} }
post.put("entryc", Integer.toString(indexcount)); post.put("entryc", Integer.toString(indexcount));
post.put("indexes", entrypost.toString()); post.put("indexes", entrypost.toString());
try { try {
Vector v = httpc.wput(new URL("http://" + address + "/yacy/transferRWI.html"), 60000, null, null, final Vector v = httpc.wput(new URL("http://" + address + "/yacy/transferRWI.html"), 60000, null, null,
yacyCore.seedDB.sb.remoteProxyHost, yacyCore.seedDB.sb.remoteProxyPort, post); yacyCore.seedDB.sb.remoteProxyHost, yacyCore.seedDB.sb.remoteProxyPort, post);
// this should return a list of urlhashes that are unknwon // this should return a list of urlhashes that are unknwon
if (v != null) { if (v != null) {
yacyCore.seedDB.mySeed.incSI(indexcount); yacyCore.seedDB.mySeed.incSI(indexcount);
} }
HashMap result = nxTools.table(v); final HashMap result = nxTools.table(v);
return result; return result;
} catch (Exception e) { } catch (Exception e) {
yacyCore.log.logSevere("yacyClient.transferRWI error:" + e.getMessage()); yacyCore.log.logSevere("yacyClient.transferRWI error:" + e.getMessage());
return null; return null;
} }
} }
private static HashMap transferURL(yacySeed targetSeed, plasmaCrawlLURL.Entry[] urls) { private static HashMap transferURL(yacySeed targetSeed, plasmaCrawlLURL.Entry[] urls) {
// this post a message to the remote message board // this post a message to the remote message board
String address = targetSeed.getAddress(); final String address = targetSeed.getAddress();
if (address == null) return null; if (address == null) { return null; }
// prepare post values // prepare post values
serverObjects post = new serverObjects(); final serverObjects post = new serverObjects();
String key = crypt.randomSalt(); final String key = crypt.randomSalt();
post.put("key", key); post.put("key", key);
post.put("iam", yacyCore.seedDB.mySeed.hash); post.put("iam", yacyCore.seedDB.mySeed.hash);
post.put("youare", targetSeed.hash); post.put("youare", targetSeed.hash);
@ -651,7 +651,7 @@ public class yacyClient {
} }
post.put("urlc", Integer.toString(urlc)); post.put("urlc", Integer.toString(urlc));
try { try {
Vector v = httpc.wput(new URL("http://" + address + "/yacy/transferURL.html"), 60000, null, null, final Vector v = httpc.wput(new URL("http://" + address + "/yacy/transferURL.html"), 60000, null, null,
yacyCore.seedDB.sb.remoteProxyHost, yacyCore.seedDB.sb.remoteProxyPort, post); yacyCore.seedDB.sb.remoteProxyHost, yacyCore.seedDB.sb.remoteProxyPort, post);
if (v != null) { if (v != null) {
yacyCore.seedDB.mySeed.incSU(urlc); yacyCore.seedDB.mySeed.incSU(urlc);
@ -662,16 +662,16 @@ public class yacyClient {
return null; return null;
} }
} }
public static HashMap getProfile(yacySeed targetSeed) { public static HashMap getProfile(yacySeed targetSeed) {
// this post a message to the remote message board // this post a message to the remote message board
serverObjects post = new serverObjects(); final serverObjects post = new serverObjects();
post.put("iam", yacyCore.seedDB.mySeed.hash); post.put("iam", yacyCore.seedDB.mySeed.hash);
post.put("youare", targetSeed.hash); post.put("youare", targetSeed.hash);
String address = targetSeed.getAddress(); String address = targetSeed.getAddress();
if (address == null) address = "localhost:8080"; if (address == null) { address = "localhost:8080"; }
try { try {
Vector v = httpc.wput(new URL("http://" + address + "/yacy/profile.html"), 20000, null, null, final Vector v = httpc.wput(new URL("http://" + address + "/yacy/profile.html"), 20000, null, null,
yacyCore.seedDB.sb.remoteProxyHost, yacyCore.seedDB.sb.remoteProxyPort, post); yacyCore.seedDB.sb.remoteProxyHost, yacyCore.seedDB.sb.remoteProxyPort, post);
return nxTools.table(v); return nxTools.table(v);
} catch (Exception e) { } catch (Exception e) {
@ -679,18 +679,18 @@ public class yacyClient {
return null; return null;
} }
} }
public static void main(String[] args) { public static void main(String[] args) {
System.out.println("yacyClient Test"); System.out.println("yacyClient Test");
try { try {
plasmaSwitchboard sb = new plasmaSwitchboard(args[0], "httpProxy.init", "DATA/SETTINGS/httpProxy.conf"); final plasmaSwitchboard sb = new plasmaSwitchboard(args[0], "httpProxy.init", "DATA/SETTINGS/httpProxy.conf");
yacyCore core = new yacyCore(sb); final yacyCore core = new yacyCore(sb);
core.peerActions.loadSeedLists(); core.peerActions.loadSeedLists();
yacySeed target = core.seedDB.getConnected(args[1]); final yacySeed target = core.seedDB.getConnected(args[1]);
String wordhashe = plasmaWordIndexEntry.word2hash("test"); final String wordhashe = plasmaWordIndexEntry.word2hash("test");
//System.out.println("permission=" + permissionMessage(args[1])); //System.out.println("permission=" + permissionMessage(args[1]));
HashMap result = nxTools.table(httpc.wget( final HashMap result = nxTools.table(httpc.wget(
new URL("http://" + target.getAddress() + new URL("http://" + target.getAddress() +
"/yacy/search.html?myseed=" + core.seedDB.mySeed.genSeedStr(null) + "/yacy/search.html?myseed=" + core.seedDB.mySeed.genSeedStr(null) +
"&youare=" + target.hash + "&key=" + "&youare=" + target.hash + "&key=" +
@ -704,4 +704,5 @@ public class yacyClient {
} }
System.exit(0); System.exit(0);
} }
} }

368
source/de/anomic/yacy/yacySeed.java
Unescape View File

@ -3,7 +3,10 @@
// (C) by Michael Peter Christen; mc@anomic.de // (C) by Michael Peter Christen; mc@anomic.de
// first published on http://www.anomic.de // first published on http://www.anomic.de
// Frankfurt, Germany, 2004 // Frankfurt, Germany, 2004
// last major change: 11.07.2004 //
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
// //
// This program is free software; you can redistribute it and/or modify // This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by // it under the terms of the GNU General Public License as published by
@ -56,7 +59,6 @@
*/ */
package de.anomic.yacy; package de.anomic.yacy;
import java.io.File; import java.io.File;
@ -67,8 +69,6 @@ import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Properties; import java.util.Properties;
import java.util.StringTokenizer;
import de.anomic.net.natLib; import de.anomic.net.natLib;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCodings; import de.anomic.server.serverCodings;
@ -77,13 +77,18 @@ import de.anomic.tools.bitfield;
import de.anomic.tools.crypt; import de.anomic.tools.crypt;
public class yacySeed { public class yacySeed {
public static final String INDEX_OUT = "sI";
public static final String INDEX_IN = "rI";
public static final String URL_OUT = "sU";
public static final String URL_IN = "rU";
public static final String PEERTYPE_VIRGIN = "virgin"; public static final String PEERTYPE_VIRGIN = "virgin";
public static final String PEERTYPE_JUNIOR = "junior"; public static final String PEERTYPE_JUNIOR = "junior";
public static final String PEERTYPE_SENIOR = "senior"; public static final String PEERTYPE_SENIOR = "senior";
public static final String PEERTYPE_PRINCIPAL = "principal"; public static final String PEERTYPE_PRINCIPAL = "principal";
public static final String PEERTYPE = "PeerType"; public static final String PEERTYPE = "PeerType";
// class variables // class variables
public String hash; public String hash;
private Map dna; private Map dna;
@ -91,102 +96,104 @@ public class yacySeed {
public int selectscore = -1; // only for debugging public int selectscore = -1; // only for debugging
public yacySeed(String hash, Map dna) { public yacySeed(String hash, Map dna) {
// create a seed with a pre-defined hash map // create a seed with a pre-defined hash map
this.hash = hash; this.hash = hash;
this.dna = dna; this.dna = dna;
this.available = 0; this.available = 0;
} }
public yacySeed(String hash) { public yacySeed(String hash) {
dna = new HashMap(); dna = new HashMap();
// settings that can only be computed by originating peer: // settings that can only be computed by originating peer:
// at first startup - // at first startup -
this.hash = hash; // the hash key of the peer - very important. should be static somehow, even after restart this.hash = hash; // the hash key of the peer - very important. should be static somehow, even after restart
dna.put("Name", "&empty;"); // the name that the peer has given itself dna.put("Name", "&empty;"); // the name that the peer has given itself
dna.put("BDate", "&empty;"); // birthdate - first startup dna.put("BDate", "&empty;"); // birthdate - first startup
// later during operation - // later during operation -
dna.put("ISpeed", "0"); // the speed of indexing (pages/minute) of the peer dna.put("ISpeed", "0"); // the speed of indexing (pages/minute) of the peer
dna.put("Uptime", "0"); // the number of minutes that the peer is up in minutes/day (moving average MA30) dna.put("Uptime", "0"); // the number of minutes that the peer is up in minutes/day (moving average MA30)
dna.put("LCount", "0"); // the number of links that the peer has stored (LURL's) dna.put("LCount", "0"); // the number of links that the peer has stored (LURL's)
dna.put("NCount", "0"); // the number of links that the peer has noticed, but not loaded (NURL's) dna.put("NCount", "0"); // the number of links that the peer has noticed, but not loaded (NURL's)
dna.put("ICount", "0"); // the number of words that the peer has indexed (as it says) dna.put("ICount", "0"); // the number of words that the peer has indexed (as it says)
dna.put("SCount", "0"); // the number of seeds that the peer has stored dna.put("SCount", "0"); // the number of seeds that the peer has stored
dna.put("CCount", "0"); // the number of clients that the peer connects (as connects/hour) dna.put("CCount", "0"); // the number of clients that the peer connects (as connects/hour)
dna.put("Version", "0"); // the applications version dna.put("Version", "0"); // the applications version
// settings that is created during the 'hello' phase - in first contact // settings that is created during the 'hello' phase - in first contact
dna.put("IP", ""); // 123.234.345.456 dna.put("IP", ""); // 123.234.345.456
dna.put("Port", "&empty;"); // dna.put("Port", "&empty;"); //
dna.put("PeerType", "virgin"); // virgin/junior/senior/principal dna.put(PEERTYPE, PEERTYPE_VIRGIN); // virgin/junior/senior/principal
dna.put("IPType", "&empty;"); // static/dynamic (if the ip changes often for any reason) dna.put("IPType", "&empty;"); // static/dynamic (if the ip changes often for any reason)
// settings that can only be computed by visiting peer // settings that can only be computed by visiting peer
dna.put("LastSeen", yacyCore.universalDateShortString()); // for last-seen date dna.put("LastSeen", yacyCore.universalDateShortString()); // for last-seen date
dna.put("USpeed", "0"); // the computated uplink speed of the peer dna.put("USpeed", "0"); // the computated uplink speed of the peer
// settings that are needed to organize the seed round-trip // settings that are needed to organize the seed round-trip
dna.put("Flags", "0000"); dna.put("Flags", "0000");
setFlagDirectConnect(false); setFlagDirectConnect(false);
setFlagAcceptRemoteCrawl(true); setFlagAcceptRemoteCrawl(true);
setFlagAcceptRemoteIndex(true); setFlagAcceptRemoteIndex(true);
// index transfer // index transfer
dna.put("sI", "0"); // send index dna.put(INDEX_OUT, "0"); // send index
dna.put("rI", "0"); // received Index dna.put(INDEX_IN, "0"); // received Index
dna.put("sU", "0"); // send url dna.put(URL_OUT, "0"); // send url
dna.put("rU", "0"); // received URL dna.put(URL_IN, "0"); // received URL
available = 0; available = 0;
} }
public String get(String key, String dflt) { public String get(String key, String dflt) {
Object o = dna.get(key); Object o = dna.get(key);
if (o == null) return dflt; else return (String) o; if (o == null) return dflt; else return (String) o;
} }
public void put(String key, String value) { public void put(String key, String value) {
dna.put(key, value); dna.put(key, value);
} }
public Map getMap() { public Map getMap() {
return dna; return dna;
} }
public String getName() { public String getName() {
return get("Name", "&empty;"); return get("Name", "&empty;");
} }
public String getHexHash() { public String getHexHash() {
return b64Hash2hexHash(hash); return b64Hash2hexHash(hash);
} }
public void incSI(int count) { public void incSI(int count) {
String v = (String) dna.get("sI"); if (v == null) v = "0"; String v = (String) dna.get(INDEX_OUT); if (v == null) v = "0";
dna.put("sI", Integer.toString(Integer.parseInt(v) + count)); dna.put(INDEX_OUT, Integer.toString(Integer.parseInt(v) + count));
} }
public void incRI(int count) { public void incRI(int count) {
String v = (String) dna.get("rI"); if (v == null) v = "0"; String v = (String) dna.get(INDEX_IN); if (v == null) v = "0";
dna.put("rI", Integer.toString(Integer.parseInt(v) + count)); dna.put(INDEX_IN, Integer.toString(Integer.parseInt(v) + count));
} }
public void incSU(int count) { public void incSU(int count) {
String v = (String) dna.get("sU"); if (v == null) v = "0"; String v = (String) dna.get(URL_OUT); if (v == null) v = "0";
dna.put("sU", Integer.toString(Integer.parseInt(v) + count)); dna.put(URL_OUT, Integer.toString(Integer.parseInt(v) + count));
} }
public void incRU(int count) { public void incRU(int count) {
String v = (String) dna.get("rU"); if (v == null) v = "0"; String v = (String) dna.get(URL_IN); if (v == null) v = "0";
dna.put("rU", Integer.toString(Integer.parseInt(v) + count)); dna.put(URL_IN, Integer.toString(Integer.parseInt(v) + count));
} }
// 12 * 6 bit = 72 bit = 9 byte // 12 * 6 bit = 72 bit = 9 byte
public static String hexHash2b64Hash(String hexHash) { public static String hexHash2b64Hash(String hexHash) {
return serverCodings.enhancedCoder.encodeBase64(serverCodings.decodeHex(hexHash)); return serverCodings.enhancedCoder.encodeBase64(serverCodings.decodeHex(hexHash));
} }
public static String b64Hash2hexHash(String b64Hash) { public static String b64Hash2hexHash(String b64Hash) {
// the hash string represents 12 * 6 bit = 72 bits. This is too much for a long integer. // the hash string represents 12 * 6 bit = 72 bits. This is too much for a long integer.
return serverCodings.encodeHex(serverCodings.enhancedCoder.decodeBase64(b64Hash)); return serverCodings.encodeHex(serverCodings.enhancedCoder.decodeBase64(b64Hash));
} }
public float getVersion() { public float getVersion() {
@ -196,11 +203,11 @@ public class yacySeed {
return 0; return 0;
} }
} }
public String getAddress() { public String getAddress() {
String ip = (String) dna.get("IP"); String ip = (String) dna.get("IP");
String port = (String) dna.get("Port"); String port = (String) dna.get("Port");
if ((ip != null) && (ip.length() >= 8) && (port != null) && (port.length() >= 2)) return ip + ":" + port; else return null; if (ip != null && ip.length() >= 8 && port != null && port.length() >= 2) return ip + ":" + port; else return null;
} }
public int getPPM() { public int getPPM() {
@ -210,18 +217,19 @@ public class yacySeed {
return 0; return 0;
} }
} }
private boolean getFlag(int flag) { private boolean getFlag(int flag) {
String flags = get("Flags", "0000"); String flags = get("Flags", "0000");
return (new bitfield(flags.getBytes())).get(flag); return (new bitfield(flags.getBytes())).get(flag);
} }
private void setFlag(int flag, boolean value) { private void setFlag(int flag, boolean value) {
String flags = get("Flags", "0000"); String flags = get("Flags", "0000");
bitfield f = new bitfield(flags.getBytes()); bitfield f = new bitfield(flags.getBytes());
f.set(flag, value); f.set(flag, value);
put("Flags", f.toString()); put("Flags", f.toString());
} }
public void setFlagDirectConnect(boolean value) {setFlag(0, value);} public void setFlagDirectConnect(boolean value) {setFlag(0, value);}
public void setFlagAcceptRemoteCrawl(boolean value) {setFlag(1, value);} public void setFlagAcceptRemoteCrawl(boolean value) {setFlag(1, value);}
public void setFlagAcceptRemoteIndex(boolean value) {setFlag(2, value);} public void setFlagAcceptRemoteIndex(boolean value) {setFlag(2, value);}
@ -235,49 +243,49 @@ public class yacySeed {
//if (getVersion() < 0.335) return false; //if (getVersion() < 0.335) return false;
return getFlag(2); return getFlag(2);
} }
public boolean isVirgin() { public boolean isVirgin() {
return get("PeerType", "").equals("virgin"); return get(PEERTYPE, "").equals(PEERTYPE_VIRGIN);
} }
public boolean isJunior() { public boolean isJunior() {
return get("PeerType", "").equals("junior"); return get(PEERTYPE, "").equals(PEERTYPE_JUNIOR);
} }
public boolean isSenior() { public boolean isSenior() {
return get("PeerType", "").equals("senior"); return get(PEERTYPE, "").equals(PEERTYPE_SENIOR);
} }
public boolean isPrincipal() { public boolean isPrincipal() {
return get("PeerType", "").equals("principal"); return get(PEERTYPE, "").equals(PEERTYPE_PRINCIPAL);
} }
public boolean isOnline() { public boolean isOnline() {
return (isSenior()) || (isPrincipal()); return (isSenior() || isPrincipal());
} }
public String encodeLex(long c, int length) { public String encodeLex(long c, int length) {
if (length < 0) length = 0; if (length < 0) length = 0;
String s = ""; String s = "";
if (c == 0) s = '-' + s; if (c == 0) s = '-' + s;
else while (c > 0) { else while (c > 0) {
s = ((char) (32 + (c % 96))) + s; s = ((char) (32 + (c % 96))) + s;
c = c / 96; c = c / 96;
} }
if ((length != 0) && (s.length() > length)) if (length != 0 && s.length() > length)
throw new RuntimeException("encodeLex result '" + s + "' exceeds demanded length of " + length + " digits"); throw new RuntimeException("encodeLex result '" + s + "' exceeds demanded length of " + length + " digits");
if (length == 0) length = 1; // rare exception for the case that c == 0 if (length == 0) length = 1; // rare exception for the case that c == 0
while (s.length() < length) s = '-' + s; while (s.length() < length) s = '-' + s;
return s; return s;
} }
public long decodeLex(String s) { public long decodeLex(String s) {
long c = 0; long c = 0;
for (int i = 0; i < s.length(); i++) c = c * 96 + (byte) s.charAt(i) - 32; for (int i = 0; i < s.length(); i++) c = c * 96 + (byte) s.charAt(i) - 32;
return c; return c;
} }
private static long maxLex(int len) { private static long maxLex(int len) {
// computes the maximum number that can be coded with a lex-encoded String of length len // computes the maximum number that can be coded with a lex-encoded String of length len
long c = 0; long c = 0;
for (int i = 0; i < len; i++) c = c * 96 + 95; for (int i = 0; i < len; i++) c = c * 96 + 95;
return c; return c;
} }
public static final long maxDHTDistance = maxLex(9); public static final long maxDHTDistance = maxLex(9);
@ -292,73 +300,73 @@ public class yacySeed {
} }
public static yacySeed genLocalSeed(plasmaSwitchboard sb) { public static yacySeed genLocalSeed(plasmaSwitchboard sb) {
// genera a seed for the local peer // genera a seed for the local peer
// this is the birthplace of a seed, that then will start to travel to other peers // this is the birthplace of a seed, that then will start to travel to other peers
// at first we need a good peer hash // at first we need a good peer hash
// that hash should be as static as possible, so that it depends mainly on system // that hash should be as static as possible, so that it depends mainly on system
// variables and can even then be reconstructed if the local seed has disappeared // variables and can even then be reconstructed if the local seed has disappeared
Properties sp = System.getProperties(); Properties sp = System.getProperties();
String slow = String slow =
sp.getProperty("file.encoding","") + sp.getProperty("file.encoding", "") +
sp.getProperty("file.separator","") + sp.getProperty("file.separator", "") +
sp.getProperty("java.class.path","") + sp.getProperty("java.class.path", "") +
sp.getProperty("java.vendor","") + sp.getProperty("java.vendor", "") +
sp.getProperty("os.arch","") + sp.getProperty("os.arch", "") +
sp.getProperty("os.name","") + sp.getProperty("os.name", "") +
sp.getProperty("path.separator","") + sp.getProperty("path.separator", "") +
sp.getProperty("user.dir","") + sp.getProperty("user.dir", "") +
sp.getProperty("user.home","") + sp.getProperty("user.home", "") +
sp.getProperty("user.language","") + sp.getProperty("user.language", "") +
sp.getProperty("user.name","") + sp.getProperty("user.name", "") +
sp.getProperty("user.timezone", ""); sp.getProperty("user.timezone", "");
String medium = String medium =
sp.getProperty("java.class.version","") + sp.getProperty("java.class.version", "") +
sp.getProperty("java.version","") + sp.getProperty("java.version", "") +
sp.getProperty("os.version","") + sp.getProperty("os.version", "") +
sb.getConfig("peerName", "noname"); sb.getConfig("peerName", "noname");
String fast = Long.toString(System.currentTimeMillis()); String fast = Long.toString(System.currentTimeMillis());
// the resultinh hash does not have any information than can be used to reconstruct the // the resultinh hash does not have any information than can be used to reconstruct the
// original system information that has been collected here to create the hash // original system information that has been collected here to create the hash
// We simply distinuguish three parts of the hash: slow, medium and fast changing character of system idenfification // We simply distinuguish three parts of the hash: slow, medium and fast changing character of system idenfification
// the Hash is constructed in such a way, that the slow part influences the main aerea of the distributed hash location // the Hash is constructed in such a way, that the slow part influences the main aerea of the distributed hash location
// more than the fast part. The effect is, that if the peer looses it's seed information and is reconstructed, it // more than the fast part. The effect is, that if the peer looses it's seed information and is reconstructed, it
// still hosts most information of the distributed hash the an appropriate 'position' // still hosts most information of the distributed hash the an appropriate 'position'
String hash = String hash =
serverCodings.encodeMD5B64(slow, true).substring(0, 4) + serverCodings.encodeMD5B64(slow, true).substring(0, 4) +
serverCodings.encodeMD5B64(medium, true).substring(0, 4) + serverCodings.encodeMD5B64(medium, true).substring(0, 4) +
serverCodings.encodeMD5B64(fast, true).substring(0, 4); serverCodings.encodeMD5B64(fast, true).substring(0, 4);
yacyCore.log.logInfo("init: OWN SEED = " + hash); yacyCore.log.logInfo("init: OWN SEED = " + hash);
if (hash.length() != yacySeedDB.commonHashLength) { if (hash.length() != yacySeedDB.commonHashLength) {
yacyCore.log.logSevere("YACY Internal error: distributed hash conceptual error"); yacyCore.log.logSevere("YACY Internal error: distributed hash conceptual error");
System.exit(-1); System.exit(-1);
} }
yacySeed newSeed = new yacySeed(hash); yacySeed newSeed = new yacySeed(hash);
// now calculate other information about the host // now calculate other information about the host
newSeed.dna.put("Name", sb.getConfig("peerName", "unnamed")); newSeed.dna.put("Name", sb.getConfig("peerName", "unnamed"));
if ((serverCore.portForwardingEnabled) && (serverCore.portForwarding != null)) { if ((serverCore.portForwardingEnabled) && (serverCore.portForwarding != null)) {
newSeed.dna.put("Port",Integer.toString(serverCore.portForwarding.getPort())); newSeed.dna.put("Port",Integer.toString(serverCore.portForwarding.getPort()));
} else { } else {
newSeed.dna.put("Port", sb.getConfig("port", "8080")); newSeed.dna.put("Port", sb.getConfig("port", "8080"));
} }
newSeed.dna.put("BDate", yacyCore.universalDateShortString()); newSeed.dna.put("BDate", yacyCore.universalDateShortString());
newSeed.dna.put("LastSeen", newSeed.dna.get("BDate")); // just as initial setting newSeed.dna.put("LastSeen", newSeed.dna.get("BDate")); // just as initial setting
newSeed.dna.put("PeerType", "virgin"); newSeed.dna.put(PEERTYPE, PEERTYPE_VIRGIN);
return newSeed; return newSeed;
} }
public static yacySeed genRemoteSeed(String seedStr, String key, Date remoteTime) { public static yacySeed genRemoteSeed(String seedStr, String key, Date remoteTime) {
// this method is used to convert the external representation of a seed into a seed object // this method is used to convert the external representation of a seed into a seed object
if (seedStr == null) return null; if (seedStr == null) return null;
String seed = crypt.simpleDecode(seedStr, key); String seed = crypt.simpleDecode(seedStr, key);
if (seed == null) return null; if (seed == null) return null;
HashMap dna = serverCodings.string2map(seed); HashMap dna = serverCodings.string2map(seed);
String hash = (String) dna.remove("Hash"); String hash = (String) dna.remove("Hash");
return new yacySeed(hash, dna); return new yacySeed(hash, dna);
} }
public String toString() { public String toString() {
@ -376,62 +384,62 @@ public class yacySeed {
} }
public String genSeedStr(String key) { public String genSeedStr(String key) {
// use a default encoding // use a default encoding
return genSeedStr('b', key); return genSeedStr('b', key);
} }
public String genSeedStr(char method, String key) { public String genSeedStr(char method, String key) {
return crypt.simpleEncode(toString(), key, method); return crypt.simpleEncode(toString(), key, method);
} }
public String isProper() { public String isProper() {
// checks if everything is ok with that seed // checks if everything is ok with that seed
if (this.hash == null) return "hash is null"; if (this.hash == null) return "hash is null";
if (this.hash.length() != yacySeedDB.commonHashLength) return "wrong hash length (" + this.hash.length() + ")"; if (this.hash.length() != yacySeedDB.commonHashLength) return "wrong hash length (" + this.hash.length() + ")";
String ip = (String) dna.get("IP"); String ip = (String) dna.get("IP");
if (ip == null) return "IP is null"; if (ip == null) return "IP is null";
if (ip.length() < 8) return "IP is too short: " + ip; if (ip.length() < 8) return "IP is too short: " + ip;
if (!(natLib.isProper(ip))) return "IP is not proper: " + ip; if (!(natLib.isProper(ip))) return "IP is not proper: " + ip;
return null; return null;
} }
public void save(File f) throws IOException { public void save(File f) throws IOException {
String out = genSeedStr('p', null); String out = genSeedStr('p', null);
FileWriter fw = new FileWriter(f); FileWriter fw = new FileWriter(f);
fw.write(out, 0, out.length()); fw.write(out, 0, out.length());
fw.close(); fw.close();
} }
public static yacySeed load(File f) throws IOException { public static yacySeed load(File f) throws IOException {
FileReader fr = new FileReader(f); FileReader fr = new FileReader(f);
char[] b = new char[(int) f.length()]; char[] b = new char[(int) f.length()];
fr.read(b, 0, b.length); fr.read(b, 0, b.length);
fr.close(); fr.close();
return genRemoteSeed(new String(b), null, new Date()); return genRemoteSeed(new String(b), null, new Date());
} }
public Object clone() { public Object clone() {
return new yacySeed(this.hash, (HashMap) (new HashMap(dna)).clone()); return new yacySeed(this.hash, (HashMap) (new HashMap(dna)).clone());
} }
/* /*
public static void main(String[] argv) { public static void main(String[] argv) {
try { try {
plasmaSwitchboard sb = new plasmaSwitchboard("../httpProxy.init", "../httpProxy.conf"); plasmaSwitchboard sb = new plasmaSwitchboard("../httpProxy.init", "../httpProxy.conf");
yacySeed ys = genLocalSeed(sb); yacySeed ys = genLocalSeed(sb);
String yp, yz, yc; String yp, yz, yc;
System.out.println("YACY String = " + ys.toString()); System.out.println("YACY String = " + ys.toString());
System.out.println("YACY SeedStr/p = " + (yp = ys.genSeedStr('p', null))); System.out.println("YACY SeedStr/p = " + (yp = ys.genSeedStr('p', null)));
//System.out.println("YACY SeedStr/z = " + (yz = ys.genSeedStr('z', null))); //System.out.println("YACY SeedStr/z = " + (yz = ys.genSeedStr('z', null)));
System.out.println("YACY SeedStr/c = " + (yc = ys.genSeedStr('c', "abc"))); System.out.println("YACY SeedStr/c = " + (yc = ys.genSeedStr('c', "abc")));
System.out.println("YACY remote/p = " + genRemoteSeed(yp, null).toString()); System.out.println("YACY remote/p = " + genRemoteSeed(yp, null).toString());
//System.out.println("YACY remote/z = " + genRemoteSeed(yz, null).toString()); //System.out.println("YACY remote/z = " + genRemoteSeed(yz, null).toString());
System.out.println("YACY remote/c = " + genRemoteSeed(yc, "abc").toString()); System.out.println("YACY remote/c = " + genRemoteSeed(yc, "abc").toString());
System.exit(0); System.exit(0);
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} }
} }
*/ */

Write Preview
Loading…
Cancel
Save