introduced citation-rank transmission protocol and activate transport for anonymisation

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1055 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 20 years ago
parent 9a441e8e77
commit 79818a320f

@ -3,7 +3,7 @@ javacSource=1.4
javacTarget=1.4
# Release Configuration
releaseVersion=0.413
releaseVersion=0.414
releaseFile=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
#releaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
releaseDir=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}

@ -106,7 +106,7 @@ public class CacheAdmin_p {
final TreeSet fList = new TreeSet();
File object;
int size = list.length - 1;
for (int i = size; i >= 0 ; i--) { // Rückwärts ist schneller
for (int i = size; i >= 0 ; i--) { // Rueckwaerts ist schneller
object = new File(dir, list[i]);
if (!object.getName().equalsIgnoreCase("responseHeader.db")) {
if (object.isDirectory()) {

@ -47,6 +47,7 @@
<td class="small"><b>Uptime</b><br>&nbsp;&nbsp;<a href="/Network.html?page=#[page]#&sort=Uptime&order=up">&lt;</a>&nbsp;<a href="/Network.html?page=#[page]#&sort=Uptime&order=down">&gt;</a></td>
<td class="small"><b>#Links</b><br>&nbsp;&nbsp;<a href="/Network.html?page=#[page]#&sort=LCount&order=up">&lt;</a>&nbsp;<a href="/Network.html?page=#[page]#&sort=LCount&order=down">&gt;</a></td>
<td class="small"><b>#RWIs</b><br>&nbsp;&nbsp;<a href="/Network.html?page=#[page]#&sort=ICount&order=up">&lt;</a>&nbsp;<a href="/Network.html?page=#[page]#&sort=ICount&order=down">&gt;</a></td>
<td class="small"><b>#CR-<br>Files</b><br>Own/Other</td>
<td class="small"><b>Sent<br>Words</b><br>&nbsp;</td>
<td class="small"><b>Sent<br>URLs</b><br>&nbsp;</td>
<td class="small"><b>Received<br>Words</b><br>&nbsp;</td>
@ -68,13 +69,14 @@
</td>
<td class="small"><a href="http://www.#[fullname]#.yacy" class="small">#[shortname]#</a></td>
<td class="small">
#(type)##(direct)#<img border="0" src="/env/grafics/JuniorPassive.gif" title="Type: Junior | Contact: passive"></img>::<img border="0" src="/env/grafics/JuniorDirect.gif" title="Type: Junior | Contact: direct"></img>::<img border="0" src="/env/grafics/JuniorOffline.gif" title="Type: Junior | Contact: offline"></img>#(/direct)#::#(direct)#<img border="0" src="/env/grafics/SeniorPassive.gif" title="Type: Senior | Contact: passive"></img>::<img border="0" src="/env/grafics/SeniorDirect.gif" title="Type: Senior | Contact: direct"></img>::<img border="0" src="/env/grafics/SeniorOffline.gif" title="Type: Senior | Contact: offline"></img>#(/direct)#::<a href="#[url]#">#(direct)#<img border="0" src="/env/grafics/PrincipalPassive.gif" title="Type: Principal | Contact: passive | Seed download: possible"></img>::<img border="0" src="/env/grafics/PrincipalDirect.gif" title="Type: Principal | Contact: direct | Seed download: possible"></img>::<img border="0" src="/env/grafics/PrincipalOffline.gif" title="Type: Principal | Contact: offline | Seed download: ?"></img>#(/direct)#</a>#(/type)##(acceptcrawl)#<img border="0" src="/env/grafics/CrawlNo.gif" title="Accept Crawl: no"></img>::<img border="0" src="/env/grafics/CrawlYes.gif" title="Accept Crawl: yes"></img>::<img border="0" src="/env/grafics/CrawlYesOffline.gif" title="Accept Crawl: yes"></img>#(/acceptcrawl)##(dhtreceive)#<img border="0" src="/env/grafics/DHTReceiveNo.gif" title="DHT Receive: no"></img>::<img border="0" src="/env/grafics/DHTReceiveYes.gif" title="DHT Receive: yes"></img>::<img border="0" src="/env/grafics/DHTReceiveYesOffline.gif" title="DHT Receive: yes"></img>#(/dhtreceive)#</td>
#(type)##(direct)#<img border="0" src="/env/grafics/JuniorPassive.gif" title="Type: Junior | Contact: passive"></img>::<img border="0" src="/env/grafics/JuniorDirect.gif" title="Type: Junior | Contact: direct"></img>::<img border="0" src="/env/grafics/JuniorOffline.gif" title="Type: Junior | Contact: offline"></img>#(/direct)#::#(direct)#<img border="0" src="/env/grafics/SeniorPassive.gif" title="Type: Senior | Contact: passive"></img>::<img border="0" src="/env/grafics/SeniorDirect.gif" title="Type: Senior | Contact: direct"></img>::<img border="0" src="/env/grafics/SeniorOffline.gif" title="Type: Senior | Contact: offline"></img>#(/direct)#::<a href="#[url]#">#(direct)#<img border="0" src="/env/grafics/PrincipalPassive.gif" title="Type: Principal | Contact: passive | Seed download: possible"></img>::<img border="0" src="/env/grafics/PrincipalDirect.gif" title="Type: Principal | Contact: direct | Seed download: possible"></img>::<img border="0" src="/env/grafics/PrincipalOffline.gif" title="Type: Principal | Contact: offline | Seed download: ?"></img>#(/direct)#</a>#(/type)##(acceptcrawl)#<img border="0" src="/env/grafics/CrawlNo.gif" title="Accept Crawl: no"></img>::<img border="0" src="/env/grafics/CrawlYes.gif" title="Accept Crawl: yes"></img>::<img border="0" src="/env/grafics/CrawlYesOffline.gif" title="Accept Crawl: yes"></img>#(/acceptcrawl)##(dhtreceive)#<img border="0" src="/env/grafics/DHTReceiveNo.gif" title="DHT Receive: no"></img>::<img border="0" src="/env/grafics/DHTReceiveYes.gif" title="DHT Receive: yes"></img>::<img border="0" src="/env/grafics/DHTReceiveYesOffline.gif" title="DHT Receive: yes"></img>#(/dhtreceive)##(rankingreceive)#<img border="0" src="/env/grafics/RankingReceiveNo.gif" title="Ranking Receive: no"></img>::<img border="0" src="/env/grafics/RankingReceiveYes.gif" title="Ranking Receive: yes"></img>::<img border="0" src="/env/grafics/RankingCollects.gif" title="Ranking Receive: collection"></img>#(/rankingreceive)#</td>
<td class="small" align="right">#[version]#</td>
<td class="small" align="right">#[lastSeen]#</td>
<td class="small" align="right">#[utc]#</td>
<td class="small" align="right"><NOBR>#[uptime]#</NOBR></td>
<td class="small" align="right">#[links]#</td>
<td class="small" align="right">#[words]#</td>
<td class="small" align="center">#[CRWCnt]#/#[CRTCnt]#</td>
<td class="small" align="right">#[sI]#</td>
<td class="small" align="right">#[sU]#</td>
<td class="small" align="right">#[rI]#</td>
@ -210,4 +212,4 @@ To do this, press this button:
#[footer]#
</body>
</html>
</html>

@ -59,6 +59,7 @@ import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyNewsRecord;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyVersion;
public class Network {
@ -338,6 +339,7 @@ public class Network {
if (page == 1) {
prop.put(STR_TABLE_LIST + conCount + "_acceptcrawl", seed.getFlagAcceptRemoteCrawl() ? 1 : 0); // green=on or red=off
prop.put(STR_TABLE_LIST + conCount + "_dhtreceive", seed.getFlagAcceptRemoteIndex() ? 1 : 0); // green=on or red=off
prop.put(STR_TABLE_LIST + conCount + "_rankingreceive", (seed.getVersion() >= yacyVersion.YACY_ACCEPTS_RANKING_TRANSMISSION) ? 1 : 0);
} else { // Passive, Potential Peers
if (seed.getFlagAcceptRemoteCrawl()) {
prop.put(STR_TABLE_LIST + conCount + "_acceptcrawl", 2); // red/green: offline, was on
@ -349,6 +351,11 @@ public class Network {
} else {
prop.put(STR_TABLE_LIST + conCount + "_dhtreceive", 0); // red/red; offline was off
}
if (seed.getVersion() >= yacyVersion.YACY_ACCEPTS_RANKING_TRANSMISSION) {
prop.put(STR_TABLE_LIST + conCount + "_rankingreceive", 1);
} else {
prop.put(STR_TABLE_LIST + conCount + "_rankingreceive", 0);
}
}
prop.put(STR_TABLE_LIST + conCount + "_version", yacy.combinedVersionString2PrettyString(seed.get(yacySeed.VERSION, "0.1")));
prop.put(STR_TABLE_LIST + conCount + "_lastSeen", lastseen);
@ -356,6 +363,8 @@ public class Network {
prop.put(STR_TABLE_LIST + conCount + "_uptime", serverDate.intervalToString(60000 * Long.parseLong(seed.get(yacySeed.UPTIME, "0"))));
prop.put(STR_TABLE_LIST + conCount + "_links", groupDigits(seed.get(yacySeed.LCOUNT, "0")));
prop.put(STR_TABLE_LIST + conCount + "_words", groupDigits(seed.get(yacySeed.ICOUNT, "0")));
prop.put(STR_TABLE_LIST + conCount + "_CRWCnt", seed.get(yacySeed.CRWCNT, "0"));
prop.put(STR_TABLE_LIST + conCount + "_CRTCnt", seed.get(yacySeed.CRTCNT, "0"));
prop.put(STR_TABLE_LIST + conCount + "_sI", groupDigits(seed.get(yacySeed.INDEX_OUT, "0")));
prop.put(STR_TABLE_LIST + conCount + "_sU", groupDigits(seed.get(yacySeed.URL_OUT, "0")));
prop.put(STR_TABLE_LIST + conCount + "_rI", groupDigits(seed.get(yacySeed.INDEX_IN, "0")));

Before

Width:  |  Height:  |  Size: 151 B

After

Width:  |  Height:  |  Size: 151 B

Before

Width:  |  Height:  |  Size: 149 B

After

Width:  |  Height:  |  Size: 149 B

@ -168,7 +168,7 @@ public class dir {
if (pos >= 0) { filename = filename.substring(pos + 1); }
final File newfile = new File(dir, filename);
final File newfilemd5 = new File(dir, filename + ".md5");
final byte[] binary = (byte[]) post.get((Object) "file$file", (Object) new byte[0]);
final byte[] binary = (byte[]) post.get("file$file", (Object) new byte[0]);
try {
serverFileUtils.write(binary, newfile);
String md5s = serverCodings.encodeMD5Hex(newfile);

@ -258,12 +258,13 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
}
public static void main(String[] args) {
String test = "Nokia kürzt bei Forschung und Entwicklung";
/*
try {
htmlFilterContentScraper scraper = new htmlFilterContentScraper(new URL("http://localhost"));
scraper.scrapeText(test.getBytes());
System.out.println(new String(scraper.getText()));
} catch (MalformedURLException e) {}
*/
}
}

@ -1128,7 +1128,8 @@ do upload
boolean ssl,
httpRemoteProxyConfig theRemoteProxyConfig,
httpHeader requestHeader,
serverObjects props
serverObjects props,
HashMap files
) throws IOException {
if (requestHeader == null) requestHeader = new httpHeader();
@ -1143,7 +1144,7 @@ do upload
} else {
con = httpc.getInstance(host, port, timeout, ssl, theRemoteProxyConfig);
}
httpc.response res = con.POST(path, requestHeader, props, null);
httpc.response res = con.POST(path, requestHeader, props, files);
//System.out.println("response=" + res.toString());
if (res.status.startsWith("2")) {
@ -1164,7 +1165,8 @@ do upload
String user,
String password,
httpRemoteProxyConfig theRemoteProxyConfig,
serverObjects props
serverObjects props,
HashMap files
) throws IOException {
int port = u.getPort();
boolean ssl = u.getProtocol().equals("https");
@ -1182,7 +1184,8 @@ do upload
ssl,
theRemoteProxyConfig,
null,
props
props,
files
);
}
@ -1198,7 +1201,8 @@ do upload
null,
null,
null,
props
props,
null
);
} catch (MalformedURLException e) {
throw new IOException("Malformed URL: " + e.getMessage());
@ -1328,7 +1332,8 @@ do upload
String user,
String password,
httpRemoteProxyConfig theRemoteProxyConfig,
serverObjects props
serverObjects props,
HashMap files
) throws IOException {
// splitting of the byte array into lines
byte[] a = singlePOST(
@ -1337,7 +1342,8 @@ do upload
user,
password,
theRemoteProxyConfig,
props
props,
files
);
//System.out.println("wput-out=" + new String(a));

@ -203,13 +203,6 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
//htLocaleSelection = switchboard.getConfig("htLocaleSelection","default");
}
// initialise an message digest for Content-MD5 support ...
try {
this.md5Digest = MessageDigest.getInstance("MD5");

@ -125,7 +125,8 @@ public class natLib {
//TODO: This is not IPv6 compatible
public static boolean isProper(String ip) {
plasmaSwitchboard sb=plasmaSwitchboard.getSwitchboard();
String yacyDebugMode = sb.getConfig("yacyDebugMode", "false");
if (sb == null) return false;
String yacyDebugMode = sb.getConfig("yacyDebugMode", "false");
if(yacyDebugMode.equals("true")){
return true;
}

@ -380,7 +380,7 @@ public final class plasmaCrawlLURL extends plasmaURL {
url = urle.url().toString();
txt = urle.url().toString();
// Kosmetik, die wirklich benutzte URL behält die ':80'
// Kosmetik, die wirklich benutzte URL behaelt die ':80'
if (txt.endsWith(":80")) txt = txt.substring(0, txt.length() - 3);
if ((p = txt.indexOf(":80/")) != -1) {
txt = txt.substring(0, p).concat(txt.substring(p + 3)); // den '/' erstmal nicht abschneiden

@ -470,7 +470,7 @@ public final class plasmaHTCache {
s = s.substring(pos + c.length());
while (s.startsWith("/")) s = s.substring(1);
// dieser Block kann später gelöscht werden
// dieser Block kann spaeter geloescht werden
pos = s.indexOf("+");
if (pos >= 0) {
s = s.substring(0, pos) + ":" + s.substring(pos + 1);

@ -0,0 +1,133 @@
// plasmaRankingDistribution.java
// -------------------------------------------
// (C) by Michael Peter Christen; mc@anomic.de
// first published on http://www.anomic.de
// Frankfurt, Germany, 2005
// created 9.11.2005
//
// $LastChangedDate: 2005-11-04 14:41:51 +0100 (Fri, 04 Nov 2005) $
// $LastChangedRevision: 1026 $
// $LastChangedBy: borg-0300 $
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
// Using this software in any meaning (reading, learning, copying, compiling,
// running) means that you agree that the Author(s) is (are) not responsible
// for cost, loss of data or any harm that may be caused directly or indirectly
// by usage of this softare or this documentation. The usage of this software
// is on your own risk. The installation and usage (starting/running) of this
// software may allow other people or application to access your computer and
// any attached devices and is highly dependent on the configuration of the
// software which must be done by the user of the software; the author(s) is
// (are) also not responsible for proper configuration and usage of the
// software, even if provoked by documentation provided together with
// the software.
//
// Any changes to this file according to the GPL as documented in the file
// gpl.txt aside this file in the shipment you received can be done to the
// lines that follows this copyright notice here, but changes must not be
// done inside the copyright notive above. A re-distribution must contain
// the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such.
package de.anomic.plasma;
import java.io.IOException;
import java.io.File;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeedDB;
import de.anomic.yacy.yacyClient;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyVersion;
import de.anomic.server.logging.serverLog;
import de.anomic.server.serverFileUtils;
public final class plasmaRankingDistribution {
public static final String CR_OWN = "GLOBAL/010_owncr";
public static final String CR_OTHER = "GLOBAL/014_othercr/";
private final serverLog log;
private File sourcePath;
public plasmaRankingDistribution(serverLog log, File sourcePath) {
this.log = log;
this.sourcePath = sourcePath;
}
public int size() {
if ((sourcePath.exists()) && (sourcePath.isDirectory())) return sourcePath.list().length; else return 0;
}
public boolean performTransferRanking() {
if (yacyCore.seedDB == null) {
log.logFine("no ranking distribution: seedDB == null");
return false;
}
if (yacyCore.seedDB.mySeed == null) {
log.logFine("no ranking distribution: mySeed == null");
return false;
}
if (yacyCore.seedDB.mySeed.isVirgin()) {
log.logFine("no ranking distribution: status is virgin");
return false;
}
String[] outfiles = sourcePath.list();
if (outfiles == null) {
log.logFine("no ranking distribution: source path does not exist");
return false;
}
if (outfiles.length == 0) {
log.logFine("no ranking distribution: source path does not contain any file");
return false;
}
yacySeed target = yacyCore.seedDB.anySeedVersion(yacyVersion.YACY_ACCEPTS_RANKING_TRANSMISSION);
//if (target == null) target = yacyCore.seedDB.getConnected("nAEhLbmYNor"); // only for debugging
//if (target == null) target = yacyCore.seedDB.lookupByName("G5"); // only for debugging
if (target == null) {
log.logFine("no ranking distribution: no target available");
return false;
}
// do the transfer
long starttime = System.currentTimeMillis();
File crfile = new File(sourcePath, outfiles[0]);
String targetaddress = target.getAddress();
String result = "unknown";
try {
byte[] b = serverFileUtils.read(crfile);
result = yacyClient.transfer(targetaddress, crfile.getName(), b);
if (result == null) {
log.logInfo("RankingDistribution - transmitted file " + crfile + " to " + targetaddress + " successfully in " + ((System.currentTimeMillis() - starttime) / 1000) + " seconds");
crfile.delete(); // the file is not needed any more locally
} else {
log.logInfo("RankingDistribution - error transmitting file " + crfile + " to " + targetaddress + ": " + result);
}
} catch (IOException e) {
log.logInfo("RankingDistribution - could not read file " + crfile + ": " + e.getMessage());
result = "input file error: " + e.getMessage();
}
// show success
return result == null;
}
}

File diff suppressed because it is too large Load Diff

@ -41,6 +41,7 @@
package de.anomic.server;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
@ -254,6 +255,10 @@ public final class serverCodings {
return encodeHex(encodeMD5Raw(file));
}
public static String encodeMD5Hex(byte[] b) {
// generate a hex representation from the md5 of a byte-array
return encodeHex(encodeMD5Raw(b));
}
private static byte[] encodeMD5Raw(String key) {
try {
@ -288,6 +293,25 @@ public final class serverCodings {
return null;
}
private static byte[] encodeMD5Raw(byte[] b) {
try {
MessageDigest digest = MessageDigest.getInstance("MD5");
digest.reset();
InputStream in = new ByteArrayInputStream(b);
byte[] buf = new byte[2048];
int n;
while ((n = in.read(buf)) > 0) digest.update(buf, 0, n);
in.close();
// now compute the hex-representation of the md5 digest
return digest.digest();
} catch (java.security.NoSuchAlgorithmException e) {
System.out.println("Internal Error at md5:" + e.getMessage());
} catch (java.io.IOException e) {
System.out.println("byte[] error: " + e.getMessage());
}
return null;
}
public static Properties s2p(String s) {
Properties p = new Properties();
int pos;

@ -308,23 +308,24 @@ public final class serverCore extends serverAbstractThread implements serverThre
try {
// if a static IP was configured, we have to return it here ...
plasmaSwitchboard sb = plasmaSwitchboard.getSwitchboard();
if(sb != null){
String staticIP=sb.getConfig("staticIP", "");
if( (!staticIP.equals("")) && sb.getConfig("yacyDebugMode", "false").equals("true") ){
return staticIP;
}
}
plasmaSwitchboard sb = plasmaSwitchboard.getSwitchboard();
if(sb != null){
String staticIP=sb.getConfig("staticIP", "");
if( (!staticIP.equals("")) && sb.getConfig("yacyDebugMode", "false").equals("true") ){
return staticIP;
}
}
// If port forwarding was enabled we need to return the remote IP Address
// If port forwarding was enabled we need to return the remote IP Address
if ((serverCore.portForwardingEnabled)&&(serverCore.portForwarding != null)) {
//does not return serverCore.portForwarding.getHost(), because hostnames are not valid, except in DebugMode
//does not return serverCore.portForwarding.getHost(), because hostnames are not valid, except in DebugMode
return InetAddress.getByName(serverCore.portForwarding.getHost()).getHostAddress();
}
// otherwise we return the real IP address of this host
return publicLocalIP().getHostAddress();
InetAddress pLIP = publicLocalIP();
if (pLIP != null) return pLIP.getHostAddress();
return null;
} catch (java.net.UnknownHostException e) {
System.err.println("ERROR: (internal) " + e.getMessage());
return null;

@ -123,14 +123,14 @@ public final class serverObjects extends Hashtable implements Cloneable {
}
// new get with default objects
public Object get(Object key, Object dflt) {
public Object get(String key, Object dflt) {
Object result = super.get(key);
if (result == null) return dflt; else return result;
}
// string variant
public String get(String key, String dflt) {
return (String) this.get((Object) key, (Object) dflt);
return (String) this.get(key, (Object) dflt);
}
public int getInt(String key, int dflt) {

@ -61,6 +61,7 @@ import de.anomic.plasma.plasmaWordIndex;
import de.anomic.plasma.plasmaSearchProfile;
import de.anomic.server.serverCore;
import de.anomic.server.serverObjects;
import de.anomic.server.serverCodings;
import de.anomic.tools.crypt;
import de.anomic.tools.nxTools;
import de.anomic.yacy.yacySeed;
@ -125,7 +126,8 @@ public final class yacyClient {
null,
null,
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
obj
obj,
null
)
);
} catch (Exception e) {
@ -407,7 +409,8 @@ public final class yacyClient {
null,
null,
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
obj
obj,
null
)
);
@ -506,18 +509,8 @@ public final class yacyClient {
post.put("youare", targetHash);
post.put(yacySeed.MYTIME, yacyCore.universalDateShortString(new Date()));
// getting target address
String address;
if (targetHash.equals(yacyCore.seedDB.mySeed.hash)) {
address = yacyCore.seedDB.mySeed.getAddress();
//System.out.println("local address: " + address);
} else {
final yacySeed targetSeed = yacyCore.seedDB.getConnected(targetHash);
if (targetSeed == null) { return null; }
address = targetSeed.getAddress();
//System.out.println("remote address: " + address);
}
if (address == null) { address = "localhost:8080"; }
// get target address
String address = targetAddress(targetHash);
// sending request
try {
@ -528,7 +521,8 @@ public final class yacyClient {
null,
null,
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
post
post,
null
)
);
} catch (Exception e) {
@ -557,24 +551,112 @@ public final class yacyClient {
post.put(yacySeed.MYTIME, yacyCore.universalDateShortString(new Date()));
post.put("message", new String(message));
// getting target address
// get target address
String address = targetAddress(targetHash);
// sending request
try {
final ArrayList v = httpc.wput(
new URL("http://" + address + "/yacy/message.html"),
20000,
null,
null,
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
post,
null
);
return nxTools.table(v);
} catch (Exception e) {
yacyCore.log.logSevere("yacyClient.postMessage error:" + e.getMessage());
return null;
}
}
public static String targetAddress(String targetHash) {
// find target address
String address;
if (targetHash.equals(yacyCore.seedDB.mySeed.hash)) {
address = yacyCore.seedDB.mySeed.getAddress();
} else {
address = yacyCore.seedDB.getConnected(targetHash).getAddress();
final yacySeed targetSeed = yacyCore.seedDB.getConnected(targetHash);
if (targetSeed == null) { return null; }
address = targetSeed.getAddress();
}
if (address == null) { address = "localhost:8080"; }
if (address == null) address = "localhost:8080";
return address;
}
public static HashMap transferPermission(String targetAddress, long filesize, String filename) {
// should we use the proxy?
boolean useProxy = (yacyCore.seedDB != null) &&
(yacyCore.seedDB.sb.remoteProxyConfig != null) &&
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
// add all needed parameters
final String key = crypt.randomSalt();
final serverObjects post = new serverObjects();
post.put("key", key);
post.put("process", "permission");
post.put("iam", yacyCore.seedDB.mySeed.hash);
post.put("purpose", "crcon");
post.put("filename", filename);
post.put("filesize", Long.toString(filesize));
post.put("can-send-protocol", "http");
// send request
try {
return nxTools.table(
httpc.wput(
new URL("http://" + targetAddress + "/yacy/transfer.html"),
8000,
null,
null,
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
post,
null
)
);
} catch (Exception e) {
// most probably a network time-out exception
yacyCore.log.logSevere("yacyClient.permissionTransfer error:" + e.getMessage());
return null;
}
}
public static HashMap transferStore(String targetAddress, String access, String filename, byte[] file) {
// should we use the proxy?
boolean useProxy = (yacyCore.seedDB != null) &&
(yacyCore.seedDB.sb.remoteProxyConfig != null) &&
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy()) &&
(yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy());
// adding all needed parameters
final String key = crypt.randomSalt();
final serverObjects post = new serverObjects(7);
post.put("key", key);
post.put("process", "store");
post.put("iam", yacyCore.seedDB.mySeed.hash);
post.put("purpose", "crcon");
post.put("filename", filename);
post.put("filesize", Long.toString(file.length));
post.put("md5", serverCodings.encodeMD5Hex(file));
post.put("access", access);
HashMap files = new HashMap();
files.put("filename", file);
// sending request
try {
final ArrayList v = httpc.wput(
new URL("http://" + address + "/yacy/message.html"),
new URL("http://" + targetAddress + "/yacy/transfer.html"),
20000,
null,
null,
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
post
post,
files
);
return nxTools.table(v);
@ -583,7 +665,32 @@ public final class yacyClient {
return null;
}
}
public static String transfer(String targetAddress, String filename, byte[] file) {
HashMap phase1 = transferPermission(targetAddress, file.length, filename);
if (phase1 == null) return "no connection to remote address " + targetAddress + "; phase 1";
String access = (String) phase1.get("access");
String nextaddress = (String) phase1.get("address");
String protocol = (String) phase1.get("protocol");
String path = (String) phase1.get("path");
String maxsize = (String) phase1.get("maxsize");
String response = (String) phase1.get("response");
if ((response == null) || (protocol == null) || (access == null)) return "wrong return values from other peer; phase 1";
if (!(response.equals("ok"))) return "remote peer rejected transfer: " + response;
String accesscode = serverCodings.encodeMD5Hex(serverCodings.standardCoder.encodeBase64String(access));
if (protocol.equals("http")) {
HashMap phase2 = transferStore(nextaddress, accesscode, filename, file);
if (phase2 == null) return "no connection to remote address " + targetAddress + "; phase 2";
response = (String) phase2.get("response");
if (response == null) return "wrong return values from other peer; phase 2";
if (!(response.equals("ok"))) {
return "remote peer failed with transfer: " + response;
}
return null;
}
return "wrong protocol: " + protocol;
}
public static HashMap crawlOrder(yacySeed targetSeed, URL url, URL referrer) {
// this post a message to the remote message board
if (targetSeed == null) { return null; }
@ -621,7 +728,8 @@ public final class yacyClient {
null,
null,
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
post
post,
null
)
);
} catch (Exception e) {
@ -797,7 +905,8 @@ public final class yacyClient {
null,
null,
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
post
post,
null
);
// this should return a list of urlhashes that are unknwon
if (v != null) {
@ -853,7 +962,8 @@ public final class yacyClient {
null,
null,
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
post
post,
null
);
if (v != null) {
@ -886,7 +996,8 @@ public final class yacyClient {
null,
null,
(useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null,
post
post,
null
);
return nxTools.table(v);

@ -398,7 +398,12 @@ public class yacyCore {
} catch (IOException e) {
log.logSevere("publishMySeed: problem with news encoding", e);
}
seedDB.mySeed.setUnusedFlags();
// include current citation-rank file count
seedDB.mySeed.put(yacySeed.CRWCNT, Integer.toString(switchboard.rankingOwnDistribution.size()));
seedDB.mySeed.put(yacySeed.CRTCNT, Integer.toString(switchboard.rankingOtherDistribution.size()));
// holding a reference to all started threads
int contactedSeedCount = 0;
final List syncList = Collections.synchronizedList(new LinkedList()); // memory for threads

@ -108,7 +108,9 @@ public class yacySeed {
public static final String NCOUNT = "NCount";
public static final String ICOUNT = "ICount";
public static final String SCOUNT = "SCount";
public static final String CCOUNT = "CCount";
public static final String CCOUNT = "CCount"; // Connection Count
public static final String CRWCNT = "CRWCnt"; // Citation Rank (Own) - Count
public static final String CRTCNT = "CRTCnt"; // Citation Rank (Other) - Count
public static final String IP = "IP";
public static final String PORT = "Port";
@ -159,12 +161,16 @@ public class yacySeed {
this.dna.put(LASTSEEN, yacyCore.universalDateShortString(new Date())); // for last-seen date
this.dna.put(USPEED, "0"); // the computated uplink speed of the peer
this.dna.put(CRWCNT, "0");
this.dna.put(CRTCNT, "0");
// settings that are needed to organize the seed round-trip
this.dna.put(FLAGS, "0000");
setFlagDirectConnect(false);
setFlagAcceptRemoteCrawl(true);
setFlagAcceptRemoteIndex(true);
setUnusedFlags();
// index transfer
this.dna.put(INDEX_OUT, "0"); // send index
this.dna.put(INDEX_IN, "0"); // received Index
@ -368,6 +374,9 @@ public class yacySeed {
//if (getVersion() < 0.335) return false;
return getFlag(2);
}
public void setUnusedFlags() {
for (int i = 3; i < 24; i++) setFlag(i, true);
}
public boolean isVirgin() {
return get(PEERTYPE, "").equals(PEERTYPE_VIRGIN);
}
@ -515,6 +524,13 @@ public class yacySeed {
return newSeed;
}
public static String randomHash() {
String hash =
serverCodings.encodeMD5B64(System.currentTimeMillis() + "a", true).substring(0, 6) +
serverCodings.encodeMD5B64(System.currentTimeMillis() + "b", true).substring(0, 6);
return hash;
}
public static yacySeed genRemoteSeed(String seedStr, String key) {
// this method is used to convert the external representation of a seed into a seed object
// yacyCore.log.logFinest("genRemoteSeed: seedStr=" + seedStr + " key=" + key);

@ -224,83 +224,45 @@ public final class yacySeedDB {
public Enumeration seedsSortedConnected(boolean up, String field) {
// enumerates seed-type objects: all seeds sequentially ordered by field
return new seedEnum(up, field, seedActiveDB);
return new seedEnum(up, field, seedActiveDB);
}
public Enumeration seedsSortedDisconnected(boolean up, String field) {
// enumerates seed-type objects: all seeds sequentially ordered by field
return new seedEnum(up, field, seedPassiveDB);
return new seedEnum(up, field, seedPassiveDB);
}
public Enumeration seedsSortedPotential(boolean up, String field) {
// enumerates seed-type objects: all seeds sequentially ordered by field
return new seedEnum(up, field, seedPotentialDB);
return new seedEnum(up, field, seedPotentialDB);
}
public Enumeration seedsConnected(boolean up, boolean rot, String firstHash) {
// enumerates seed-type objects: all seeds sequentially without order
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash.getBytes(), seedActiveDB);
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash.getBytes(), seedActiveDB);
}
public Enumeration seedsDisconnected(boolean up, boolean rot, String firstHash) {
// enumerates seed-type objects: all seeds sequentially without order
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash.getBytes(), seedPassiveDB);
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash.getBytes(), seedPassiveDB);
}
public Enumeration seedsPotential(boolean up, boolean rot, String firstHash) {
// enumerates seed-type objects: all seeds sequentially without order
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash.getBytes(), seedPotentialDB);
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash.getBytes(), seedPotentialDB);
}
public yacySeed anySeed() {
// return just any probe candidate
yacySeed seed;
if ((seedQueue == null) || (seedQueue.size() == 0)) {
if (seedActiveDB.size() <= 0) return null;
// fill up the queue
seedQueue = new disorderHeap();
Iterator keyIt;
try {
keyIt = seedActiveDB.keys(true, false); // iteration of String - Objects
} catch (IOException e) {
yacyCore.log.logSevere("yacySeedCache.anySeed: seed.db not available: " + e.getMessage());
keyIt = (new HashSet()).iterator();
}
String seedHash;
String myIP = (mySeed == null) ? "" : ((String) mySeed.get(yacySeed.IP, "127.0.0.1"));
while (keyIt.hasNext()) {
seedHash = (String) keyIt.next();
try {
seed = new yacySeed(seedHash, seedActiveDB.get(seedHash));
// check here if the seed is equal to the own seed
// this should never be the case, but it happens if a redistribution circle exists
if ((mySeed != null) && (seedHash.equals(mySeed.hash))) {
// this seed should not be in the database
seedActiveDB.remove(seedHash);
} else {
// add to queue
seedQueue.add(seed);
}
} catch (IOException e) {}
public yacySeed anySeedVersion(float minVersion) {
// return just any seed that has a specific minimum version number
yacySeed seed;
Enumeration e = seedsConnected(true, true, yacySeed.randomHash());
int maxtry = seedActiveDB.size();
for (int i = 0; i < maxtry; i++) {
seed = (yacySeed) e.nextElement();
System.out.println("ENUMSEED: " + ((seed == null) ? "NULL" : seed.getName()));
if ((seed != null) && (seed.getVersion() >= minVersion)) return seed;
}
// the queue is filled up!
}
if ((seedQueue == null) || (seedQueue.size() == 0)) return null;
return (yacySeed) seedQueue.remove();
}
public yacySeed anySeedType(String type) {
// this returns any seed that has a special PeerType
yacySeed ys;
String t;
for (int i = 0; i < seedActiveDB.size(); i++) {
ys = anySeed();
if (ys == null) return null;
t = (String) ys.get(yacySeed.PEERTYPE, "");
if ((t != null) && (t.equals(type))) return ys;
}
return null;
return null;
}
public yacySeed[] seedsByAge(boolean up, int count) {
@ -648,8 +610,9 @@ public final class yacySeedDB {
// store other seeds
yacySeed ys;
for (int i = 0; i < seedActiveDB.size(); i++) {
ys = anySeed();
Enumeration se = seedsConnected(true, false, null);
while (se.hasMoreElements()) {
ys = (yacySeed) se.nextElement();
if (ys != null) {
line = ys.genSeedStr(null);
v.add(line);

@ -3,4 +3,5 @@ package de.anomic.yacy;
public final class yacyVersion {
public static final float YACY_SUPPORTS_PORT_FORWARDING = (float) 0.383;
public static final float YACY_SUPPORTS_GZIP_POST_REQUESTS = (float) 0.40300772;
public static final float YACY_ACCEPTS_RANKING_TRANSMISSION = (float) 0.414;
}

@ -83,6 +83,7 @@ import de.anomic.server.serverSystem;
import de.anomic.server.logging.serverLog;
import de.anomic.tools.enumerateFiles;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyClient;
/**
* This is the main class of the proxy. Several threads are started from here:
@ -1069,6 +1070,20 @@ public final class yacy {
serverLog.logConfig("DELETE-STOPWORDS", "FINISHED");
}
private static void transferCR(String targetaddress, String crfile) {
File f = new File(crfile);
try {
byte[] b = serverFileUtils.read(f);
String result = yacyClient.transfer(targetaddress, f.getName(), b);
if (result == null)
serverLog.logInfo("TRANSFER-CR", "transmitted file " + crfile + " to " + targetaddress + " successfully");
else
serverLog.logInfo("TRANSFER-CR", "error transmitting file " + crfile + " to " + targetaddress + ": " + result);
} catch (IOException e) {
serverLog.logInfo("TRANSFER-CR", "could not read file " + crfile);
}
}
/**
* Main-method which is started by java. Checks for special arguments or
* starts up the application.
@ -1134,6 +1149,11 @@ public final class yacy {
int minlength = Integer.parseInt(args[2]);
int maxlength = Integer.parseInt(args[3]);
cleanwordlist(args[1], minlength, maxlength);
} else if ((args.length >= 1) && (args[0].equals("-transfercr"))) {
// transfer a single cr file to a remote peer
String targetaddress = args[1];
String crfile = args[2];
transferCR(targetaddress, crfile);
} else {
if (args.length == 1) applicationRoot= args[0];
startup(applicationRoot, startupMemFree, startupMemTotal);

@ -490,10 +490,10 @@ ramCachePreNURL = 4194304
# is only valid in unix/shell environments and
# not for first startup of YaCy
# -Xms<size> set initial Java heap size
# -Xmx<size> set maximum Java heap size
javastart_Xmx=Xmx64m
# -Xmx<size> set maximum Java heap size
# -Xms<size> set initial Java heap size
javastart_Xms=Xms10m

Loading…
Cancel
Save