boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

redesign of last-seen fieln inside seed:

Browse Source 
the field contains now a time in UDC-0 (instead relative to local UDC offset)
this fixes a bug in peer selection, where an iteration over all seeds
ordered by lastseen did not work correctly.
Problems may occur because the new meaning of this field may mix with
the different meaning of that field in older peers

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3322 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 18 years ago
parent e68cdeeeb3
commit b2f4087400
9 changed files with 51 additions and 55 deletions
Show Stats Download Patch File Download Diff File

2
build.properties
Unescape View File

@ -3,7 +3,7 @@ javacSource=1.4
javacTarget=1.4
# Release Configuration
releaseVersion=0.502
releaseVersion=0.503
releaseFile=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
#releaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
releaseDir=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}

2
htroot/Network.java
Unescape View File

@ -391,7 +391,7 @@ public class Network {
}
prop.put(STR_TABLE_LIST + conCount + "_type_url", seed.get("seedURL", "http://nowhere/"));
final long lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenTime()) / 1000 / 60);
final long lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenUTC()) / 1000 / 60);
if (page == 2 || lastseen > 1440) { // Passive Peers should be passive, also Peers without contact greater than an day
// principal/senior/junior: red/red=offline
prop.put(STR_TABLE_LIST + conCount + "_type_direct", 2);

4
htroot/ViewProfile.java
Unescape View File

@ -105,10 +105,6 @@ public class ViewProfile {
// read the profile from remote peer
yacySeed seed = yacyCore.seedDB.getConnected(hash);
if (seed == null) seed = yacyCore.seedDB.getDisconnected(hash);
if (seed != null) {
long lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenTime()) / 1000 / 60);
if (lastseen > 120) seed = null; // if contact is too old, we treat it as disconnected
}
if (seed == null) {
prop.put("success", "1"); // peer unknown
} else {

6
htroot/yacy/hello.java
Unescape View File

@ -50,7 +50,6 @@ import java.util.Date;
import de.anomic.http.httpHeader;
import de.anomic.server.serverCore;
import de.anomic.server.serverDate;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyClient;
@ -124,7 +123,9 @@ public final class hello {
}
// System.out.println("YACYHELLO: YOUR IP=" + clientip);
// set lastseen value (we have seen that peer, it contacted us!)
remoteSeed.setLastSeenUTC();
// assign status
if (urls >= 0) {
if (remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) == null) {
@ -140,7 +141,6 @@ public final class hello {
yacyCore.peerActions.peerArrival(remoteSeed, true);
} else {
prop.putASIS(yacySeed.YOURTYPE, yacySeed.PEERTYPE_JUNIOR);
remoteSeed.put(yacySeed.LASTSEEN, yacyCore.universalDateShortString(new Date(System.currentTimeMillis() + serverDate.UTCDiff() - remoteSeed.getUTCDiff())) );
yacyCore.peerActions.juniorConnects++; // update statistics
remoteSeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
yacyCore.log.logInfo("hello: responded remote junior peer '" + remoteSeed.getName() + "' from " + reportedip);

4
source/de/anomic/plasma/plasmaGrafics.java
Unescape View File

@ -182,7 +182,7 @@ public class plasmaGrafics {
while (e.hasMoreElements() && count < maxCount) {
seed = (yacySeed) e.nextElement();
if (seed != null) {
lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenTime()) / 1000 / 60);
lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenUTC()) / 1000 / 60);
if (lastseen > passiveLimit) break; // we have enough, this list is sorted so we don't miss anything
drawNetworkPicturePeer(networkPicture, width / 2, height / 2, innerradius, outerradius, seed, COL_PASSIVE_DOT, COL_PASSIVE_LINE, COL_PASSIVE_TEXT, corona);
count++;
@ -196,7 +196,7 @@ public class plasmaGrafics {
while (e.hasMoreElements() && count < maxCount) {
seed = (yacySeed) e.nextElement();
if (seed != null) {
lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenTime()) / 1000 / 60);
lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenUTC()) / 1000 / 60);
if (lastseen > potentialLimit) break; // we have enough, this list is sorted so we don't miss anything
drawNetworkPicturePeer(networkPicture, width / 2, height / 2, innerradius, outerradius, seed, COL_POTENTIAL_DOT, COL_POTENTIAL_LINE, COL_POTENTIAL_TEXT, corona);
count++;

4
source/de/anomic/yacy/yacyCore.java
Unescape View File

@ -6,7 +6,7 @@
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
// $LastChangedBy: $
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@ -346,7 +346,7 @@ public class yacyCore {
// if we cannot reach ourself, we call a forced publishMySeed and return false
final int urlc = yacyClient.queryUrlCount(seedDB.mySeed);
if (urlc >= 0) {
seedDB.mySeed.put(yacySeed.LASTSEEN, universalDateShortString(new Date()));
seedDB.mySeed.setLastSeenUTC();
return true;
}
log.logInfo("re-connect own seed");

26
source/de/anomic/yacy/yacyPeerActions.java
Unescape View File

@ -48,7 +48,6 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
@ -137,7 +136,7 @@ public class yacyPeerActions {
seedDB.mySeed.put("seedURL", sb.getConfig("seedURL", ""));
}
seedDB.mySeed.setFlagDirectConnect(true);
seedDB.mySeed.put(yacySeed.LASTSEEN, yacyCore.universalDateShortString(new Date()));
seedDB.mySeed.setLastSeenUTC();
seedDB.mySeed.put(yacySeed.UTC, serverDate.UTCDiffString());
seedDB.mySeed.setFlagAcceptRemoteCrawl(sb.getConfig("crawlResponse", "").equals("true"));
seedDB.mySeed.setFlagAcceptRemoteIndex(sb.getConfig("allowReceiveIndex", "").equals("true"));
@ -298,20 +297,20 @@ public class yacyPeerActions {
if (seed.get(yacySeed.LASTSEEN, "").length() != 14) {
// hack for peers that do not have a LastSeen date
seed.put(yacySeed.LASTSEEN, "20040101000000");
seed.setLastSeenUTC();
yacyCore.log.logFine("connect: reset wrong date (" + seed.getName() + "/" + seed.hash + ")");
}
// connection time
final long nowUTC0Time = System.currentTimeMillis(); // is better to have this value in a variable for debugging
final long ctimeUTC0 = seed.getLastSeenTime();
// maybe correct it slightly
/*
* if (ctime > yacyCore.universalTime()) { ctime = ((2 * ctime) +
* yacyCore.universalTime()) / 3; seed.put(yacySeed.LASTSEEN,
* yacyCore.shortFormatter.format(new Date(ctime))); }
*/
long ctimeUTC0 = seed.getLastSeenUTC();
if (ctimeUTC0 > nowUTC0Time) {
// the peer is future-dated, correct it
seed.setLastSeenUTC();
ctimeUTC0 = nowUTC0Time;
assert (seed.getLastSeenUTC() - ctimeUTC0 < 100);
}
if (Math.abs(nowUTC0Time - ctimeUTC0) > 60 * 60 * 24 * 1000) {
// the new connection is out-of-age, we reject the connection
yacyCore.log.logFine("connect: rejecting out-dated peer '" + seed.getName() + "' from " + seed.getAddress() + "; nowUTC0=" + nowUTC0Time + ", seedUTC0=" + ctimeUTC0 + ", TimeDiff=" + serverDate.intervalToString(Math.abs(nowUTC0Time - ctimeUTC0)));
@ -336,7 +335,7 @@ public class yacyPeerActions {
// Date applies the local UTC offset, which is wrong
// we correct that by subtracting the local offset and adding
// the remote offset.
seed.setLastSeenTime();
seed.setLastSeenUTC();
seed.setFlagDirectConnect(true);
} else {
// set connection flag
@ -371,7 +370,7 @@ public class yacyPeerActions {
try {
// if the old LastSeen date is later then the other
// info, then we reject the info
if ((ctimeUTC0 < (yacyCore.parseUniversalDate(connectedSeed.get(yacySeed.LASTSEEN, "20040101000000")).getTime() - connectedSeed.getUTCDiff() + serverDate.UTCDiff())) && (!direct)) {
if ((ctimeUTC0 < (connectedSeed.getLastSeenUTC())) && (!direct)) {
yacyCore.log.logFine("connect: rejecting old info about peer '" + seed.getName() + "'");
return false;
}
@ -379,9 +378,6 @@ public class yacyPeerActions {
if (connectedSeed.getName() != seed.getName()) {
// TODO: update seed name lookup cache
}
} catch (ParseException e) {
yacyCore.log.logFine("connect: rejecting wrong peer '" + seed.getName() + "' from " + seed.getAddress() + ". Cause: " + e.getMessage());
return false;
} catch (NumberFormatException e) {
yacyCore.log.logFine("connect: rejecting wrong peer '" + seed.getName() + "' from " + seed.getAddress() + ". Cause: " + e.getMessage());
return false;

56
source/de/anomic/yacy/yacySeed.java
Unescape View File

@ -262,14 +262,14 @@ public class yacySeed {
return (String) o;
}
public final void setIP() { put(yacySeed.IP, ""); }
public final void setIP(final String ip) { put(yacySeed.IP, ip); }
public final void setPort(final String port) { put(yacySeed.PORT, port); }
public final void setJunior() { put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR); }
public final void setSenior() { put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); }
public final void setPrincipal() { put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_PRINCIPAL); }
public final void setLastSeen() { put(yacySeed.LASTSEEN, yacyCore.universalDateShortString(new Date(System.currentTimeMillis() + serverDate.UTCDiff() - getUTCDiff()))); }
public final void setIP() { dna.put(yacySeed.IP, ""); }
public final void setIP(final String ip) { dna.put(yacySeed.IP, ip); }
public final void setPort(final String port) { dna.put(yacySeed.PORT, port); }
public final void setJunior() { dna.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR); }
public final void setSenior() { dna.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); }
public final void setPrincipal() { dna.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_PRINCIPAL); }
public final void put(String key, String value) {
synchronized (this.dna) {
this.dna.put(key, value);
@ -291,25 +291,25 @@ public class yacySeed {
public final void incSI(int count) {
String v = (String) this.dna.get(yacySeed.INDEX_OUT);
if (v == null) { v = yacySeed.ZERO; }
put(yacySeed.INDEX_OUT, Integer.toString(Integer.parseInt(v) + count));
dna.put(yacySeed.INDEX_OUT, Integer.toString(Integer.parseInt(v) + count));
}
public final void incRI(int count) {
String v = (String) this.dna.get(yacySeed.INDEX_IN);
if (v == null) { v = yacySeed.ZERO; }
put(yacySeed.INDEX_IN, Integer.toString(Integer.parseInt(v) + count));
dna.put(yacySeed.INDEX_IN, Integer.toString(Integer.parseInt(v) + count));
}
public final void incSU(int count) {
String v = (String) this.dna.get(yacySeed.URL_OUT);
if (v == null) { v = yacySeed.ZERO; }
put(yacySeed.URL_OUT, Integer.toString(Integer.parseInt(v) + count));
dna.put(yacySeed.URL_OUT, Integer.toString(Integer.parseInt(v) + count));
}
public final void incRU(int count) {
String v = (String) this.dna.get(yacySeed.URL_IN);
if (v == null) { v = yacySeed.ZERO; }
put(yacySeed.URL_IN, Integer.toString(Integer.parseInt(v) + count));
dna.put(yacySeed.URL_IN, Integer.toString(Integer.parseInt(v) + count));
}
// 12 * 6 bit = 72 bit = 18 characters hex-hash
@ -379,15 +379,24 @@ public class yacySeed {
}
}
public final long getLastSeenTime() {
public final void setLastSeenUTC() {
// we put the current time into the lastseen field
// because java thinks it must apply the UTC offset to the current time,
// to create a string that looks like our current time, it adds the local UTC offset to the
// time. To create a corrected UTC Date string, we first subtract the local UTC offset.
dna.put(yacySeed.LASTSEEN, yacyCore.universalDateShortString(new Date(System.currentTimeMillis() - serverDate.UTCDiff())) );
}
public final long getLastSeenUTC() {
try {
final long t = yacyCore.parseUniversalDate(get(yacySeed.LASTSEEN, "20040101000000")).getTime();
// the problem here is: getTime applies a time shift according to local time zone:
// it substracts the local UTF offset, but it should subtract the remote UTC offset
// so we correct it by first adding the local UTF offset and then subtracting the remote
// the time zone was originally the seeds time zone
// we correct this here
return t - getUTCDiff() + serverDate.UTCDiff();
// getTime creates a UTC time number. But in this case java thinks, that the given
// time string is a local time, which has a local UTC offset applied.
// Therefore java subtracts the local UTC offset, to get a UTC number.
// But the given time string is already in UTC time, so the subtraction
// of the local UTC offset is wrong. We correct this here by adding the local UTC
// offset again.
return t + serverDate.UTCDiff();
} catch (java.text.ParseException e) {
return System.currentTimeMillis();
} catch (java.lang.NumberFormatException e) {
@ -411,13 +420,8 @@ public class yacySeed {
}
}
public final void setLastSeenTime() {
// if we set a last seen time, then we need to respect the seeds UTC offset
put(yacySeed.LASTSEEN, yacyCore.universalDateShortString(new Date(System.currentTimeMillis() - serverDate.UTCDiff() + getUTCDiff())));
}
public void setPeerTags(Set keys) {
put(PEERTAGS, serverCodings.set2string(keys, "|", false));
dna.put(PEERTAGS, serverCodings.set2string(keys, "|", false));
}
public Set getPeerTags() {
@ -458,7 +462,7 @@ public class yacySeed {
if (flags.length() != 4) { flags = yacySeed.FLAGSZERO; }
final bitfield f = new bitfield(flags.getBytes());
f.set(flag, value);
put(yacySeed.FLAGS, new String(f.getBytes()));
dna.put(yacySeed.FLAGS, new String(f.getBytes()));
}
public final void setFlagDirectConnect(final boolean value) { setFlag(0, value); }

2
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -316,7 +316,7 @@ public final class yacySeedDB {
while ((s.hasMoreElements()) && (searchcount-- > 0)) {
ys = (yacySeed) s.nextElement();
if ((ys != null) && (ys.get(yacySeed.LASTSEEN, "").length() > 10)) try {
absage = Math.abs(System.currentTimeMillis() - ys.getLastSeenTime());
absage = Math.abs(System.currentTimeMillis() - ys.getLastSeenUTC());
seedScore.addScore(ys.hash, (int) absage);
} catch (Exception e) {}
}

Write Preview
Loading…
Cancel
Save