boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

better ping

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4472 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
borg-0300 17 years ago
parent fc54d4519e
commit 2589290ded
4 changed files with 88 additions and 58 deletions
Show Stats Download Patch File Download Diff File

67
source/de/anomic/yacy/yacyCore.java
Unescape View File

@ -81,24 +81,23 @@ public class yacyCore {
// statics
public static final ThreadGroup publishThreadGroup = new ThreadGroup("publishThreadGroup");
public static yacySeedDB seedDB = null;
public static yacyNewsPool newsPool = null;
public static final HashMap<String, String> seedUploadMethods = new HashMap<String, String>();
public static yacyPeerActions peerActions = null;
public static yacyDHTAction dhtAgent = null;
public static yacySeedDB seedDB;
public static yacyNewsPool newsPool;
public static final HashMap seedUploadMethods = new HashMap();
public static yacyPeerActions peerActions;
public static yacyDHTAction dhtAgent;
public static serverLog log;
public static long lastOnlineTime = 0;
/** pseudo-random key derived from a time-interval while YaCy startup*/
public static long speedKey = 0;
public static long lastOnlineTime;
/** pseudo-random key derived from a time-interval while YaCy startup. */
public static long speedKey;
public static File yacyDBPath;
public static final Map<String, yacyAccessible> amIAccessibleDB = Collections.synchronizedMap(new HashMap<String, yacyAccessible>()); // Holds PeerHash / yacyAccessible Relations
public static final Map amIAccessibleDB = Collections.synchronizedMap(new HashMap()); // Holds PeerHash / yacyAccessible Relations
// constants for PeerPing behaviour
private static final int PING_INITIAL = 16;
private static final int PING_MAX_RUNNING = 8;
private static final int PING_MIN_RUNNING = 4;
private static final int PING_INITIAL = 64;
private static final int PING_MIN_LASTSEEN = 240000; // in milliseconds
private static final int PING_MIN_PEERSEEN = 2; // min. accessible to force senior
private static final long PING_MAX_DBAGE = 15 * 60 * 1000; // in milliseconds
// public static yacyShare shareManager = null;
// public static boolean terminate = false;
@ -372,8 +371,8 @@ public class yacyCore {
int attempts = seedDB.sizeConnected();
// getting a list of peers to contact
if (seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN).equals(yacySeed.PEERTYPE_VIRGIN)) {
if (attempts > PING_INITIAL) { attempts = PING_INITIAL; }
if (seedDB.mySeed().isVirgin()) {
attempts = Math.min(attempts, PING_INITIAL);
final Map<String, String> ch = plasmaSwitchboard.getSwitchboard().clusterhashes;
seeds = seedDB.seedsByAge(true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection
// add also all peers from cluster if this is a public robinson cluster
@ -395,20 +394,10 @@ public class yacyCore {
}
}
} else {
if (amIAccessibleDB.size() > PING_MAX_RUNNING) {
attempts = PING_MAX_RUNNING;
} else {
attempts = Math.min(attempts, PING_MIN_RUNNING);
}
seeds = seedDB.seedsByAge(false, attempts); // best for seed list maintenance/cleaning
seeds = seedDB.getOldestSeeds(attempts / 5, PING_MIN_LASTSEEN);
}
if (seeds == null || seeds.size() == 0) { return 0; }
if (seeds.size() < attempts) { attempts = seeds.size(); }
// This will try to get Peers that are not currently in amIAccessibleDB
Iterator<yacySeed> si = seeds.values().iterator();
yacySeed seed;
attempts = seeds.size();
// include a YaCyNews record to my seed
try {
@ -433,10 +422,14 @@ public class yacyCore {
final List<Thread> syncList = Collections.synchronizedList(new LinkedList<Thread>()); // memory for threads
final serverSemaphore sync = new serverSemaphore(attempts);
// This will try to get Peers that are not currently in amIAccessibleDB
Iterator<yacySeed> si = seeds.values().iterator();
yacySeed seed;
// going through the peer list and starting a new publisher thread for each peer
int i = 0;
while (si.hasNext()) {
seed = (yacySeed) si.next();
seed = si.next();
if (seed == null) {
sync.P();
continue;
@ -445,8 +438,8 @@ public class yacyCore {
final String address = seed.getClusterAddress();
log.logFine("HELLO #" + i + " to peer '" + seed.get(yacySeed.NAME, "") + "' at " + address); // debug
String seederror = seed.isProper();
if ((address == null) || (seederror != null)) {
final String seederror = seed.isProper();
if (address == null || seederror != null) {
// we don't like that address, delete it
peerActions.peerDeparture(seed, "peer ping to peer resulted in address = " + address + "; seederror = " + seederror);
sync.P();
@ -489,9 +482,9 @@ public class yacyCore {
final int dbSize;
synchronized (amIAccessibleDB) {
dbSize = amIAccessibleDB.size();
Iterator<String> ai = amIAccessibleDB.keySet().iterator();
final Iterator<String> ai = amIAccessibleDB.keySet().iterator();
while (ai.hasNext()) {
yacyAccessible ya = (yacyAccessible) amIAccessibleDB.get(ai.next());
final yacyAccessible ya = (yacyAccessible) amIAccessibleDB.get(ai.next());
if (ya.lastUpdated < cutofftime) {
ai.remove();
} else {
@ -507,11 +500,11 @@ public class yacyCore {
log.logInfo("PeerPing: I am accessible for " + accessible +
" peer(s), not accessible for " + notaccessible + " peer(s).");
if ((accessible + notaccessible) > 0) {
if (accessible + notaccessible > 0) {
final String newPeerType;
// At least one other Peer told us our type
if ((accessible >= PING_MIN_PEERSEEN) ||
(accessible >= notaccessible)) {
if (accessible >= PING_MIN_PEERSEEN ||
accessible >= notaccessible) {
// We can be reached from a majority of other Peers
if (yacyCore.seedDB.mySeed().isPrincipal()) {
newPeerType = yacySeed.PEERTYPE_PRINCIPAL;
@ -522,11 +515,11 @@ public class yacyCore {
// We cannot be reached from the outside
newPeerType = yacySeed.PEERTYPE_JUNIOR;
}
if (yacyCore.seedDB.mySeed().orVirgin().equals(newPeerType)) {
if (yacyCore.seedDB.mySeed().isType(newPeerType)) {
log.logInfo("PeerPing: myType is " + yacyCore.seedDB.mySeed().orVirgin());
} else {
log.logInfo("PeerPing: changing myType from '" + yacyCore.seedDB.mySeed().orVirgin() + "' to '" + newPeerType + "'");
yacyCore.seedDB.mySeed().put(yacySeed.PEERTYPE, newPeerType);
yacyCore.seedDB.mySeed().setType(newPeerType);
}
} else {
log.logInfo("PeerPing: No data, staying at myType: " + yacyCore.seedDB.mySeed().orVirgin());

41
source/de/anomic/yacy/yacySeed.java
Unescape View File

@ -84,7 +84,7 @@ import de.anomic.tools.crypt;
public class yacySeed {
public static final int maxsize = 4096;
public static final int maxsize = 4096;
/**
* <b>substance</b> "sI" (send index/words)
*/
@ -281,9 +281,9 @@ public class yacySeed {
* that is used instead the address stored in the seed DNA
*/
public void setAlternativeAddress(String ipport) {
if (ipport == null) return;
int p = ipport.indexOf(':');
if (p < 0) this.alternativeIP = ipport; else this.alternativeIP = ipport.substring(0, p);
if (ipport == null) return;
int p = ipport.indexOf(':');
if (p < 0) this.alternativeIP = ipport; else this.alternativeIP = ipport.substring(0, p);
}
/**
@ -298,7 +298,7 @@ public class yacySeed {
public final String getPeerType() { return get(yacySeed.PEERTYPE, ""); }
/**
* try to get the peertype<br>
* @return the peertype or "Virgin"
* @return the peertype or "virgin"
*/
public final String orVirgin() { return get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN); }
/**
@ -342,14 +342,14 @@ public class yacySeed {
} else return dflt;
}
public final void setIP() { dna.put(yacySeed.IP, ""); }
public final void setIP(final String ip) { dna.put(yacySeed.IP, ip); }
public final void setPort(final String port) { dna.put(yacySeed.PORT, port); }
public final void setJunior() { dna.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR); }
public final void setSenior() { dna.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); }
public final void setPrincipal() { dna.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_PRINCIPAL); }
public final void setIP() { dna.put(yacySeed.IP, ""); }
public final void setIP(String ip) { dna.put(yacySeed.IP, ip); }
public final void setPort(String port) { dna.put(yacySeed.PORT, port); }
public final void setType(String type) { dna.put(yacySeed.PEERTYPE, type); }
public final void setJunior() { dna.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR); }
public final void setSenior() { dna.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); }
public final void setPrincipal() { dna.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_PRINCIPAL); }
public final void put(String key, String value) {
synchronized (this.dna) {
this.dna.put(key, value);
@ -643,8 +643,8 @@ public class yacySeed {
public final void setUnusedFlags() {
for (int i = 4; i < 24; i++) { setFlag(i, true); }
}
public final boolean isPotential() {
return isVirgin() || isJunior();
public final boolean isType(String type) {
return get(yacySeed.PEERTYPE, "").equals(type);
}
public final boolean isVirgin() {
return get(yacySeed.PEERTYPE, "").equals(yacySeed.PEERTYPE_VIRGIN);
@ -652,22 +652,25 @@ public class yacySeed {
public final boolean isJunior() {
return get(yacySeed.PEERTYPE, "").equals(yacySeed.PEERTYPE_JUNIOR);
}
public final boolean isActive() {
return isSenior() || isPrincipal();
}
public final boolean isSenior() {
return get(yacySeed.PEERTYPE, "").equals(yacySeed.PEERTYPE_SENIOR);
}
public final boolean isPrincipal() {
return get(yacySeed.PEERTYPE, "").equals(yacySeed.PEERTYPE_PRINCIPAL);
}
public final boolean isPotential() {
return isVirgin() || isJunior();
}
public final boolean isActive() {
return isSenior() || isPrincipal();
}
public final boolean isOnline() {
return isSenior() || isPrincipal();
}
public final boolean isOnline(final String type) {
return type.equals(yacySeed.PEERTYPE_SENIOR) || type.equals(yacySeed.PEERTYPE_PRINCIPAL);
}
public static final long minDHTNumber = kelondroBase64Order.enhancedCoder.cardinal(kelondroBase64Order.zero(12));
public static final long maxDHTDistance = Long.MAX_VALUE;

34
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -381,6 +381,40 @@ public final class yacySeedDB {
}
}
public HashMap<String, yacySeed> getOldestSeeds(int count, int minage) {
// returns a peerhash/yacySeed relation
try {
final kelondroMScoreCluster seedScore = new kelondroMScoreCluster();
final Iterator<yacySeed> s = seedsConnected(true, false, null, (float) 0.0);
yacySeed ys;
int age;
int searchcount = 1000;
while (s.hasNext() && searchcount-- > 0) {
ys = s.next();
if (ys == null) { continue; }
age = (int) Math.abs(System.currentTimeMillis() + serverDate.dayMillis - ys.getLastSeenUTC());
if (age < minage) { continue; }
seedScore.addScore(ys.hash, age); // the higher age, the older is the peer
}
if (seedScore.size() == 0) { return null; }
// result is now in the score object; create a result vector
final HashMap<String, yacySeed> result = new HashMap<String, yacySeed>();
final Iterator<String> it = seedScore.scores(false);
searchcount = Math.min(count, seedScore.size());
int c = 0;
while (c++ < searchcount && it.hasNext()) {
ys = getConnected(it.next());
if (ys != null && ys.hash != null) { result.put(ys.hash, ys); }
}
return result;
} catch (kelondroException e) {
seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
yacyCore.log.logFine("Internal Error at yacySeedDB.seedsByAge: " + e.getMessage(), e);
return null;
}
}
public int sizeConnected() {
return seedActiveDB.size();
/*

4
yacy.init
Unescape View File

@ -546,8 +546,8 @@ filterOutStopwordsFromTopwords=true
20_dhtdistribution_idlesleep=30000
20_dhtdistribution_busysleep=10000
20_dhtdistribution_memprereq=6291456
30_peerping_idlesleep=90000
30_peerping_busysleep=90000
30_peerping_idlesleep=120000
30_peerping_busysleep=120000
30_peerping_memprereq=1048576
40_peerseedcycle_idlesleep=1800000
40_peerseedcycle_busysleep=1200000

Write Preview
Loading…
Cancel
Save