boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

*) use majority voting for peer type decision

Browse Source 
*) reduce the number of peer pings sent out
see: http://www.yacy-forum.de/viewtopic.php?t=1748



git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1411 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
hermens 19 years ago
parent 77548017f0
commit 75b268f16d
4 changed files with 193 additions and 15 deletions
Show Stats Download Patch File Download Diff File

52
source/de/anomic/yacy/yacyAccessible.java
Unescape View File

@ -0,0 +1,52 @@
// yacyAccessible.java
// -------------------------------------
// (C) by Michael Peter Christen; mc@anomic.de
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004
//
// this file is contributed by Stephan Hermens
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
// Using this software in any meaning (reading, learning, copying, compiling,
// running) means that you agree that the Author(s) is (are) not responsible
// for cost, loss of data or any harm that may be caused directly or indirectly
// by usage of this softare or this documentation. The usage of this software
// is on your own risk. The installation and usage (starting/running) of this
// software may allow other people or application to access your computer and
// any attached devices and is highly dependent on the configuration of the
// software which must be done by the user of the software; the author(s) is
// (are) also not responsible for proper configuration and usage of the
// software, even if provoked by documentation provided together with
// the software.
//
// Any changes to this file according to the GPL as documented in the file
// gpl.txt aside this file in the shipment you received can be done to the
// lines that follows this copyright notice here, but changes must not be
// done inside the copyright notive above. A re-distribution must contain
// the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such.
/* This class just defines a container */
package de.anomic.yacy;
public class yacyAccessible {
public long lastUpdated;
public boolean IWasAccessed;
}

14
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -177,12 +177,17 @@ public final class yacyClient {
if (!serverCore.portForwardingEnabled || otherPeerVersion >= yacyVersion.YACY_SUPPORTS_PORT_FORWARDING) {
String mytype = (String) result.get(yacySeed.YOURTYPE);
if (mytype == null) { mytype = yacySeed.PEERTYPE_JUNIOR; }
if (
(yacyCore.seedDB.mySeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR).equals(yacySeed.PEERTYPE_PRINCIPAL)) &&
(mytype.equals(yacySeed.PEERTYPE_SENIOR))
) {
yacyAccessible accessible = new yacyAccessible();
if (mytype.equals(yacySeed.PEERTYPE_SENIOR)) {
accessible.IWasAccessed = true;
if (yacyCore.seedDB.mySeed.isPrincipal()) {
mytype = yacySeed.PEERTYPE_PRINCIPAL;
}
} else {
accessible.IWasAccessed = false;
}
accessible.lastUpdated = System.currentTimeMillis();
yacyCore.amIAccessibleDB.put(otherHash, accessible);
/*
* If we were reported as junior we have to check if your port forwarding channel is broken
@ -206,6 +211,7 @@ public final class yacyClient {
} else {
yacyCore.log.logFine("yacyClient.publishMySeed: Peer '" + ((otherPeer==null)?"unknown":otherPeer.getName()) + "' reported us as " + mytype + ".");
}
if (yacyCore.seedDB.mySeed.orVirgin().equals(yacySeed.PEERTYPE_VIRGIN))
yacyCore.seedDB.mySeed.put(yacySeed.PEERTYPE, mytype);
}

133
source/de/anomic/yacy/yacyCore.java
Unescape View File

@ -68,8 +68,10 @@ import java.util.Collections;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
import de.anomic.http.httpc;
import de.anomic.net.natLib;
@ -95,6 +97,14 @@ public class yacyCore {
public static float latestVersion = (float) 0.1;
public static long speedKey = 0;
public static File yacyDBPath;
public static final Map amIAccessibleDB = Collections.synchronizedMap(new HashMap()); // Holds PeerHash / yacyAccessible Relations
// constants for PeerPing behaviour
private static final int peerPingInitial = 10;
private static final int peerPingMaxRunning = 3;
private static final int peerPingMinRunning = 1;
private static final int peerPingMinDBSize = 5;
private static final int peerPingMinAccessibleToForceSenior = 3;
private static final long peerPingMaxDBAge = 15 * 60 * 1000; // in milliseconds
// public static yacyShare shareManager = null;
// public static boolean terminate = false;
@ -377,15 +387,42 @@ public class yacyCore {
yacySeed[] seeds;
int attempts = seedDB.sizeConnected();
if (attempts > 10) { attempts = 10; }
// getting a list of peers to contact
if (seedDB.mySeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN).equals(yacySeed.PEERTYPE_VIRGIN)) {
seeds = seedDB.seedsByAge(true, attempts); // best for fast connection
if (attempts > peerPingInitial) { attempts = peerPingInitial; }
seeds = seedDB.seedsByAge(true, attempts + 10); // best for fast connection
} else {
seeds = seedDB.seedsByAge(false, attempts); // best for seed list maintenance/cleaning
int diff = peerPingMinDBSize - amIAccessibleDB.size();
if (diff > peerPingMinRunning) {
diff = Math.max(diff, peerPingMaxRunning);
if (attempts > diff) { attempts = diff; }
} else {
if (attempts > peerPingMinRunning) { attempts = peerPingMinRunning; }
}
seeds = seedDB.seedsByAge(false, attempts + 10); // best for seed list maintenance/cleaning
}
if (seeds == null) { return 0; }
if (seeds.length < attempts) { attempts = seeds.length; }
// This will try to get Peers that are not currently in amIAccessibleDB
int i = 0;
int j = seeds.length - 1; // attempts;
while (i < j) {
while ((i < j)&&(!amIAccessibleDB.containsKey(seeds[i].hash))) {
i++;
}
while ((i < j)&&(amIAccessibleDB.containsKey(seeds[j].hash))) {
j--;
}
if (i >= j) break;
yacySeed seed = seeds[i];
seeds[i] = seeds[j];
seeds[j] = seed;
i++;
j--;
}
// include a YaCyNews record to my seed
try {
@ -403,14 +440,15 @@ public class yacyCore {
// include current citation-rank file count
seedDB.mySeed.put(yacySeed.CRWCNT, Integer.toString(switchboard.rankingOwnDistribution.size()));
seedDB.mySeed.put(yacySeed.CRTCNT, Integer.toString(switchboard.rankingOtherDistribution.size()));
int newSeeds = -1;
//if (seeds.length > 1) {
// holding a reference to all started threads
int contactedSeedCount = 0;
final List syncList = Collections.synchronizedList(new LinkedList()); // memory for threads
final serverSemaphore sync = new serverSemaphore(attempts);
// going through the peer list and starting a new publisher thread for each peer
for (int i = 0; i < seeds.length; i++) {
for (i = 0; i < attempts; i++) {
if (seeds[i] == null) continue;
final String address = seeds[i].getAddress();
@ -418,23 +456,27 @@ public class yacyCore {
if ((address == null) || (seeds[i].isProper() != null)) {
// we don't like that address, delete it
peerActions.peerDeparture(seeds[i]);
sync.V();
sync.P();
} else {
// starting a new publisher thread
contactedSeedCount++;
(new publishThread(yacyCore.publishThreadGroup,seeds[i],sync,syncList)).start();
}
}
int reserveSeedsAt = i; // This is just a precaution
// receiving the result of all started publisher threads
int newSeeds = -1;
for (int j=0; j < contactedSeedCount; j++) {
for (j = 0; j < contactedSeedCount; j++) {
// waiting for the next thread to finish
sync.P();
// if this is true something is wrong ...
if (syncList.isEmpty()) return 0;
if (syncList.isEmpty()) {
log.logWarning("PeerPing: syncList.isEmpty()==true");
continue;
//return 0;
}
// getting a reference to the finished thread
final publishThread t = (publishThread) syncList.remove(0);
@ -449,6 +491,79 @@ public class yacyCore {
}
}
// Nobody contacted yet, try again until peerPingInitial attempts are through
i = reserveSeedsAt; // This is just a precaution
while ((newSeeds < 0) && (contactedSeedCount < peerPingInitial) && (i < seeds.length)) {
if (seeds[i] != null) {
final String address = seeds[i].getAddress();
log.logFine("HELLO x" + i + " to peer '" + seeds[i].get(yacySeed.NAME, "") + "' at " + address); // debug
if ((address == null) || (seeds[i].isProper() != null)) {
peerActions.peerDeparture(seeds[i]);
} else {
contactedSeedCount++;
//new publishThread(yacyCore.publishThreadGroup,seeds[i],sync,syncList)).start();
try {
newSeeds = yacyClient.publishMySeed(seeds[i].getAddress(), seeds[i].hash);
if (newSeeds < 0) {
log.logInfo("publish: disconnected " + seeds[i].get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + seeds[i].getName() + "' from " + seeds[i].getAddress());
peerActions.peerDeparture(seeds[i]);
} else {
log.logInfo("publish: handshaked " + seeds[i].get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + seeds[i].getName() + "' at " + seeds[i].getAddress());
}
} catch (Exception e) {
log.logSevere("publishMySeed: error with target seed " + seeds[i].toString() + ": " + e.getMessage(), e);
}
}
}
i++;
}
int accessible = 0;
int notaccessible = 0;
final long cutofftime = System.currentTimeMillis() - peerPingMaxDBAge;
final int dbSize;
synchronized(amIAccessibleDB) {
dbSize = amIAccessibleDB.size();
Iterator ai = amIAccessibleDB.keySet().iterator();
while (ai.hasNext()) {
yacyAccessible ya = (yacyAccessible) amIAccessibleDB.get(ai.next());
if (ya.lastUpdated < cutofftime) {
ai.remove();
} else {
if (ya.IWasAccessed) { accessible++; }
else { notaccessible++; }
}
}
}
log.logFine("DBSize before -> after Cleanup: " + dbSize + " -> " + amIAccessibleDB.size());
log.logInfo("PeerPing: I am accessible for " + accessible +
"peer(s), not accessible for " + notaccessible + " peer(s).");
if ((accessible + notaccessible) > 0) {
final String newPeerType;
// At least one other Peer told us our type
if ((accessible >= peerPingMinAccessibleToForceSenior) ||
(accessible >= notaccessible)) {
// We can be reached from a majority of other Peers
if (yacyCore.seedDB.mySeed.isPrincipal()) {
newPeerType = yacySeed.PEERTYPE_PRINCIPAL;
} else {
newPeerType = yacySeed.PEERTYPE_SENIOR;
}
} else {
// We cannot be reached from the outside
newPeerType = yacySeed.PEERTYPE_JUNIOR;
}
if (yacyCore.seedDB.mySeed.orVirgin().equals(newPeerType)) {
log.logInfo("PeerPing: myType is " + yacyCore.seedDB.mySeed.orVirgin());
} else {
log.logInfo("PeerPing: changing myType from '" + yacyCore.seedDB.mySeed.orVirgin() + "' to '" + newPeerType + "'");
yacyCore.seedDB.mySeed.put(yacySeed.PEERTYPE, newPeerType);
}
} else {
log.logInfo("PeerPing: No data, staying at myType: " + yacyCore.seedDB.mySeed.orVirgin());
}
if (newSeeds >= 0) {
// success! we have published our peer to a senior peer
// update latest news from the other peer

5
source/de/anomic/yacy/yacySeed.java
Unescape View File

@ -195,6 +195,11 @@ public class yacySeed {
* @return the peertype or null
*/
public String getPeerType() { return get(PEERTYPE, ""); }
/**
* try to get the peertype<br>
* @return the peertype or "Virgin"
*/
public String orVirgin() { return get(PEERTYPE, PEERTYPE_VIRGIN); }
/**
* try to get the peertype<br>
* @return the peertype or "junior"

Write Preview
Loading…
Cancel
Save