From 75b268f16dcf004252fcbc39cd7fa42664edd538 Mon Sep 17 00:00:00 2001 From: hermens Date: Sun, 22 Jan 2006 23:14:37 +0000 Subject: [PATCH] *) use majority voting for peer type decision *) reduce the number of peer pings sent out see: http://www.yacy-forum.de/viewtopic.php?t=1748 git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1411 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/de/anomic/yacy/yacyAccessible.java | 52 +++++++++ source/de/anomic/yacy/yacyClient.java | 18 ++- source/de/anomic/yacy/yacyCore.java | 133 ++++++++++++++++++++-- source/de/anomic/yacy/yacySeed.java | 5 + 4 files changed, 193 insertions(+), 15 deletions(-) create mode 100644 source/de/anomic/yacy/yacyAccessible.java diff --git a/source/de/anomic/yacy/yacyAccessible.java b/source/de/anomic/yacy/yacyAccessible.java new file mode 100644 index 000000000..aa666697d --- /dev/null +++ b/source/de/anomic/yacy/yacyAccessible.java @@ -0,0 +1,52 @@ +// yacyAccessible.java +// ------------------------------------- +// (C) by Michael Peter Christen; mc@anomic.de +// first published on http://www.anomic.de +// Frankfurt, Germany, 2004 +// +// this file is contributed by Stephan Hermens +// $LastChangedDate$ +// $LastChangedRevision$ +// $LastChangedBy$ +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// Using this software in any meaning (reading, learning, copying, compiling, +// running) means that you agree that the Author(s) is (are) not responsible +// for cost, loss of data or any harm that may be caused directly or indirectly +// by usage of this softare or this documentation. The usage of this software +// is on your own risk. The installation and usage (starting/running) of this +// software may allow other people or application to access your computer and +// any attached devices and is highly dependent on the configuration of the +// software which must be done by the user of the software; the author(s) is +// (are) also not responsible for proper configuration and usage of the +// software, even if provoked by documentation provided together with +// the software. +// +// Any changes to this file according to the GPL as documented in the file +// gpl.txt aside this file in the shipment you received can be done to the +// lines that follows this copyright notice here, but changes must not be +// done inside the copyright notive above. A re-distribution must contain +// the intact and unchanged copyright notice. +// Contributions and changes to the program code must be marked as such. + +/* This class just defines a container */ + +package de.anomic.yacy; + +public class yacyAccessible { + public long lastUpdated; + public boolean IWasAccessed; +} diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index 831b43810..a8550244c 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -177,12 +177,17 @@ public final class yacyClient { if (!serverCore.portForwardingEnabled || otherPeerVersion >= yacyVersion.YACY_SUPPORTS_PORT_FORWARDING) { String mytype = (String) result.get(yacySeed.YOURTYPE); if (mytype == null) { mytype = yacySeed.PEERTYPE_JUNIOR; } - if ( - (yacyCore.seedDB.mySeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR).equals(yacySeed.PEERTYPE_PRINCIPAL)) && - (mytype.equals(yacySeed.PEERTYPE_SENIOR)) - ) { - mytype = yacySeed.PEERTYPE_PRINCIPAL; + yacyAccessible accessible = new yacyAccessible(); + if (mytype.equals(yacySeed.PEERTYPE_SENIOR)) { + accessible.IWasAccessed = true; + if (yacyCore.seedDB.mySeed.isPrincipal()) { + mytype = yacySeed.PEERTYPE_PRINCIPAL; + } + } else { + accessible.IWasAccessed = false; } + accessible.lastUpdated = System.currentTimeMillis(); + yacyCore.amIAccessibleDB.put(otherHash, accessible); /* * If we were reported as junior we have to check if your port forwarding channel is broken @@ -206,7 +211,8 @@ public final class yacyClient { } else { yacyCore.log.logFine("yacyClient.publishMySeed: Peer '" + ((otherPeer==null)?"unknown":otherPeer.getName()) + "' reported us as " + mytype + "."); } - yacyCore.seedDB.mySeed.put(yacySeed.PEERTYPE, mytype); + if (yacyCore.seedDB.mySeed.orVirgin().equals(yacySeed.PEERTYPE_VIRGIN)) + yacyCore.seedDB.mySeed.put(yacySeed.PEERTYPE, mytype); } final String error = yacyCore.seedDB.mySeed.isProper(); diff --git a/source/de/anomic/yacy/yacyCore.java b/source/de/anomic/yacy/yacyCore.java index b637fe318..4ae3caeef 100644 --- a/source/de/anomic/yacy/yacyCore.java +++ b/source/de/anomic/yacy/yacyCore.java @@ -68,8 +68,10 @@ import java.util.Collections; import java.util.Date; import java.util.GregorianCalendar; import java.util.HashMap; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.TimeZone; import de.anomic.http.httpc; import de.anomic.net.natLib; @@ -95,6 +97,14 @@ public class yacyCore { public static float latestVersion = (float) 0.1; public static long speedKey = 0; public static File yacyDBPath; + public static final Map amIAccessibleDB = Collections.synchronizedMap(new HashMap()); // Holds PeerHash / yacyAccessible Relations + // constants for PeerPing behaviour + private static final int peerPingInitial = 10; + private static final int peerPingMaxRunning = 3; + private static final int peerPingMinRunning = 1; + private static final int peerPingMinDBSize = 5; + private static final int peerPingMinAccessibleToForceSenior = 3; + private static final long peerPingMaxDBAge = 15 * 60 * 1000; // in milliseconds // public static yacyShare shareManager = null; // public static boolean terminate = false; @@ -377,15 +387,42 @@ public class yacyCore { yacySeed[] seeds; int attempts = seedDB.sizeConnected(); - if (attempts > 10) { attempts = 10; } // getting a list of peers to contact if (seedDB.mySeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN).equals(yacySeed.PEERTYPE_VIRGIN)) { - seeds = seedDB.seedsByAge(true, attempts); // best for fast connection + if (attempts > peerPingInitial) { attempts = peerPingInitial; } + seeds = seedDB.seedsByAge(true, attempts + 10); // best for fast connection } else { - seeds = seedDB.seedsByAge(false, attempts); // best for seed list maintenance/cleaning + int diff = peerPingMinDBSize - amIAccessibleDB.size(); + if (diff > peerPingMinRunning) { + diff = Math.max(diff, peerPingMaxRunning); + if (attempts > diff) { attempts = diff; } + } else { + if (attempts > peerPingMinRunning) { attempts = peerPingMinRunning; } + } + seeds = seedDB.seedsByAge(false, attempts + 10); // best for seed list maintenance/cleaning } + if (seeds == null) { return 0; } + if (seeds.length < attempts) { attempts = seeds.length; } + + // This will try to get Peers that are not currently in amIAccessibleDB + int i = 0; + int j = seeds.length - 1; // attempts; + while (i < j) { + while ((i < j)&&(!amIAccessibleDB.containsKey(seeds[i].hash))) { + i++; + } + while ((i < j)&&(amIAccessibleDB.containsKey(seeds[j].hash))) { + j--; + } + if (i >= j) break; + yacySeed seed = seeds[i]; + seeds[i] = seeds[j]; + seeds[j] = seed; + i++; + j--; + } // include a YaCyNews record to my seed try { @@ -403,14 +440,15 @@ public class yacyCore { // include current citation-rank file count seedDB.mySeed.put(yacySeed.CRWCNT, Integer.toString(switchboard.rankingOwnDistribution.size())); seedDB.mySeed.put(yacySeed.CRTCNT, Integer.toString(switchboard.rankingOtherDistribution.size())); - + int newSeeds = -1; + //if (seeds.length > 1) { // holding a reference to all started threads int contactedSeedCount = 0; final List syncList = Collections.synchronizedList(new LinkedList()); // memory for threads final serverSemaphore sync = new serverSemaphore(attempts); // going through the peer list and starting a new publisher thread for each peer - for (int i = 0; i < seeds.length; i++) { + for (i = 0; i < attempts; i++) { if (seeds[i] == null) continue; final String address = seeds[i].getAddress(); @@ -418,23 +456,27 @@ public class yacyCore { if ((address == null) || (seeds[i].isProper() != null)) { // we don't like that address, delete it peerActions.peerDeparture(seeds[i]); - sync.V(); + sync.P(); } else { // starting a new publisher thread contactedSeedCount++; (new publishThread(yacyCore.publishThreadGroup,seeds[i],sync,syncList)).start(); } } + int reserveSeedsAt = i; // This is just a precaution // receiving the result of all started publisher threads - int newSeeds = -1; - for (int j=0; j < contactedSeedCount; j++) { + for (j = 0; j < contactedSeedCount; j++) { // waiting for the next thread to finish sync.P(); // if this is true something is wrong ... - if (syncList.isEmpty()) return 0; + if (syncList.isEmpty()) { + log.logWarning("PeerPing: syncList.isEmpty()==true"); + continue; + //return 0; + } // getting a reference to the finished thread final publishThread t = (publishThread) syncList.remove(0); @@ -449,6 +491,79 @@ public class yacyCore { } } + // Nobody contacted yet, try again until peerPingInitial attempts are through + i = reserveSeedsAt; // This is just a precaution + while ((newSeeds < 0) && (contactedSeedCount < peerPingInitial) && (i < seeds.length)) { + if (seeds[i] != null) { + final String address = seeds[i].getAddress(); + log.logFine("HELLO x" + i + " to peer '" + seeds[i].get(yacySeed.NAME, "") + "' at " + address); // debug + if ((address == null) || (seeds[i].isProper() != null)) { + peerActions.peerDeparture(seeds[i]); + } else { + contactedSeedCount++; + //new publishThread(yacyCore.publishThreadGroup,seeds[i],sync,syncList)).start(); + try { + newSeeds = yacyClient.publishMySeed(seeds[i].getAddress(), seeds[i].hash); + if (newSeeds < 0) { + log.logInfo("publish: disconnected " + seeds[i].get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + seeds[i].getName() + "' from " + seeds[i].getAddress()); + peerActions.peerDeparture(seeds[i]); + } else { + log.logInfo("publish: handshaked " + seeds[i].get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + seeds[i].getName() + "' at " + seeds[i].getAddress()); + } + } catch (Exception e) { + log.logSevere("publishMySeed: error with target seed " + seeds[i].toString() + ": " + e.getMessage(), e); + } + } + } + i++; + } + + int accessible = 0; + int notaccessible = 0; + final long cutofftime = System.currentTimeMillis() - peerPingMaxDBAge; + final int dbSize; + synchronized(amIAccessibleDB) { + dbSize = amIAccessibleDB.size(); + Iterator ai = amIAccessibleDB.keySet().iterator(); + while (ai.hasNext()) { + yacyAccessible ya = (yacyAccessible) amIAccessibleDB.get(ai.next()); + if (ya.lastUpdated < cutofftime) { + ai.remove(); + } else { + if (ya.IWasAccessed) { accessible++; } + else { notaccessible++; } + } + } + } + log.logFine("DBSize before -> after Cleanup: " + dbSize + " -> " + amIAccessibleDB.size()); + log.logInfo("PeerPing: I am accessible for " + accessible + + "peer(s), not accessible for " + notaccessible + " peer(s)."); + + if ((accessible + notaccessible) > 0) { + final String newPeerType; + // At least one other Peer told us our type + if ((accessible >= peerPingMinAccessibleToForceSenior) || + (accessible >= notaccessible)) { + // We can be reached from a majority of other Peers + if (yacyCore.seedDB.mySeed.isPrincipal()) { + newPeerType = yacySeed.PEERTYPE_PRINCIPAL; + } else { + newPeerType = yacySeed.PEERTYPE_SENIOR; + } + } else { + // We cannot be reached from the outside + newPeerType = yacySeed.PEERTYPE_JUNIOR; + } + if (yacyCore.seedDB.mySeed.orVirgin().equals(newPeerType)) { + log.logInfo("PeerPing: myType is " + yacyCore.seedDB.mySeed.orVirgin()); + } else { + log.logInfo("PeerPing: changing myType from '" + yacyCore.seedDB.mySeed.orVirgin() + "' to '" + newPeerType + "'"); + yacyCore.seedDB.mySeed.put(yacySeed.PEERTYPE, newPeerType); + } + } else { + log.logInfo("PeerPing: No data, staying at myType: " + yacyCore.seedDB.mySeed.orVirgin()); + } + if (newSeeds >= 0) { // success! we have published our peer to a senior peer // update latest news from the other peer diff --git a/source/de/anomic/yacy/yacySeed.java b/source/de/anomic/yacy/yacySeed.java index 434e37882..9656a65a2 100644 --- a/source/de/anomic/yacy/yacySeed.java +++ b/source/de/anomic/yacy/yacySeed.java @@ -195,6 +195,11 @@ public class yacySeed { * @return the peertype or null */ public String getPeerType() { return get(PEERTYPE, ""); } + /** + * try to get the peertype
+ * @return the peertype or "Virgin" + */ + public String orVirgin() { return get(PEERTYPE, PEERTYPE_VIRGIN); } /** * try to get the peertype
* @return the peertype or "junior"