diff --git a/htroot/rct_p.java b/htroot/rct_p.java index ce8ed8835..32147db53 100644 --- a/htroot/rct_p.java +++ b/htroot/rct_p.java @@ -39,9 +39,9 @@ import de.anomic.server.serverSwitch; import de.anomic.xml.RSSFeed; import de.anomic.xml.RSSMessage; import de.anomic.yacy.yacyClient; -import de.anomic.yacy.yacyPeerSelection; import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacyURL; +import de.anomic.yacy.dht.PeerSelection; public class rct_p { @@ -116,7 +116,7 @@ public class rct_p { yacySeed seed; int hc = 0; if (sb.webIndex.seedDB != null && sb.webIndex.seedDB.sizeConnected() > 0) { - final Iterator e = yacyPeerSelection.getProvidesRemoteCrawlURLs(sb.webIndex.seedDB); + final Iterator e = PeerSelection.getProvidesRemoteCrawlURLs(sb.webIndex.seedDB); while (e.hasNext()) { seed = e.next(); if (seed != null) { diff --git a/htroot/yacy/hello.java b/htroot/yacy/hello.java index 75b521060..824410fb6 100644 --- a/htroot/yacy/hello.java +++ b/htroot/yacy/hello.java @@ -41,9 +41,9 @@ import de.anomic.server.serverSwitch; import de.anomic.yacy.yacyClient; import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacyNetwork; -import de.anomic.yacy.yacyPeerSelection; import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacyVersion; +import de.anomic.yacy.dht.PeerSelection; public final class hello { @@ -184,7 +184,7 @@ public final class hello { if (count > 100) { count = 100; } // latest seeds - final Map ySeeds = yacyPeerSelection.seedsByAge(sb.webIndex.seedDB, true, count); // peerhash/yacySeed relation + final Map ySeeds = PeerSelection.seedsByAge(sb.webIndex.seedDB, true, count); // peerhash/yacySeed relation // attach also my own seed seeds.append("seed0=").append(sb.webIndex.seedDB.mySeed().genSeedStr(key)).append(serverCore.CRLF_STRING); diff --git a/source/de/anomic/crawler/CrawlQueues.java b/source/de/anomic/crawler/CrawlQueues.java index cea4b32bb..737d28567 100644 --- a/source/de/anomic/crawler/CrawlQueues.java +++ b/source/de/anomic/crawler/CrawlQueues.java @@ -47,9 +47,9 @@ import de.anomic.server.serverProcessorJob; import de.anomic.xml.RSSFeed; import de.anomic.xml.RSSMessage; import de.anomic.yacy.yacyClient; -import de.anomic.yacy.yacyPeerSelection; import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacyURL; +import de.anomic.yacy.dht.PeerSelection; public class CrawlQueues { @@ -335,7 +335,7 @@ public class CrawlQueues { yacySeed seed; if (remoteCrawlProviderHashes.size() == 0) { if (sb.webIndex.seedDB != null && sb.webIndex.seedDB.sizeConnected() > 0) { - final Iterator e = yacyPeerSelection.getProvidesRemoteCrawlURLs(sb.webIndex.seedDB); + final Iterator e = PeerSelection.getProvidesRemoteCrawlURLs(sb.webIndex.seedDB); while (e.hasNext()) { seed = e.next(); if (seed != null) { diff --git a/source/de/anomic/index/indexRAMRI.java b/source/de/anomic/index/indexRAMRI.java index 6bf4e61a9..b32763fe3 100644 --- a/source/de/anomic/index/indexRAMRI.java +++ b/source/de/anomic/index/indexRAMRI.java @@ -138,6 +138,7 @@ public final class indexRAMRI implements indexRI, indexRIReader, Iterable e = yacyPeerSelection.getAcceptRemoteIndexSeeds(sb.webIndex.seedDB, startHash, sb.webIndex.seedDB.sizeConnected(), true); + final Iterator e = PeerSelection.getAcceptRemoteIndexSeeds(sb.webIndex.seedDB, startHash, sb.webIndex.seedDB.sizeConnected(), true); while (e.hasNext()) { seed = e.next(); if (seed != null) { diff --git a/source/de/anomic/yacy/dht/Transmission.java b/source/de/anomic/yacy/dht/Transmission.java index d0737ca5c..359e3f959 100644 --- a/source/de/anomic/yacy/dht/Transmission.java +++ b/source/de/anomic/yacy/dht/Transmission.java @@ -194,7 +194,7 @@ public class Transmission { log.logInfo("Transfer of chunk to myself-target"); return true; } - log.logInfo("starting new index transmission thread " + this.primaryTarget); + log.logInfo("starting new index transmission request to " + this.primaryTarget); long start = System.currentTimeMillis(); final HashMap ohm = yacyClient.transferIndex(target, this.containers, this.references, gzipBody4Transfer, timeout4Transfer); final String result = (String) ohm.get("result"); @@ -219,7 +219,9 @@ public class Transmission { return true; } this.miss++; - seeds.peerActions.peerDeparture(target, "did not accept index"); + // write information that peer does not receive index transmissions + target.setFlagAcceptRemoteIndex(false); + seeds.update(target.hash, target); log.logInfo("Transfer failed of chunk to target " + target.hash + "/" + target.getName()); return false; } diff --git a/source/de/anomic/yacy/yacyCore.java b/source/de/anomic/yacy/yacyCore.java index d7300303b..cb0eb7ae0 100644 --- a/source/de/anomic/yacy/yacyCore.java +++ b/source/de/anomic/yacy/yacyCore.java @@ -58,6 +58,7 @@ import de.anomic.server.serverCore; import de.anomic.server.serverSemaphore; import de.anomic.xml.RSSFeed; import de.anomic.xml.RSSMessage; +import de.anomic.yacy.dht.PeerSelection; public class yacyCore { @@ -305,7 +306,7 @@ public class yacyCore { if (sb.webIndex.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN).equals(yacySeed.PEERTYPE_VIRGIN)) { if (attempts > PING_INITIAL) { attempts = PING_INITIAL; } final Map ch = plasmaSwitchboard.getSwitchboard().clusterhashes; - seeds = yacyPeerSelection.seedsByAge(sb.webIndex.seedDB, true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection + seeds = PeerSelection.seedsByAge(sb.webIndex.seedDB, true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection // add also all peers from cluster if this is a public robinson cluster if (ch != null) { final Iterator> i = ch.entrySet().iterator(); @@ -332,7 +333,7 @@ public class yacyCore { } else { if (attempts > PING_MIN_RUNNING) { attempts = PING_MIN_RUNNING; } } - seeds = yacyPeerSelection.seedsByAge(sb.webIndex.seedDB, false, attempts); // best for seed list maintenance/cleaning + seeds = PeerSelection.seedsByAge(sb.webIndex.seedDB, false, attempts); // best for seed list maintenance/cleaning } if ((seeds == null) || seeds.size() == 0) { return 0; } diff --git a/source/de/anomic/yacy/yacyPeerSelection.java b/source/de/anomic/yacy/yacyPeerSelection.java deleted file mode 100644 index 9a74ae725..000000000 --- a/source/de/anomic/yacy/yacyPeerSelection.java +++ /dev/null @@ -1,249 +0,0 @@ -// yacyPeerSelection.java -// ------------------------------------- -// (C) by Michael Peter Christen; mc@yacy.net -// first published 05.11.2008 on http://yacy.net -// Frankfurt, Germany, 2008 -// -// $LastChangedDate: 2008-09-03 02:30:21 +0200 (Mi, 03 Sep 2008) $ -// $LastChangedRevision: 5102 $ -// $LastChangedBy: orbiter $ -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -package de.anomic.yacy; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; - -import de.anomic.kelondro.order.Base64Order; -import de.anomic.kelondro.order.DateFormatter; -import de.anomic.kelondro.util.ScoreCluster; -import de.anomic.kelondro.util.kelondroException; - - -/* - * this package is a collection of peer selection iterations that had been - * part of yacyPeerActions, yacyDHTActions and yacySeedDB - */ - -public class yacyPeerSelection { - - public static Iterator getAcceptRemoteIndexSeeds(yacySeedDB seedDB, final String starthash, int max, boolean alsoMyOwn) { - // returns an enumeration of yacySeed-Objects - // that have the AcceptRemoteIndex-Flag set - // the seeds are enumerated in the right order according DHT - return new acceptRemoteIndexSeedEnum(seedDB, starthash, Math.min(max, seedDB.sizeConnected()), alsoMyOwn); - } - - private static class acceptRemoteIndexSeedEnum implements Iterator { - - private Iterator se; - private yacySeed nextSeed; - private yacySeedDB seedDB; - private HashSet doublecheck; - private int remaining; - private boolean alsoMyOwn; - - public acceptRemoteIndexSeedEnum(yacySeedDB seedDB, final String starthash, int max, boolean alsoMyOwn) { - this.seedDB = seedDB; - this.se = getDHTSeeds(seedDB, starthash, yacyVersion.YACY_HANDLES_COLLECTION_INDEX); - this.remaining = max; - this.doublecheck = new HashSet(); - this.nextSeed = nextInternal(); - this.alsoMyOwn = alsoMyOwn && nextSeed != null && (Base64Order.enhancedCoder.compare(seedDB.mySeed().hash.getBytes(), nextSeed.hash.getBytes()) > 0); - } - - public boolean hasNext() { - return nextSeed != null; - } - - private yacySeed nextInternal() { - if (this.remaining <= 0) return null; - yacySeed s; - try { - while (se.hasNext()) { - s = se.next(); - if (s == null) return null; - if (doublecheck.contains(s.hash)) return null; - this.doublecheck.add(s.hash); - if (s.getFlagAcceptRemoteIndex()) { - this.remaining--; - return s; - } - } - } catch (final kelondroException e) { - System.out.println("DEBUG acceptRemoteIndexSeedEnum:" + e.getMessage()); - yacyCore.log.logSevere("database inconsistency (" + e.getMessage() + "), re-set of db."); - seedDB.resetActiveTable(); - return null; - } - return null; - } - - public yacySeed next() { - if (alsoMyOwn && Base64Order.enhancedCoder.compare(seedDB.mySeed().hash.getBytes(), nextSeed.hash.getBytes()) < 0) { - // take my own seed hash instead the enumeration result - alsoMyOwn = false; - return seedDB.mySeed(); - } else { - final yacySeed next = nextSeed; - nextSeed = nextInternal(); - return next; - } - } - - public void remove() { - throw new UnsupportedOperationException(); - } - - } - - public static Iterator getDHTSeeds(yacySeedDB seedDB, final String firstHash, final float minVersion) { - // enumerates seed-type objects: all seeds with starting point in the middle, rotating at the end/beginning - return new seedDHTEnum(seedDB, firstHash, minVersion); - } - - private static class seedDHTEnum implements Iterator { - - Iterator e1, e2; - int steps; - float minVersion; - yacySeedDB seedDB; - - public seedDHTEnum(yacySeedDB seedDB, final String firstHash, final float minVersion) { - this.seedDB = seedDB; - this.steps = seedDB.sizeConnected(); - this.minVersion = minVersion; - this.e1 = seedDB.seedsConnected(true, false, firstHash, minVersion); - this.e2 = null; - } - - public boolean hasNext() { - return (steps > 0) && ((e2 == null) || (e2.hasNext())); - } - - public yacySeed next() { - if (steps == 0) return null; - steps--; - - if (e1 == null || !e1.hasNext()) { - if (e2 == null) { - e1 = null; - e2 = seedDB.seedsConnected(true, false, null, minVersion); - } - return e2.next(); - } - - final yacySeed n = e1.next(); - if (!(e1.hasNext())) { - e1 = null; - e2 = seedDB.seedsConnected(true, false, null, minVersion); - } - return n; - } - - public void remove() { - throw new UnsupportedOperationException(); - } - } - - public static Iterator getProvidesRemoteCrawlURLs(yacySeedDB seedDB) { - return new providesRemoteCrawlURLsEnum(seedDB); - } - - private static class providesRemoteCrawlURLsEnum implements Iterator { - - Iterator se; - yacySeed nextSeed; - yacySeedDB seedDB; - - public providesRemoteCrawlURLsEnum(yacySeedDB seedDB) { - this.seedDB = seedDB; - se = getDHTSeeds(seedDB, null, yacyVersion.YACY_POVIDES_REMOTECRAWL_LISTS); - nextSeed = nextInternal(); - } - - public boolean hasNext() { - return nextSeed != null; - } - - private yacySeed nextInternal() { - yacySeed s; - try { - while (se.hasNext()) { - s = se.next(); - if (s == null) return null; - if (s.getLong(yacySeed.RCOUNT, 0) > 0) return s; - } - } catch (final kelondroException e) { - System.out.println("DEBUG providesRemoteCrawlURLsEnum:" + e.getMessage()); - yacyCore.log.logSevere("database inconsistency (" + e.getMessage() + "), re-set of db."); - seedDB.resetActiveTable(); - return null; - } - return null; - } - - public yacySeed next() { - final yacySeed next = nextSeed; - nextSeed = nextInternal(); - return next; - } - - public void remove() { - throw new UnsupportedOperationException(); - } - - } - - public static HashMap seedsByAge(yacySeedDB seedDB, final boolean up, int count) { - // returns a peerhash/yacySeed relation - // to get most recent peers, set up = true; for oldest peers, set up = false - - if (count > seedDB.sizeConnected()) count = seedDB.sizeConnected(); - - // fill a score object - final ScoreCluster seedScore = new ScoreCluster(); - yacySeed ys; - long absage; - final Iterator s = seedDB.seedsConnected(true, false, null, (float) 0.0); - int searchcount = 1000; - if (searchcount > seedDB.sizeConnected()) searchcount = seedDB.sizeConnected(); - try { - while ((s.hasNext()) && (searchcount-- > 0)) { - ys = s.next(); - if ((ys != null) && (ys.get(yacySeed.LASTSEEN, "").length() > 10)) try { - absage = Math.abs(System.currentTimeMillis() + DateFormatter.dayMillis - ys.getLastSeenUTC()); - seedScore.addScore(ys.hash, (int) absage); // the higher absage, the older is the peer - } catch (final Exception e) {} - } - - // result is now in the score object; create a result vector - final HashMap result = new HashMap(); - final Iterator it = seedScore.scores(up); - int c = 0; - while ((c < count) && (it.hasNext())) { - c++; - ys = seedDB.getConnected(it.next()); - if ((ys != null) && (ys.hash != null)) result.put(ys.hash, ys); - } - return result; - } catch (final kelondroException e) { - yacyCore.log.logSevere("Internal Error at yacySeedDB.seedsByAge: " + e.getMessage(), e); - return null; - } - } -}