From b33cef421ee5698af52573ee131acbc45be13dce Mon Sep 17 00:00:00 2001 From: orbiter Date: Sun, 29 Apr 2007 00:08:38 +0000 Subject: [PATCH] better routing for public clusters git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3620 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/yacy/hello.java | 16 ++++++---- .../de/anomic/soap/services/ShareService.java | 2 +- source/de/anomic/yacy/yacyClient.java | 4 +-- source/de/anomic/yacy/yacyCore.java | 30 ++++++++++++------ source/de/anomic/yacy/yacySearch.java | 2 +- source/de/anomic/yacy/yacySeedDB.java | 31 ++++++++++++------- 6 files changed, 55 insertions(+), 30 deletions(-) diff --git a/htroot/yacy/hello.java b/htroot/yacy/hello.java index e40f2d2f9..2f8980740 100644 --- a/htroot/yacy/hello.java +++ b/htroot/yacy/hello.java @@ -47,6 +47,8 @@ // if the shell's current path is HTROOT import java.util.Date; +import java.util.Iterator; +import java.util.Map; import de.anomic.http.httpHeader; import de.anomic.plasma.plasmaSwitchboard; @@ -74,7 +76,6 @@ public final class hello { final String key = post.get("key", ""); // transmission key for response final String seed = post.get("seed", ""); final String countStr = post.get("count", "0"); - int i; int count = 0; try {count = (countStr == null) ? 0 : Integer.parseInt(countStr);} catch (NumberFormatException e) {count = 0;} // final Date remoteTime = yacyCore.parseUniversalDate((String) post.get(MYTIME)); // read remote time @@ -173,7 +174,7 @@ public final class hello { if (count > 100) { count = 100; } // latest seeds - final yacySeed[] ySeeds = yacyCore.seedDB.seedsByAge(true, count); + final Map ySeeds = yacyCore.seedDB.seedsByAge(true, count); // peerhash/yacySeed relation // attach also my own seed seeds.append("seed0=").append(yacyCore.seedDB.mySeed.genSeedStr(key)).append(serverCore.crlfString); @@ -181,10 +182,13 @@ public final class hello { // attach other seeds if (ySeeds != null) { - seeds.ensureCapacity((ySeeds.length + 1) * 768); - for (i = 0; i < ySeeds.length; i++) { - if ((ySeeds[i] != null) && (ySeeds[i].isProper() == null)) { - seeds.append("seed").append(count).append('=').append(ySeeds[i].genSeedStr(key)).append(serverCore.crlfString); + seeds.ensureCapacity((ySeeds.size() + 1) * 768); + Iterator si = ySeeds.values().iterator(); + yacySeed s; + while (si.hasNext()) { + s = (yacySeed) si.next(); + if ((s != null) && (s.isProper() == null)) { + seeds.append("seed").append(count).append('=').append(s.genSeedStr(key)).append(serverCore.crlfString); count++; } } diff --git a/source/de/anomic/soap/services/ShareService.java b/source/de/anomic/soap/services/ShareService.java index 420fcd209..ff68a63d4 100644 --- a/source/de/anomic/soap/services/ShareService.java +++ b/source/de/anomic/soap/services/ShareService.java @@ -78,7 +78,7 @@ public class ShareService extends AbstractService { private static final int FILEINFO_COMMENT = 1; private static final int GENMD5_MD5_ARRAY = 0; - private static final int GENMD5_MD5_STRING = 1; + //private static final int GENMD5_MD5_STRING = 1; /* ===================================================================== * Used XML Templates diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index 39c463e62..82a57a591 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -410,7 +410,7 @@ public final class yacyClient { (yacyCore.seedDB.sb.remoteProxyConfig.useProxy4Yacy()); // building url - final String url = "http://" + targetPeer.getAddress() + "/yacy/search.html"; + final String url = "http://" + ((targetPeer.getIP().equals(yacyCore.seedDB.mySeed.getIP())) ? "localhost:" + targetPeer.getPort() : targetPeer.getAddress()) + "/yacy/search.html"; // adding all needed parameters /* @@ -421,7 +421,7 @@ public final class yacyClient { "&count=" + count + "&resource=" + ((global) ? "global" : "local") + "&query=" + wordhashes; */ - final serverObjects obj = new serverObjects(9); + final serverObjects obj = new serverObjects(20); long duetime = timingProfile.duetime(); obj.put("myseed", yacyCore.seedDB.mySeed.genSeedStr(key)); obj.put("youare", targetPeer.hash); diff --git a/source/de/anomic/yacy/yacyCore.java b/source/de/anomic/yacy/yacyCore.java index ff9ac87d5..c66c1d353 100644 --- a/source/de/anomic/yacy/yacyCore.java +++ b/source/de/anomic/yacy/yacyCore.java @@ -423,7 +423,7 @@ public class yacyCore { // probed until we get a valid response. // init yacyHello-process - yacySeed[] seeds; + Map seeds; // hash/yacySeed relation int attempts = seedDB.sizeConnected(); @@ -431,6 +431,16 @@ public class yacyCore { if (seedDB.mySeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN).equals(yacySeed.PEERTYPE_VIRGIN)) { if (attempts > peerPingInitial) { attempts = peerPingInitial; } seeds = seedDB.seedsByAge(true, attempts + 10); // best for fast connection + // add also all peers from cluster if this is a public robinson cluster + if (plasmaSwitchboard.getSwitchboard().clusterhashes != null) { + Iterator i = plasmaSwitchboard.getSwitchboard().clusterhashes.iterator(); + String hash; + while (i.hasNext()) { + hash = (String) i.next(); + if (seeds.containsKey(hash)) continue; + seeds.put(hash, seedDB.get(hash)); + } + } } else { int diff = peerPingMinDBSize - amIAccessibleDB.size(); if (diff > peerPingMinRunning) { @@ -447,13 +457,15 @@ public class yacyCore { // This will try to get Peers that are not currently in amIAccessibleDB LinkedList seedList = new LinkedList(); LinkedList tmpSeedList = new LinkedList(); - for(int i = 0; i < seeds.length; i++) { - if (seeds[i] != null) { - if (amIAccessibleDB.containsKey(seeds[i].hash)) { - tmpSeedList.add(seeds[i]); + Iterator si = seeds.values().iterator(); + yacySeed seed; + while (si.hasNext()) { + seed = (yacySeed) si.next(); + if (seed == null) continue; + if (amIAccessibleDB.containsKey(seed.hash)) { + tmpSeedList.add(seed); } else { - seedList.add(seeds[i]); - } + seedList.add(seed); } } while (!tmpSeedList.isEmpty()) { seedList.add(tmpSeedList.remove(0)); } @@ -484,7 +496,7 @@ public class yacyCore { // going through the peer list and starting a new publisher thread for each peer for (int i = 0; i < attempts; i++) { - yacySeed seed = (yacySeed) seedList.remove(0); + seed = (yacySeed) seedList.remove(0); if (seed == null) continue; final String address = seed.getAddress(); @@ -528,7 +540,7 @@ public class yacyCore { // Nobody contacted yet, try again until peerPingInitial attempts are through while ((newSeeds < 0) && (contactedSeedCount < peerPingInitial) && (!seedList.isEmpty())) { - yacySeed seed = (yacySeed) seedList.remove(0); + seed = (yacySeed) seedList.remove(0); if (seed != null) { final String address = seed.getAddress(); log.logFine("HELLO x" + contactedSeedCount + " to peer '" + seed.get(yacySeed.NAME, "") + "' at " + address); // debug diff --git a/source/de/anomic/yacy/yacySearch.java b/source/de/anomic/yacy/yacySearch.java index 8c5b57cb4..3771fd520 100644 --- a/source/de/anomic/yacy/yacySearch.java +++ b/source/de/anomic/yacy/yacySearch.java @@ -152,7 +152,7 @@ public class yacySearch extends Thread { ArrayList l = new ArrayList(); yacySeed s; while (i.hasNext()) { - s = yacyCore.seedDB.getConnected((String) i.next()); + s = yacyCore.seedDB.get((String) i.next()); // should be getConnected; get only during testing time if (s != null) l.add(s); } yacySeed[] result = new yacySeed[l.size()]; diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index 3677a0c70..9faf38bf8 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -54,6 +54,7 @@ import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Enumeration; +import java.util.HashMap; import java.util.Hashtable; import java.util.Iterator; import java.util.Map; @@ -315,7 +316,9 @@ public final class yacySeedDB { return (yacySeed) e.nextElement(); } - public yacySeed[] seedsByAge(boolean up, int count) { + public HashMap seedsByAge(boolean up, int count) { + // returns a peerhash/yacySeed relation + if (count > sizeConnected()) count = sizeConnected(); // fill a score object @@ -335,10 +338,14 @@ public final class yacySeedDB { } // result is now in the score object; create a result vector - yacySeed[] result = new yacySeed[count]; + HashMap result = new HashMap(); Iterator it = seedScore.scores(up); int c = 0; - while ((c < count) && (it.hasNext())) result[c++] = getConnected((String) it.next()); + while ((c < count) && (it.hasNext())) { + c++; + ys = getConnected((String) it.next()); + result.put(ys.hash, ys); + } return result; } catch (kelondroException e) { seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile); @@ -535,15 +542,17 @@ public final class yacySeedDB { if (seed != null) return seed; // enumerate the cache and simultanous insert values - Enumeration e = seedsConnected(true, false, null, (float) 0.0); String name; - while (e.hasMoreElements()) { - seed = (yacySeed) e.nextElement(); - if (seed != null) { - name = seed.getName().toLowerCase(); - if (seed.isProper() == null) nameLookupCache.put(name, seed); - if (name.equals(peerName)) return seed; - } + for (int table = 0; table < 2; table++) { + Enumeration e = (table == 0) ? seedsConnected(true, false, null, (float) 0.0) : seedsDisconnected(true, false, null, (float) 0.0); + while (e.hasMoreElements()) { + seed = (yacySeed) e.nextElement(); + if (seed != null) { + name = seed.getName().toLowerCase(); + if (seed.isProper() == null) nameLookupCache.put(name, seed); + if (name.equals(peerName)) return seed; + } + } } // check local seed name = mySeed.getName().toLowerCase();