From fcc11391a86cfdc851d75fd5dea07961206bba2a Mon Sep 17 00:00:00 2001 From: orbiter Date: Fri, 2 Feb 2007 13:12:31 +0000 Subject: [PATCH] some redesign attempts because sorting of lastseen does not work correctly not finished yet target: better selection of peer-ping targets, which should enhance stabilization of the net git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3319 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/Network.java | 2 +- .../kelondro/kelondroMScoreCluster.java | 19 +++++-- .../anomic/kelondro/kelondroMapObjects.java | 53 +++++++++++++------ .../de/anomic/kelondro/kelondroMapTable.java | 2 +- source/de/anomic/server/serverCore.java | 6 +-- source/de/anomic/yacy/yacySeed.java | 4 ++ source/de/anomic/yacy/yacySeedDB.java | 27 +++++++++- source/yacy.java | 2 +- 8 files changed, 87 insertions(+), 28 deletions(-) diff --git a/htroot/Network.java b/htroot/Network.java index 03e239731..35f679c8b 100644 --- a/htroot/Network.java +++ b/htroot/Network.java @@ -431,7 +431,7 @@ public class Network { prop.put(STR_TABLE_LIST + conCount + "_dhtreceive_peertags", ((peertags == null) || (peertags.length() == 0)) ? "no tags given" : ("tags = " + peertags)); } prop.put(STR_TABLE_LIST + conCount + "_version", yacy.combined2prettyVersion(seed.get(yacySeed.VERSION, "0.1"), shortname)); - prop.put(STR_TABLE_LIST + conCount + "_lastSeen", lastseen); + prop.put(STR_TABLE_LIST + conCount + "_lastSeen", seed.getLastSeenString() + " " + lastseen); prop.put(STR_TABLE_LIST + conCount + "_utc", seed.get(yacySeed.UTC, "-")); prop.put(STR_TABLE_LIST + conCount + "_uptime", serverDate.intervalToString(60000 * Long.parseLong(seed.get(yacySeed.UPTIME, "0")))); prop.put(STR_TABLE_LIST + conCount + "_links", groupDigits(seed.get(yacySeed.LCOUNT, "0"))); diff --git a/source/de/anomic/kelondro/kelondroMScoreCluster.java b/source/de/anomic/kelondro/kelondroMScoreCluster.java index 769a52684..84aa3fbb5 100644 --- a/source/de/anomic/kelondro/kelondroMScoreCluster.java +++ b/source/de/anomic/kelondro/kelondroMScoreCluster.java @@ -73,7 +73,20 @@ public final class kelondroMScoreCluster { } catch (ParseException e) {} } - public static int string2score(String s) { + public static int object2score(Object o) { + if (o instanceof Integer) return ((Integer) o).intValue(); + if (o instanceof Long) { + long l = ((Long) o).longValue(); + if (l < (long) Integer.MAX_VALUE) return (int) l; + o = ((Long) o).toString(); + } + if (o instanceof Double) { + double d = 1000d * ((Double) o).doubleValue(); + return (int) Math.round(d); + } + String s = ""; + if (o instanceof String) s = (String) o; + // this can be used to calculate a score from a string if ((s == null) || (s.length() == 0) || (s.charAt(0) == '-')) return 0; try { @@ -410,9 +423,9 @@ public final class kelondroMScoreCluster { public static void main(String[] args) { String t = "ZZZZZZZZZZ"; - System.out.println("score of " + t + ": " + string2score(t)); + System.out.println("score of " + t + ": " + object2score(t)); if (args.length > 0) { - System.out.println("score of " + args[0] + ": " + string2score(args[0])); + System.out.println("score of " + args[0] + ": " + object2score(args[0])); System.exit(0); } diff --git a/source/de/anomic/kelondro/kelondroMapObjects.java b/source/de/anomic/kelondro/kelondroMapObjects.java index 0567f0712..80f263808 100644 --- a/source/de/anomic/kelondro/kelondroMapObjects.java +++ b/source/de/anomic/kelondro/kelondroMapObjects.java @@ -28,6 +28,8 @@ package de.anomic.kelondro; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.HashMap; import java.util.Iterator; import java.util.Map; @@ -40,10 +42,10 @@ public class kelondroMapObjects extends kelondroObjects { private int elementCount; public kelondroMapObjects(kelondroDyn dyn, int cachesize) { - this(dyn, cachesize, null, null, null); + this(dyn, cachesize, null, null, null, null, null); } - public kelondroMapObjects(kelondroDyn dyn, int cachesize, String[] sortfields, String[] longaccfields, String[] doubleaccfields) { + public kelondroMapObjects(kelondroDyn dyn, int cachesize, String[] sortfields, String[] longaccfields, String[] doubleaccfields, Method externalInitializer, Object externalHandler) { super(dyn, cachesize); // create fast ordering clusters and acc fields @@ -81,35 +83,52 @@ public class kelondroMapObjects extends kelondroObjects { // fill cluster and accumulator with values if ((sortfields != null) || (longaccfields != null) || (doubleaccfields != null)) try { kelondroDyn.dynKeyIterator it = dyn.dynKeys(true, false); - String key, value; + String mapname; + Object cell; long valuel; double valued; Map map; while (it.hasNext()) { - key = (String) it.next(); - map = getMap(key); + mapname = (String) it.next(); + map = getMap(mapname); if (map == null) break; if (sortfields != null) for (int i = 0; i < sortfields.length; i++) { - value = (String) map.get(sortfields[i]); - if (value != null) cluster[i].setScore(key, kelondroMScoreCluster.string2score(value)); + cell = map.get(sortfields[i]); + if (cell != null) cluster[i].setScore(mapname, kelondroMScoreCluster.object2score(cell)); } if (longaccfields != null) for (int i = 0; i < longaccfields.length; i++) { - value = (String) map.get(longaccfields[i]); - if (value != null) try { - valuel = Long.parseLong(value); + cell = map.get(longaccfields[i]); + valuel = 0; + if (cell != null) try { + if (cell instanceof Long) valuel = ((Long) cell).longValue(); + if (cell instanceof String) valuel = Long.parseLong((String) cell); longaccumulator[i] = new Long(longaccumulator[i].longValue() + valuel); } catch (NumberFormatException e) {} } if (doubleaccfields != null) for (int i = 0; i < doubleaccfields.length; i++) { - value = (String) map.get(doubleaccfields[i]); - if (value != null) try { - valued = Double.parseDouble(value); + cell = map.get(doubleaccfields[i]); + valued = 0d; + if (cell != null) try { + if (cell instanceof Double) valued = ((Double) cell).doubleValue(); + if (cell instanceof String) valued = Double.parseDouble((String) cell); doubleaccumulator[i] = new Double(doubleaccumulator[i].doubleValue() + valued); } catch (NumberFormatException e) {} } + + if ((externalHandler != null) && (externalInitializer != null)) { + try { + externalInitializer.invoke(externalHandler, new Object[]{mapname, map}); + } catch (IllegalArgumentException e) { + e.printStackTrace(); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } catch (InvocationTargetException e) { + e.printStackTrace(); + } + } elementCount++; } } catch (IOException e) {} @@ -185,13 +204,13 @@ public class kelondroMapObjects extends kelondroObjects { } private void updateSortCluster(final String key, final Map map) { - String value; + Object cell; kelondroMScoreCluster cluster; for (int i = 0; i < sortfields.length; i++) { - value = (String) map.get(sortfields[i]); - if (value != null) { + cell = map.get(sortfields[i]); + if (cell != null) { cluster = (kelondroMScoreCluster) sortClusterMap.get(sortfields[i]); - cluster.setScore(key, kelondroMScoreCluster.string2score(value)); + cluster.setScore(key, kelondroMScoreCluster.object2score(cell)); sortClusterMap.put(sortfields[i], cluster); } } diff --git a/source/de/anomic/kelondro/kelondroMapTable.java b/source/de/anomic/kelondro/kelondroMapTable.java index f69482a4b..8233fa90f 100644 --- a/source/de/anomic/kelondro/kelondroMapTable.java +++ b/source/de/anomic/kelondro/kelondroMapTable.java @@ -83,7 +83,7 @@ public class kelondroMapTable { kelondroDyn dyn; if (!(tablefile.exists())) tablefile.getParentFile().mkdirs(); dyn = new kelondroDyn(tablefile, buffersize, preloadTime, keysize, nodesize, fillChar, true, false); - kelondroMapObjects map = new kelondroMapObjects(dyn, cacheslots, sortfields, longaccfields, doubleaccfields); + kelondroMapObjects map = new kelondroMapObjects(dyn, cacheslots, sortfields, longaccfields, doubleaccfields, null, null); mTables.put(tablename, map); } diff --git a/source/de/anomic/server/serverCore.java b/source/de/anomic/server/serverCore.java index ecd367902..a9225b3da 100644 --- a/source/de/anomic/server/serverCore.java +++ b/source/de/anomic/server/serverCore.java @@ -95,6 +95,9 @@ public final class serverCore extends serverAbstractThread implements serverThre public static final byte lf = 10; public static final byte[] crlf = {cr, lf}; public static final String crlfString = new String(crlf); + public static final Class[] stringType = {"".getClass()}; // set up some reflection + //Class[] exceptionType = {Class.forName("java.lang.Throwable")}; + // static variables public static final Boolean TERMINATE_CONNECTION = Boolean.FALSE; @@ -1138,9 +1141,6 @@ public final class serverCore extends serverAbstractThread implements serverThre private void listen() { try { - // set up some reflection - Class[] stringType = {"".getClass()}; - //Class[] exceptionType = {Class.forName("java.lang.Throwable")}; Object result; // // send greeting diff --git a/source/de/anomic/yacy/yacySeed.java b/source/de/anomic/yacy/yacySeed.java index 2f88531c5..689d229cd 100644 --- a/source/de/anomic/yacy/yacySeed.java +++ b/source/de/anomic/yacy/yacySeed.java @@ -399,6 +399,10 @@ public class yacySeed { return System.currentTimeMillis(); } } + + public final String getLastSeenString() { + return get(yacySeed.LASTSEEN, "20040101000000"); + } public final int getAge() { // returns the age as number of days diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index 7d7bd0f2c..871b0bf0f 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -49,6 +49,7 @@ import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; import java.lang.ref.SoftReference; +import java.lang.reflect.Method; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; @@ -206,12 +207,29 @@ public final class yacySeedDB { private synchronized kelondroMapObjects openSeedTable(File seedDBFile) { new File(seedDBFile.getParent()).mkdirs(); + Class[] args; try { - return new kelondroMapObjects(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, longaccFields, doubleaccFields); + args = new Class[]{"".getClass(), Class.forName("java.util.Map")}; + } catch (ClassNotFoundException e2){ + e2.printStackTrace(); + args = null; + } + Method initializeHandlerMethod; + try { + initializeHandlerMethod = this.getClass().getMethod("initializeHandler", args); + } catch (SecurityException e1) { + e1.printStackTrace(); + initializeHandlerMethod = null; + } catch (NoSuchMethodException e1) { + e1.printStackTrace(); + initializeHandlerMethod = null; + } + try { + return new kelondroMapObjects(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this); } catch (Exception e) { seedDBFile.delete(); // try again - return new kelondroMapObjects(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, longaccFields, doubleaccFields); + return new kelondroMapObjects(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this); } } @@ -243,6 +261,11 @@ public final class yacySeedDB { } } + public void initializeHandler(String mapname, Map map) { + // this is used to set up a lastSeen lookup table + + } + public Enumeration seedsSortedConnected(boolean up, String field) { // enumerates seed-type objects: all seeds sequentially ordered by field return new seedEnum(up, field, seedActiveDB); diff --git a/source/yacy.java b/source/yacy.java index c568310c9..d062542db 100644 --- a/source/yacy.java +++ b/source/yacy.java @@ -1327,7 +1327,7 @@ public final class yacy { String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"}; for (int i=0; i < dbFileNames.length; i++) { File dbFile = new File(yacyDBPath,dbFileNames[i]); - kelondroMapObjects db = new kelondroMapObjects(new kelondroDyn(dbFile, (1024 * 0x400) / 3, 3000, yacySeedDB.commonHashLength, 480, '#', true, false), 500, yacySeedDB.sortFields, yacySeedDB.longaccFields, yacySeedDB.doubleaccFields); + kelondroMapObjects db = new kelondroMapObjects(new kelondroDyn(dbFile, (1024 * 0x400) / 3, 3000, yacySeedDB.commonHashLength, 480, '#', true, false), 500, yacySeedDB.sortFields, yacySeedDB.longaccFields, yacySeedDB.doubleaccFields, null, null); kelondroMapObjects.mapIterator it; it = db.maps(true, false);