From 8d827cdb30ad0a084665d5fa65aa8e9e74a7837c Mon Sep 17 00:00:00 2001 From: orbiter Date: Mon, 24 Oct 2005 14:07:43 +0000 Subject: [PATCH] tried to fix problems with order of network list by last-seen (which could also improve the network picture) git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@980 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/Network.html | 2 +- htroot/Network.java | 4 +- .../kelondro/kelondroMScoreCluster.java | 134 ++++++++++++++---- source/de/anomic/plasma/plasmaGrafics.java | 4 +- source/de/anomic/yacy/yacySeedDB.java | 74 +++++----- 5 files changed, 150 insertions(+), 68 deletions(-) diff --git a/htroot/Network.html b/htroot/Network.html index d1ab9c389..ed9444473 100644 --- a/htroot/Network.html +++ b/htroot/Network.html @@ -42,7 +42,7 @@ Name
  Info
  Release/
SVN
  -Last
Seen
  < > +Last
Seen
  < > UTC
Offset
  Uptime
  < > #Links
  < > diff --git a/htroot/Network.java b/htroot/Network.java index 14871ce5d..183782439 100644 --- a/htroot/Network.java +++ b/htroot/Network.java @@ -261,8 +261,8 @@ public class Network { Enumeration e = null; switch (page) { case 1 : e = yacyCore.seedDB.seedsSortedConnected(post.get("order", "down").equals("up"), post.get("sort", yacySeed.LCOUNT)); break; - case 2 : e = yacyCore.seedDB.seedsSortedDisconnected(post.get("order", "up").equals("up"), post.get("sort", yacySeed.LASTSEEN)); break; - case 3 : e = yacyCore.seedDB.seedsSortedPotential(post.get("order", "up").equals("up"), post.get("sort", yacySeed.LASTSEEN)); break; + case 2 : e = yacyCore.seedDB.seedsSortedDisconnected(post.get("order", "down").equals("up"), post.get("sort", yacySeed.LASTSEEN)); break; + case 3 : e = yacyCore.seedDB.seedsSortedPotential(post.get("order", "down").equals("up"), post.get("sort", yacySeed.LASTSEEN)); break; default: break; } String startURL; diff --git a/source/de/anomic/kelondro/kelondroMScoreCluster.java b/source/de/anomic/kelondro/kelondroMScoreCluster.java index 9208ba972..4f44e94d6 100644 --- a/source/de/anomic/kelondro/kelondroMScoreCluster.java +++ b/source/de/anomic/kelondro/kelondroMScoreCluster.java @@ -45,6 +45,8 @@ import java.text.SimpleDateFormat; import java.util.Iterator; import java.util.Map; import java.util.TreeMap; +import java.util.SortedMap; +import java.util.Random; // only for testing public final class kelondroMScoreCluster { @@ -60,27 +62,54 @@ public final class kelondroMScoreCluster { encnt = 0; } - public static SimpleDateFormat shortFormatter = new SimpleDateFormat("yyyyMMddHHmmss"); + public static final String shortDateFormatString = "yyyyMMddHHmmss"; + public static final SimpleDateFormat shortFormatter = new SimpleDateFormat(shortDateFormatString); + public static final long minutemillis = 60000; + public static long date2000 = 0; + + static { + try { + date2000 = shortFormatter.parse("20000101000000").getTime(); + } catch (ParseException e) {} + } + + /* + public static int string2score(String s) { + int i = string2scoreX(s); + System.out.println("string2core(" + s + ") = " + i); + return i; + } + */ public static int string2score(String s) { // this can be used to calculate a score from a string - try { // try a number - return Integer.parseInt(s); - } catch (NumberFormatException e) { - try { // try a date - return (int) ((90000000 + System.currentTimeMillis() - shortFormatter.parse(s).getTime()) / 60000); - } catch (ParseException ee) { - // try it lex - int len = s.length(); - if (len > 5) len = 5; - int c = 0; - for (int i = 0; i < len; i++) { - c <<= 6; - c += plainByteArray[(byte) s.charAt(i)]; - } - for (int i = len; i < 5; i++) c <<= 6; - return c; + + try { + long l = 0; + if (s.length() == shortDateFormatString.length()) { + // try a date + l = ((shortFormatter.parse(s).getTime() - date2000) / minutemillis); + if (l > (60 + ((System.currentTimeMillis() - date2000) / minutemillis))) l = 0; // future date, more than one hour + if (l < 0) l = 0; + } else { + // try a number + l = Long.parseLong(s); + } + // fix out-of-ranges + if (l > (long) Integer.MAX_VALUE) return Integer.MAX_VALUE; + if (l < (long) Integer.MIN_VALUE) return Integer.MIN_VALUE; + return (int) l; + } catch (Exception e) { + // try it lex + int len = s.length(); + if (len > 5) len = 5; + int c = 0; + for (int i = 0; i < len; i++) { + c <<= 6; + c += plainByteArray[(byte) s.charAt(i)]; } + for (int i = len; i < 5; i++) c <<= 6; + return c; } } @@ -263,13 +292,15 @@ public final class kelondroMScoreCluster { public synchronized Iterator scores(boolean up) { if (up) return new simpleScoreIterator(); - else return scores(false, Integer.MIN_VALUE, Integer.MAX_VALUE); + //else return scores(false, Integer.MIN_VALUE, Integer.MAX_VALUE); + else return new reverseScoreIterator(); } public synchronized Iterator scores(boolean up, int minScore, int maxScore) { return new komplexScoreIterator(up, minScore, maxScore); } + private class komplexScoreIterator implements Iterator { boolean up; @@ -318,6 +349,34 @@ public final class kelondroMScoreCluster { } + private class reverseScoreIterator implements Iterator { + + SortedMap view; + Object key; + + public reverseScoreIterator() { + view = keyrefDB; + } + + public boolean hasNext() { + return view.size() > 0; + } + + public Object next() { + key = view.lastKey(); + view = view.headMap(key); + Object value = keyrefDB.get(key); + //System.out.println("cluster reverse iterator: score = " + ((((Long) key).longValue() & 0xFFFFFFFF00000000L) >> 32) + ", handle = " + (((Long) key).longValue() & 0xFFFFFFFFL) + ", value = " + value); + return value; + } + + public void remove() { + Object val = keyrefDB.remove(key); + if (val != null) refkeyDB.remove(val); + } + + } + private class simpleScoreIterator implements Iterator { Iterator ii; @@ -333,11 +392,13 @@ public final class kelondroMScoreCluster { public Object next() { entry = (Map.Entry) ii.next(); + //System.out.println("cluster simple iterator: score = " + ((((Long) entry.getKey()).longValue() & 0xFFFFFFFF00000000L) >> 32) + ", handle = " + (((Long) entry.getKey()).longValue() & 0xFFFFFFFFL) + ", value = " + entry.getValue()); return entry.getValue(); } public void remove() { ii.remove(); + if (entry.getValue() != null) refkeyDB.remove(entry.getValue()); } } @@ -349,25 +410,46 @@ public final class kelondroMScoreCluster { System.out.println("Test for Score: start"); kelondroMScoreCluster s = new kelondroMScoreCluster(); - int c = 0; + long c = 0; // create cluster long time = System.currentTimeMillis(); - for (int i = 0; i < 10000; i++) { - s.addScore("score#" + i + "xxx" + i + "xxx" + i + "xxx" + i + "xxx", i/10); - c += i/10; - } - /* + Random random = new Random(1234); + int r; + int count = 20; + int[] mem = new int[count]; + + for (int x = 0; x < 100; x++) { + for (int i = 0; i < count; i++) { + r = random.nextInt(); + mem[i] = r; + s.addScore("score#" + r, r); + c += (long) r; + } + + // delete some + int p; + for (int i = 0; i < (count / 2); i++) { + p = (int) (random.nextFloat() * count); + if (s.existsScore("score#" + mem[p])) { + System.out.println("delete score#" + mem[p]); + s.deleteScore("score#" + mem[p]); + c -= mem[p]; + } + } + } + System.out.println("result:"); Object[] result; result = s.getScores(s.size(), true); for (int i = 0; i < s.size(); i++) System.out.println("up: " + result[i]); result = s.getScores(s.size(), false); for (int i = 0; i < s.size(); i++) System.out.println("down: " + result[i]); - */ + System.out.println("finished create. time = " + (System.currentTimeMillis() - time)); System.out.println("total=" + s.totalCount() + ", elements=" + s.size() + ", redundant count=" + c); + /* // delete cluster time = System.currentTimeMillis(); for (int i = 0; i < 10000; i++) { @@ -376,6 +458,6 @@ public final class kelondroMScoreCluster { } System.out.println("finished delete. time = " + (System.currentTimeMillis() - time)); System.out.println("total=" + s.totalCount() + ", elements=" + s.size() + ", redundant count=" + c); - + */ } } diff --git a/source/de/anomic/plasma/plasmaGrafics.java b/source/de/anomic/plasma/plasmaGrafics.java index 259a8cfa0..14ec02d7a 100644 --- a/source/de/anomic/plasma/plasmaGrafics.java +++ b/source/de/anomic/plasma/plasmaGrafics.java @@ -153,7 +153,7 @@ public class plasmaGrafics { // draw disconnected senior and principals that have been seen lately count = 0; - e = yacyCore.seedDB.seedsSortedDisconnected(true, yacySeed.LASTSEEN); + e = yacyCore.seedDB.seedsSortedDisconnected(false, yacySeed.LASTSEEN); while (e.hasMoreElements() && count < maxCount) { seed = (yacySeed) e.nextElement(); if (seed != null) { @@ -167,7 +167,7 @@ public class plasmaGrafics { // draw juniors that have been seen lately count = 0; - e = yacyCore.seedDB.seedsSortedPotential(true, yacySeed.LASTSEEN); + e = yacyCore.seedDB.seedsSortedPotential(false, yacySeed.LASTSEEN); while (e.hasMoreElements() && count < maxCount) { seed = (yacySeed) e.nextElement(); if (seed != null) { diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index 5a817f973..5eae6537b 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -782,60 +782,60 @@ public final class yacySeedDB { } class seedEnum implements Enumeration { - - kelondroMap.mapIterator it; - yacySeed nextSeed; + + kelondroMap.mapIterator it; + yacySeed nextSeed; kelondroMap database; - - public seedEnum(boolean up, boolean rot, byte[] firstKey, kelondroMap database) { + + public seedEnum(boolean up, boolean rot, byte[] firstKey, kelondroMap database) { this.database = database; - try { - it = (firstKey == null) ? database.maps(up, rot) : database.maps(up, rot, firstKey); - nextSeed = internalNext(); - } catch (IOException e) { - yacyCore.log.logFine("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); + try { + it = (firstKey == null) ? database.maps(up, rot) : database.maps(up, rot, firstKey); + nextSeed = internalNext(); + } catch (IOException e) { + yacyCore.log.logFine("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile); if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile); - it = null; - } catch (kelondroException e) { - yacyCore.log.logFine("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); + it = null; + } catch (kelondroException e) { + yacyCore.log.logFine("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile); if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile); - it = null; + it = null; + } } - } - + public seedEnum(boolean up, String field, kelondroMap database) { this.database = database; - try { - it = database.maps(up, field); - nextSeed = internalNext(); - } catch (kelondroException e) { - yacyCore.log.logFine("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); + try { + it = database.maps(up, field); + nextSeed = internalNext(); + } catch (kelondroException e) { + yacyCore.log.logFine("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile); if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile); if (database == seedPotentialDB) seedPotentialDB = resetSeedTable(seedPotentialDB, seedPotentialDBFile); - it = null; + it = null; + } } + + public boolean hasMoreElements() { + return (nextSeed != null); } - - public boolean hasMoreElements() { - return (nextSeed != null); - } - - public yacySeed internalNext() { - if ((it == null) || (!(it.hasNext()))) return null; + + public yacySeed internalNext() { + if ((it == null) || (!(it.hasNext()))) return null; Map dna = (Map) it.next(); String hash = (String) dna.remove("key"); return new yacySeed(hash, dna); - } - - public Object nextElement() { - yacySeed seed = nextSeed; - nextSeed = internalNext(); - return seed; - } - + } + + public Object nextElement() { + yacySeed seed = nextSeed; + nextSeed = internalNext(); + return seed; + } + } } \ No newline at end of file