boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

some redesign attempts because sorting of lastseen does not work correctly

Browse Source 
not finished yet
target: better selection of peer-ping targets, which should enhance stabilization of the net

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3319 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 18 years ago
parent c80707ccd5
commit fcc11391a8
8 changed files with 87 additions and 28 deletions
Show Stats Download Patch File Download Diff File

2
htroot/Network.java
Unescape View File

@ -431,7 +431,7 @@ public class Network {
prop.put(STR_TABLE_LIST + conCount + "_dhtreceive_peertags", ((peertags == null) || (peertags.length() == 0)) ? "no tags given" : ("tags = " + peertags));
}
prop.put(STR_TABLE_LIST + conCount + "_version", yacy.combined2prettyVersion(seed.get(yacySeed.VERSION, "0.1"), shortname));
prop.put(STR_TABLE_LIST + conCount + "_lastSeen", lastseen);
prop.put(STR_TABLE_LIST + conCount + "_lastSeen", seed.getLastSeenString() + " " + lastseen);
prop.put(STR_TABLE_LIST + conCount + "_utc", seed.get(yacySeed.UTC, "-"));
prop.put(STR_TABLE_LIST + conCount + "_uptime", serverDate.intervalToString(60000 * Long.parseLong(seed.get(yacySeed.UPTIME, "0"))));
prop.put(STR_TABLE_LIST + conCount + "_links", groupDigits(seed.get(yacySeed.LCOUNT, "0")));

19
source/de/anomic/kelondro/kelondroMScoreCluster.java
Unescape View File

@ -73,7 +73,20 @@ public final class kelondroMScoreCluster {
} catch (ParseException e) {}
}
public static int string2score(String s) {
public static int object2score(Object o) {
if (o instanceof Integer) return ((Integer) o).intValue();
if (o instanceof Long) {
long l = ((Long) o).longValue();
if (l < (long) Integer.MAX_VALUE) return (int) l;
o = ((Long) o).toString();
}
if (o instanceof Double) {
double d = 1000d * ((Double) o).doubleValue();
return (int) Math.round(d);
}
String s = "";
if (o instanceof String) s = (String) o;
// this can be used to calculate a score from a string
if ((s == null) || (s.length() == 0) || (s.charAt(0) == '-')) return 0;
try {
@ -410,9 +423,9 @@ public final class kelondroMScoreCluster {
public static void main(String[] args) {
String t = "ZZZZZZZZZZ";
System.out.println("score of " + t + ": " + string2score(t));
System.out.println("score of " + t + ": " + object2score(t));
if (args.length > 0) {
System.out.println("score of " + args[0] + ": " + string2score(args[0]));
System.out.println("score of " + args[0] + ": " + object2score(args[0]));
System.exit(0);
}

53
source/de/anomic/kelondro/kelondroMapObjects.java
Unescape View File

@ -28,6 +28,8 @@
package de.anomic.kelondro;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
@ -40,10 +42,10 @@ public class kelondroMapObjects extends kelondroObjects {
private int elementCount;
public kelondroMapObjects(kelondroDyn dyn, int cachesize) {
this(dyn, cachesize, null, null, null);
this(dyn, cachesize, null, null, null, null, null);
}
public kelondroMapObjects(kelondroDyn dyn, int cachesize, String[] sortfields, String[] longaccfields, String[] doubleaccfields) {
public kelondroMapObjects(kelondroDyn dyn, int cachesize, String[] sortfields, String[] longaccfields, String[] doubleaccfields, Method externalInitializer, Object externalHandler) {
super(dyn, cachesize);
// create fast ordering clusters and acc fields
@ -81,35 +83,52 @@ public class kelondroMapObjects extends kelondroObjects {
// fill cluster and accumulator with values
if ((sortfields != null) || (longaccfields != null) || (doubleaccfields != null)) try {
kelondroDyn.dynKeyIterator it = dyn.dynKeys(true, false);
String key, value;
String mapname;
Object cell;
long valuel;
double valued;
Map map;
while (it.hasNext()) {
key = (String) it.next();
map = getMap(key);
mapname = (String) it.next();
map = getMap(mapname);
if (map == null) break;
if (sortfields != null) for (int i = 0; i < sortfields.length; i++) {
value = (String) map.get(sortfields[i]);
if (value != null) cluster[i].setScore(key, kelondroMScoreCluster.string2score(value));
cell = map.get(sortfields[i]);
if (cell != null) cluster[i].setScore(mapname, kelondroMScoreCluster.object2score(cell));
}
if (longaccfields != null) for (int i = 0; i < longaccfields.length; i++) {
value = (String) map.get(longaccfields[i]);
if (value != null) try {
valuel = Long.parseLong(value);
cell = map.get(longaccfields[i]);
valuel = 0;
if (cell != null) try {
if (cell instanceof Long) valuel = ((Long) cell).longValue();
if (cell instanceof String) valuel = Long.parseLong((String) cell);
longaccumulator[i] = new Long(longaccumulator[i].longValue() + valuel);
} catch (NumberFormatException e) {}
}
if (doubleaccfields != null) for (int i = 0; i < doubleaccfields.length; i++) {
value = (String) map.get(doubleaccfields[i]);
if (value != null) try {
valued = Double.parseDouble(value);
cell = map.get(doubleaccfields[i]);
valued = 0d;
if (cell != null) try {
if (cell instanceof Double) valued = ((Double) cell).doubleValue();
if (cell instanceof String) valued = Double.parseDouble((String) cell);
doubleaccumulator[i] = new Double(doubleaccumulator[i].doubleValue() + valued);
} catch (NumberFormatException e) {}
}
if ((externalHandler != null) && (externalInitializer != null)) {
try {
externalInitializer.invoke(externalHandler, new Object[]{mapname, map});
} catch (IllegalArgumentException e) {
e.printStackTrace();
} catch (IllegalAccessException e) {
e.printStackTrace();
} catch (InvocationTargetException e) {
e.printStackTrace();
}
}
elementCount++;
}
} catch (IOException e) {}
@ -185,13 +204,13 @@ public class kelondroMapObjects extends kelondroObjects {
}
private void updateSortCluster(final String key, final Map map) {
String value;
Object cell;
kelondroMScoreCluster cluster;
for (int i = 0; i < sortfields.length; i++) {
value = (String) map.get(sortfields[i]);
if (value != null) {
cell = map.get(sortfields[i]);
if (cell != null) {
cluster = (kelondroMScoreCluster) sortClusterMap.get(sortfields[i]);
cluster.setScore(key, kelondroMScoreCluster.string2score(value));
cluster.setScore(key, kelondroMScoreCluster.object2score(cell));
sortClusterMap.put(sortfields[i], cluster);
}
}

2
source/de/anomic/kelondro/kelondroMapTable.java
Unescape View File

@ -83,7 +83,7 @@ public class kelondroMapTable {
kelondroDyn dyn;
if (!(tablefile.exists())) tablefile.getParentFile().mkdirs();
dyn = new kelondroDyn(tablefile, buffersize, preloadTime, keysize, nodesize, fillChar, true, false);
kelondroMapObjects map = new kelondroMapObjects(dyn, cacheslots, sortfields, longaccfields, doubleaccfields);
kelondroMapObjects map = new kelondroMapObjects(dyn, cacheslots, sortfields, longaccfields, doubleaccfields, null, null);
mTables.put(tablename, map);
}

6
source/de/anomic/server/serverCore.java
Unescape View File

@ -95,6 +95,9 @@ public final class serverCore extends serverAbstractThread implements serverThre
public static final byte lf = 10;
public static final byte[] crlf = {cr, lf};
public static final String crlfString = new String(crlf);
public static final Class[] stringType = {"".getClass()}; // set up some reflection
//Class[] exceptionType = {Class.forName("java.lang.Throwable")};
// static variables
public static final Boolean TERMINATE_CONNECTION = Boolean.FALSE;
@ -1138,9 +1141,6 @@ public final class serverCore extends serverAbstractThread implements serverThre
private void listen() {
try {
// set up some reflection
Class[] stringType = {"".getClass()};
//Class[] exceptionType = {Class.forName("java.lang.Throwable")};
Object result;
// // send greeting

4
source/de/anomic/yacy/yacySeed.java
Unescape View File

@ -399,6 +399,10 @@ public class yacySeed {
return System.currentTimeMillis();
}
}
public final String getLastSeenString() {
return get(yacySeed.LASTSEEN, "20040101000000");
}
public final int getAge() {
// returns the age as number of days

27
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -49,6 +49,7 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.lang.ref.SoftReference;
import java.lang.reflect.Method;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
@ -206,12 +207,29 @@ public final class yacySeedDB {
private synchronized kelondroMapObjects openSeedTable(File seedDBFile) {
new File(seedDBFile.getParent()).mkdirs();
Class[] args;
try {
return new kelondroMapObjects(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, longaccFields, doubleaccFields);
args = new Class[]{"".getClass(), Class.forName("java.util.Map")};
} catch (ClassNotFoundException e2){
e2.printStackTrace();
args = null;
}
Method initializeHandlerMethod;
try {
initializeHandlerMethod = this.getClass().getMethod("initializeHandler", args);
} catch (SecurityException e1) {
e1.printStackTrace();
initializeHandlerMethod = null;
} catch (NoSuchMethodException e1) {
e1.printStackTrace();
initializeHandlerMethod = null;
}
try {
return new kelondroMapObjects(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
} catch (Exception e) {
seedDBFile.delete();
// try again
return new kelondroMapObjects(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, longaccFields, doubleaccFields);
return new kelondroMapObjects(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
}
}
@ -243,6 +261,11 @@ public final class yacySeedDB {
}
}
public void initializeHandler(String mapname, Map map) {
// this is used to set up a lastSeen lookup table
}
public Enumeration seedsSortedConnected(boolean up, String field) {
// enumerates seed-type objects: all seeds sequentially ordered by field
return new seedEnum(up, field, seedActiveDB);

2
source/yacy.java
Unescape View File

@ -1327,7 +1327,7 @@ public final class yacy {
String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"};
for (int i=0; i < dbFileNames.length; i++) {
File dbFile = new File(yacyDBPath,dbFileNames[i]);
kelondroMapObjects db = new kelondroMapObjects(new kelondroDyn(dbFile, (1024 * 0x400) / 3, 3000, yacySeedDB.commonHashLength, 480, '#', true, false), 500, yacySeedDB.sortFields, yacySeedDB.longaccFields, yacySeedDB.doubleaccFields);
kelondroMapObjects db = new kelondroMapObjects(new kelondroDyn(dbFile, (1024 * 0x400) / 3, 3000, yacySeedDB.commonHashLength, 480, '#', true, false), 500, yacySeedDB.sortFields, yacySeedDB.longaccFields, yacySeedDB.doubleaccFields, null, null);
kelondroMapObjects.mapIterator it;
it = db.maps(true, false);

Write Preview
Loading…
Cancel
Save