boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added double computation to kelondroMapObjects

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3316 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 18 years ago
parent 8a0ed1ce50
commit f696d3c1eb
5 changed files with 97 additions and 48 deletions
Show Stats Download Patch File Download Diff File

2
htroot/Network.java
Unescape View File

@ -95,7 +95,7 @@ public class Network {
// final boolean complete = ((post == null) ? false : post.get("links", "false").equals("true")); // final boolean complete = ((post == null) ? false : post.get("links", "false").equals("true"));
final long otherppm = yacyCore.seedDB.countActivePPM(); final long otherppm = yacyCore.seedDB.countActivePPM();
final long otherqpm = yacyCore.seedDB.countActiveQPM(); final double otherqpm = yacyCore.seedDB.countActiveQPM();
long myppm = 0; long myppm = 0;
double myqph = 0d; double myqph = 0d;

96
source/de/anomic/kelondro/kelondroMapObjects.java
Unescape View File

@ -34,21 +34,22 @@ import java.util.Map;
public class kelondroMapObjects extends kelondroObjects { public class kelondroMapObjects extends kelondroObjects {
private String[] sortfields, accfields; private String[] sortfields, longaccfields, doubleaccfields;
private HashMap sortClusterMap; // a String-kelondroMScoreCluster - relation private HashMap sortClusterMap; // a String-kelondroMScoreCluster - relation
private HashMap accMap; // to store accumulations of specific fields private HashMap accMap; // to store accumulations of specific fields
private int elementCount; private int elementCount;
public kelondroMapObjects(kelondroDyn dyn, int cachesize) { public kelondroMapObjects(kelondroDyn dyn, int cachesize) {
this(dyn, cachesize, null, null); this(dyn, cachesize, null, null, null);
} }
public kelondroMapObjects(kelondroDyn dyn, int cachesize, String[] sortfields, String[] accfields) { public kelondroMapObjects(kelondroDyn dyn, int cachesize, String[] sortfields, String[] longaccfields, String[] doubleaccfields) {
super(dyn, cachesize); super(dyn, cachesize);
// create fast ordering clusters and acc fields // create fast ordering clusters and acc fields
this.sortfields = sortfields; this.sortfields = sortfields;
this.accfields = accfields; this.longaccfields = longaccfields;
this.doubleaccfields = doubleaccfields;
kelondroMScoreCluster[] cluster = null; kelondroMScoreCluster[] cluster = null;
if (sortfields == null) sortClusterMap = null; else { if (sortfields == null) sortClusterMap = null; else {
@ -59,20 +60,30 @@ public class kelondroMapObjects extends kelondroObjects {
} }
} }
Long[] accumulator = null; Long[] longaccumulator = null;
if (accfields == null) accMap = null; else { if (longaccfields == null) accMap = null; else {
accMap = new HashMap(); accMap = new HashMap();
accumulator = new Long[accfields.length]; longaccumulator = new Long[longaccfields.length];
for (int i = 0; i < accfields.length; i++) { for (int i = 0; i < longaccfields.length; i++) {
accumulator[i] = new Long(0); longaccumulator[i] = new Long(0);
}
}
Double[] doubleaccumulator = null;
if (doubleaccfields == null) accMap = null; else {
accMap = new HashMap();
doubleaccumulator = new Double[doubleaccfields.length];
for (int i = 0; i < doubleaccfields.length; i++) {
doubleaccumulator[i] = new Double(0);
} }
} }
// fill cluster and accumulator with values // fill cluster and accumulator with values
if ((sortfields != null) || (accfields != null)) try { if ((sortfields != null) || (longaccfields != null) || (doubleaccfields != null)) try {
kelondroDyn.dynKeyIterator it = dyn.dynKeys(true, false); kelondroDyn.dynKeyIterator it = dyn.dynKeys(true, false);
String key, value; String key, value;
long valuel; long valuel;
double valued;
Map map; Map map;
while (it.hasNext()) { while (it.hasNext()) {
key = (String) it.next(); key = (String) it.next();
@ -84,11 +95,19 @@ public class kelondroMapObjects extends kelondroObjects {
if (value != null) cluster[i].setScore(key, kelondroMScoreCluster.string2score(value)); if (value != null) cluster[i].setScore(key, kelondroMScoreCluster.string2score(value));
} }
if (accfields != null) for (int i = 0; i < accfields.length; i++) { if (longaccfields != null) for (int i = 0; i < longaccfields.length; i++) {
value = (String) map.get(accfields[i]); value = (String) map.get(longaccfields[i]);
if (value != null) try { if (value != null) try {
valuel = Long.parseLong(value); valuel = Long.parseLong(value);
accumulator[i] = new Long(accumulator[i].longValue() + valuel); longaccumulator[i] = new Long(longaccumulator[i].longValue() + valuel);
} catch (NumberFormatException e) {}
}
if (doubleaccfields != null) for (int i = 0; i < doubleaccfields.length; i++) {
value = (String) map.get(doubleaccfields[i]);
if (value != null) try {
valued = Double.parseDouble(value);
doubleaccumulator[i] = new Double(doubleaccumulator[i].doubleValue() + valued);
} catch (NumberFormatException e) {} } catch (NumberFormatException e) {}
} }
elementCount++; elementCount++;
@ -99,7 +118,8 @@ public class kelondroMapObjects extends kelondroObjects {
if (sortfields != null) for (int i = 0; i < sortfields.length; i++) sortClusterMap.put(sortfields[i], cluster[i]); if (sortfields != null) for (int i = 0; i < sortfields.length; i++) sortClusterMap.put(sortfields[i], cluster[i]);
// fill acc map // fill acc map
if (accfields != null) for (int i = 0; i < accfields.length; i++) accMap.put(accfields[i], accumulator[i]); if (longaccfields != null) for (int i = 0; i < longaccfields.length; i++) accMap.put(longaccfields[i], longaccumulator[i]);
if (doubleaccfields != null) for (int i = 0; i < doubleaccfields.length; i++) accMap.put(doubleaccfields[i], doubleaccumulator[i]);
} }
public synchronized void set(String key, Map newMap) throws IOException { public synchronized void set(String key, Map newMap) throws IOException {
@ -108,14 +128,14 @@ public class kelondroMapObjects extends kelondroObjects {
assert (newMap != null); assert (newMap != null);
// update elementCount // update elementCount
if ((sortfields != null) || (accfields != null)) { if ((longaccfields != null) || (doubleaccfields != null)) {
final Map oldMap = getMap(key, false); final Map oldMap = getMap(key, false);
if (oldMap == null) { if (oldMap == null) {
// new element // new element
elementCount++; elementCount++;
} else { } else {
// element exists, update acc // element exists, update acc
if (accfields != null) updateAcc(oldMap, false); if ((longaccfields != null) || (doubleaccfields != null)) updateAcc(oldMap, false);
} }
} }
@ -125,23 +145,39 @@ public class kelondroMapObjects extends kelondroObjects {
if (sortClusterMap != null) updateSortCluster(key, newMap); if (sortClusterMap != null) updateSortCluster(key, newMap);
// update accumulators with new values (add) // update accumulators with new values (add)
if (accfields != null) updateAcc(newMap, true); if ((longaccfields != null) || (doubleaccfields != null)) updateAcc(newMap, true);
} }
private void updateAcc(Map map, boolean add) { private void updateAcc(Map map, boolean add) {
String value; String value;
long valuel; long valuel;
Long accumulator; double valued;
for (int i = 0; i < accfields.length; i++) { Long longaccumulator;
value = (String) map.get(accfields[i]); Double doubleaccumulator;
if (longaccfields != null) for (int i = 0; i < longaccfields.length; i++) {
value = (String) map.get(longaccfields[i]);
if (value != null) { if (value != null) {
try { try {
valuel = Long.parseLong(value); valuel = Long.parseLong(value);
accumulator = (Long) accMap.get(accfields[i]); longaccumulator = (Long) accMap.get(longaccfields[i]);
if (add) { if (add) {
accMap.put(accfields[i], new Long(accumulator.longValue() + valuel)); accMap.put(longaccfields[i], new Long(longaccumulator.longValue() + valuel));
} else { } else {
accMap.put(accfields[i], new Long(accumulator.longValue() - valuel)); accMap.put(longaccfields[i], new Long(longaccumulator.longValue() - valuel));
}
} catch (NumberFormatException e) {}
}
}
if (doubleaccfields != null) for (int i = 0; i < doubleaccfields.length; i++) {
value = (String) map.get(doubleaccfields[i]);
if (value != null) {
try {
valued = Double.parseDouble(value);
doubleaccumulator = (Double) accMap.get(doubleaccfields[i]);
if (add) {
accMap.put(doubleaccfields[i], new Double(doubleaccumulator.doubleValue() + valued));
} else {
accMap.put(doubleaccfields[i], new Double(doubleaccumulator.doubleValue() - valued));
} }
} catch (NumberFormatException e) {} } catch (NumberFormatException e) {}
} }
@ -165,14 +201,14 @@ public class kelondroMapObjects extends kelondroObjects {
if (key == null) return; if (key == null) return;
// update elementCount // update elementCount
if ((sortfields != null) || (accfields != null)) { if ((sortfields != null) || (longaccfields != null) || (doubleaccfields != null)) {
final Map map = getMap(key); final Map map = getMap(key);
if (map != null) { if (map != null) {
// update count // update count
elementCount--; elementCount--;
// update accumulators (subtract) // update accumulators (subtract)
if (accfields != null) updateAcc(map, false); if ((longaccfields != null) || (doubleaccfields != null)) updateAcc(map, false);
// remove from sortCluster // remove from sortCluster
if (sortfields != null) deleteSortCluster(key); if (sortfields != null) deleteSortCluster(key);
@ -234,14 +270,20 @@ public class kelondroMapObjects extends kelondroObjects {
return new mapIterator(keys(up, rotating, firstKey)); return new mapIterator(keys(up, rotating, firstKey));
} }
public synchronized long getAcc(final String field) { public synchronized long getLongAcc(final String field) {
final Long accumulator = (Long) accMap.get(field); final Long accumulator = (Long) accMap.get(field);
if (accumulator == null) return -1; if (accumulator == null) return -1;
return accumulator.longValue(); return accumulator.longValue();
} }
public synchronized double getDoubleAcc(final String field) {
final Double accumulator = (Double) accMap.get(field);
if (accumulator == null) return -1;
return accumulator.doubleValue();
}
public synchronized int size() { public synchronized int size() {
if ((sortfields != null) || (accfields != null)) return elementCount; if ((sortfields != null) || (longaccfields != null) || (doubleaccfields != null)) return elementCount;
return super.size(); return super.size();
} }

20
source/de/anomic/kelondro/kelondroMapTable.java
Unescape View File

@ -64,18 +64,18 @@ public class kelondroMapTable {
public void declareMaps( public void declareMaps(
String tablename, int keysize, int nodesize, int cacheslots, String tablename, int keysize, int nodesize, int cacheslots,
char fillChar) throws IOException { char fillChar) throws IOException {
declareMaps(tablename, keysize, nodesize, cacheslots, null, null, fillChar); declareMaps(tablename, keysize, nodesize, cacheslots, null, null, null, fillChar);
} }
public void declareMaps( public void declareMaps(
String tablename, int keysize, int nodesize, int cacheslots, String tablename, int keysize, int nodesize, int cacheslots,
String[] sortfields, String[] accfields, char fillChar) throws IOException { String[] sortfields, String[] longaccfields, String[] doubleaccfields, char fillChar) throws IOException {
declareMaps(tablename, keysize, nodesize, cacheslots, sortfields, accfields, fillChar, 0x800, 0); declareMaps(tablename, keysize, nodesize, cacheslots, sortfields, longaccfields, doubleaccfields, fillChar, 0x800, 0);
} }
public void declareMaps( public void declareMaps(
String tablename, int keysize, int nodesize, int cacheslots, String tablename, int keysize, int nodesize, int cacheslots,
String[] sortfields, String[] accfields, char fillChar, String[] sortfields, String[] longaccfields, String[] doubleaccfields, char fillChar,
long buffersize /*bytes*/, long preloadTime) throws IOException { long buffersize /*bytes*/, long preloadTime) throws IOException {
if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared twice."); if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared twice.");
if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared already in other context."); if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared already in other context.");
@ -83,7 +83,7 @@ public class kelondroMapTable {
kelondroDyn dyn; kelondroDyn dyn;
if (!(tablefile.exists())) tablefile.getParentFile().mkdirs(); if (!(tablefile.exists())) tablefile.getParentFile().mkdirs();
dyn = new kelondroDyn(tablefile, buffersize, preloadTime, keysize, nodesize, fillChar, true, false); dyn = new kelondroDyn(tablefile, buffersize, preloadTime, keysize, nodesize, fillChar, true, false);
kelondroMapObjects map = new kelondroMapObjects(dyn, cacheslots, sortfields, accfields); kelondroMapObjects map = new kelondroMapObjects(dyn, cacheslots, sortfields, longaccfields, doubleaccfields);
mTables.put(tablename, map); mTables.put(tablename, map);
} }
@ -160,10 +160,16 @@ public class kelondroMapTable {
throw new RuntimeException("kelondroTables.delete: table '" + tablename + "' does not exist."); throw new RuntimeException("kelondroTables.delete: table '" + tablename + "' does not exist.");
} }
public synchronized long accumulator(String tablename, String field) { public synchronized long longAccumulator(String tablename, String field) {
kelondroMapObjects table = (kelondroMapObjects) mTables.get(tablename); kelondroMapObjects table = (kelondroMapObjects) mTables.get(tablename);
if (table == null) throw new RuntimeException("kelondroTables.accumulator: map table '" + tablename + "' does not exist."); if (table == null) throw new RuntimeException("kelondroTables.accumulator: map table '" + tablename + "' does not exist.");
return table.getAcc(field); return table.getLongAcc(field);
}
public synchronized double doubleAccumulator(String tablename, String field) {
kelondroMapObjects table = (kelondroMapObjects) mTables.get(tablename);
if (table == null) throw new RuntimeException("kelondroTables.accumulator: map table '" + tablename + "' does not exist.");
return table.getDoubleAcc(field);
} }
public synchronized int size(String tablename) { public synchronized int size(String tablename) {

25
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -87,8 +87,9 @@ public final class yacySeedDB {
*/ */
public static final int commonHashLength = 12; public static final int commonHashLength = 12;
public static final String[] sortFields = new String[] {yacySeed.LCOUNT, yacySeed.ICOUNT, yacySeed.UPTIME, yacySeed.VERSION, yacySeed.LASTSEEN}; public static final String[] sortFields = new String[] {yacySeed.LCOUNT, yacySeed.ICOUNT, yacySeed.UPTIME, yacySeed.VERSION, yacySeed.LASTSEEN};
public static final String[] accFields = new String[] {yacySeed.LCOUNT, yacySeed.ICOUNT, yacySeed.ISPEED, yacySeed.RSPEED}; public static final String[] longaccFields = new String[] {yacySeed.LCOUNT, yacySeed.ICOUNT, yacySeed.ISPEED};
public static final String[] doubleaccFields = new String[] {yacySeed.RSPEED};
// class objects // class objects
protected File seedActiveDBFile, seedPassiveDBFile, seedPotentialDBFile; protected File seedActiveDBFile, seedPassiveDBFile, seedPotentialDBFile;
@ -206,11 +207,11 @@ public final class yacySeedDB {
private synchronized kelondroMapObjects openSeedTable(File seedDBFile) { private synchronized kelondroMapObjects openSeedTable(File seedDBFile) {
new File(seedDBFile.getParent()).mkdirs(); new File(seedDBFile.getParent()).mkdirs();
try { try {
return new kelondroMapObjects(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, accFields); return new kelondroMapObjects(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, longaccFields, doubleaccFields);
} catch (Exception e) { } catch (Exception e) {
seedDBFile.delete(); seedDBFile.delete();
// try again // try again
return new kelondroMapObjects(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, accFields); return new kelondroMapObjects(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, longaccFields, doubleaccFields);
} }
} }
@ -337,14 +338,14 @@ public final class yacySeedDB {
*/ */
} }
public long countActiveURL() { return seedActiveDB.getAcc(yacySeed.LCOUNT); } public long countActiveURL() { return seedActiveDB.getLongAcc(yacySeed.LCOUNT); }
public long countActiveRWI() { return seedActiveDB.getAcc(yacySeed.ICOUNT); } public long countActiveRWI() { return seedActiveDB.getLongAcc(yacySeed.ICOUNT); }
public long countActivePPM() { return seedActiveDB.getAcc(yacySeed.ISPEED); } public long countActivePPM() { return seedActiveDB.getLongAcc(yacySeed.ISPEED); }
public long countActiveQPM() { return seedActiveDB.getAcc(yacySeed.RSPEED); } public double countActiveQPM() { return seedActiveDB.getDoubleAcc(yacySeed.RSPEED); }
public long countPassiveURL() { return seedPassiveDB.getAcc(yacySeed.LCOUNT); } public long countPassiveURL() { return seedPassiveDB.getLongAcc(yacySeed.LCOUNT); }
public long countPassiveRWI() { return seedPassiveDB.getAcc(yacySeed.ICOUNT); } public long countPassiveRWI() { return seedPassiveDB.getLongAcc(yacySeed.ICOUNT); }
public long countPotentialURL() { return seedPotentialDB.getAcc(yacySeed.LCOUNT); } public long countPotentialURL() { return seedPotentialDB.getLongAcc(yacySeed.LCOUNT); }
public long countPotentialRWI() { return seedPotentialDB.getAcc(yacySeed.ICOUNT); } public long countPotentialRWI() { return seedPotentialDB.getLongAcc(yacySeed.ICOUNT); }
public synchronized void addConnected(yacySeed seed) { public synchronized void addConnected(yacySeed seed) {
if ((seed == null) || (seed.isProper() != null)) return; if ((seed == null) || (seed.isProper() != null)) return;

2
source/yacy.java
Unescape View File

@ -1327,7 +1327,7 @@ public final class yacy {
String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"}; String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"};
for (int i=0; i < dbFileNames.length; i++) { for (int i=0; i < dbFileNames.length; i++) {
File dbFile = new File(yacyDBPath,dbFileNames[i]); File dbFile = new File(yacyDBPath,dbFileNames[i]);
kelondroMapObjects db = new kelondroMapObjects(new kelondroDyn(dbFile, (1024 * 0x400) / 3, 3000, yacySeedDB.commonHashLength, 480, '#', true, false), 500, yacySeedDB.sortFields, yacySeedDB.accFields); kelondroMapObjects db = new kelondroMapObjects(new kelondroDyn(dbFile, (1024 * 0x400) / 3, 3000, yacySeedDB.commonHashLength, 480, '#', true, false), 500, yacySeedDB.sortFields, yacySeedDB.longaccFields, yacySeedDB.doubleaccFields);
kelondroMapObjects.mapIterator it; kelondroMapObjects.mapIterator it;
it = db.maps(true, false); it = db.maps(true, false);

Write Preview
Loading…
Cancel
Save