boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added double computation to kelondroMapObjects

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3316 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 18 years ago
parent 8a0ed1ce50
commit f696d3c1eb
5 changed files with 97 additions and 48 deletions
Show Stats Download Patch File Download Diff File

2
htroot/Network.java
Unescape View File

@ -95,7 +95,7 @@ public class Network {
// final boolean complete = ((post == null) ? false : post.get("links", "false").equals("true"));
final long otherppm = yacyCore.seedDB.countActivePPM();
final long otherqpm = yacyCore.seedDB.countActiveQPM();
final double otherqpm = yacyCore.seedDB.countActiveQPM();
long myppm = 0;
double myqph = 0d;

96
source/de/anomic/kelondro/kelondroMapObjects.java
Unescape View File

@ -34,21 +34,22 @@ import java.util.Map;
public class kelondroMapObjects extends kelondroObjects {
private String[] sortfields, accfields;
private String[] sortfields, longaccfields, doubleaccfields;
private HashMap sortClusterMap; // a String-kelondroMScoreCluster - relation
private HashMap accMap; // to store accumulations of specific fields
private int elementCount;
public kelondroMapObjects(kelondroDyn dyn, int cachesize) {
this(dyn, cachesize, null, null);
this(dyn, cachesize, null, null, null);
}
public kelondroMapObjects(kelondroDyn dyn, int cachesize, String[] sortfields, String[] accfields) {
public kelondroMapObjects(kelondroDyn dyn, int cachesize, String[] sortfields, String[] longaccfields, String[] doubleaccfields) {
super(dyn, cachesize);
// create fast ordering clusters and acc fields
this.sortfields = sortfields;
this.accfields = accfields;
this.longaccfields = longaccfields;
this.doubleaccfields = doubleaccfields;
kelondroMScoreCluster[] cluster = null;
if (sortfields == null) sortClusterMap = null; else {
@ -59,20 +60,30 @@ public class kelondroMapObjects extends kelondroObjects {
}
}
Long[] accumulator = null;
if (accfields == null) accMap = null; else {
Long[] longaccumulator = null;
if (longaccfields == null) accMap = null; else {
accMap = new HashMap();
accumulator = new Long[accfields.length];
for (int i = 0; i < accfields.length; i++) {
accumulator[i] = new Long(0);
longaccumulator = new Long[longaccfields.length];
for (int i = 0; i < longaccfields.length; i++) {
longaccumulator[i] = new Long(0);
}
}
Double[] doubleaccumulator = null;
if (doubleaccfields == null) accMap = null; else {
accMap = new HashMap();
doubleaccumulator = new Double[doubleaccfields.length];
for (int i = 0; i < doubleaccfields.length; i++) {
doubleaccumulator[i] = new Double(0);
}
}
// fill cluster and accumulator with values
if ((sortfields != null) || (accfields != null)) try {
if ((sortfields != null) || (longaccfields != null) || (doubleaccfields != null)) try {
kelondroDyn.dynKeyIterator it = dyn.dynKeys(true, false);
String key, value;
long valuel;
double valued;
Map map;
while (it.hasNext()) {
key = (String) it.next();
@ -84,11 +95,19 @@ public class kelondroMapObjects extends kelondroObjects {
if (value != null) cluster[i].setScore(key, kelondroMScoreCluster.string2score(value));
}
if (accfields != null) for (int i = 0; i < accfields.length; i++) {
value = (String) map.get(accfields[i]);
if (longaccfields != null) for (int i = 0; i < longaccfields.length; i++) {
value = (String) map.get(longaccfields[i]);
if (value != null) try {
valuel = Long.parseLong(value);
accumulator[i] = new Long(accumulator[i].longValue() + valuel);
longaccumulator[i] = new Long(longaccumulator[i].longValue() + valuel);
} catch (NumberFormatException e) {}
}
if (doubleaccfields != null) for (int i = 0; i < doubleaccfields.length; i++) {
value = (String) map.get(doubleaccfields[i]);
if (value != null) try {
valued = Double.parseDouble(value);
doubleaccumulator[i] = new Double(doubleaccumulator[i].doubleValue() + valued);
} catch (NumberFormatException e) {}
}
elementCount++;
@ -99,7 +118,8 @@ public class kelondroMapObjects extends kelondroObjects {
if (sortfields != null) for (int i = 0; i < sortfields.length; i++) sortClusterMap.put(sortfields[i], cluster[i]);
// fill acc map
if (accfields != null) for (int i = 0; i < accfields.length; i++) accMap.put(accfields[i], accumulator[i]);
if (longaccfields != null) for (int i = 0; i < longaccfields.length; i++) accMap.put(longaccfields[i], longaccumulator[i]);
if (doubleaccfields != null) for (int i = 0; i < doubleaccfields.length; i++) accMap.put(doubleaccfields[i], doubleaccumulator[i]);
}
public synchronized void set(String key, Map newMap) throws IOException {
@ -108,14 +128,14 @@ public class kelondroMapObjects extends kelondroObjects {
assert (newMap != null);
// update elementCount
if ((sortfields != null) || (accfields != null)) {
if ((longaccfields != null) || (doubleaccfields != null)) {
final Map oldMap = getMap(key, false);
if (oldMap == null) {
// new element
elementCount++;
} else {
// element exists, update acc
if (accfields != null) updateAcc(oldMap, false);
if ((longaccfields != null) || (doubleaccfields != null)) updateAcc(oldMap, false);
}
}
@ -125,23 +145,39 @@ public class kelondroMapObjects extends kelondroObjects {
if (sortClusterMap != null) updateSortCluster(key, newMap);
// update accumulators with new values (add)
if (accfields != null) updateAcc(newMap, true);
if ((longaccfields != null) || (doubleaccfields != null)) updateAcc(newMap, true);
}
private void updateAcc(Map map, boolean add) {
String value;
long valuel;
Long accumulator;
for (int i = 0; i < accfields.length; i++) {
value = (String) map.get(accfields[i]);
double valued;
Long longaccumulator;
Double doubleaccumulator;
if (longaccfields != null) for (int i = 0; i < longaccfields.length; i++) {
value = (String) map.get(longaccfields[i]);
if (value != null) {
try {
valuel = Long.parseLong(value);
accumulator = (Long) accMap.get(accfields[i]);
longaccumulator = (Long) accMap.get(longaccfields[i]);
if (add) {
accMap.put(accfields[i], new Long(accumulator.longValue() + valuel));
accMap.put(longaccfields[i], new Long(longaccumulator.longValue() + valuel));
} else {
accMap.put(accfields[i], new Long(accumulator.longValue() - valuel));
accMap.put(longaccfields[i], new Long(longaccumulator.longValue() - valuel));
}
} catch (NumberFormatException e) {}
}
}
if (doubleaccfields != null) for (int i = 0; i < doubleaccfields.length; i++) {
value = (String) map.get(doubleaccfields[i]);
if (value != null) {
try {
valued = Double.parseDouble(value);
doubleaccumulator = (Double) accMap.get(doubleaccfields[i]);
if (add) {
accMap.put(doubleaccfields[i], new Double(doubleaccumulator.doubleValue() + valued));
} else {
accMap.put(doubleaccfields[i], new Double(doubleaccumulator.doubleValue() - valued));
}
} catch (NumberFormatException e) {}
}
@ -165,14 +201,14 @@ public class kelondroMapObjects extends kelondroObjects {
if (key == null) return;
// update elementCount
if ((sortfields != null) || (accfields != null)) {
if ((sortfields != null) || (longaccfields != null) || (doubleaccfields != null)) {
final Map map = getMap(key);
if (map != null) {
// update count
elementCount--;
// update accumulators (subtract)
if (accfields != null) updateAcc(map, false);
if ((longaccfields != null) || (doubleaccfields != null)) updateAcc(map, false);
// remove from sortCluster
if (sortfields != null) deleteSortCluster(key);
@ -234,14 +270,20 @@ public class kelondroMapObjects extends kelondroObjects {
return new mapIterator(keys(up, rotating, firstKey));
}
public synchronized long getAcc(final String field) {
public synchronized long getLongAcc(final String field) {
final Long accumulator = (Long) accMap.get(field);
if (accumulator == null) return -1;
return accumulator.longValue();
}
public synchronized double getDoubleAcc(final String field) {
final Double accumulator = (Double) accMap.get(field);
if (accumulator == null) return -1;
return accumulator.doubleValue();
}
public synchronized int size() {
if ((sortfields != null) || (accfields != null)) return elementCount;
if ((sortfields != null) || (longaccfields != null) || (doubleaccfields != null)) return elementCount;
return super.size();
}

20
source/de/anomic/kelondro/kelondroMapTable.java
Unescape View File

@ -64,18 +64,18 @@ public class kelondroMapTable {
public void declareMaps(
String tablename, int keysize, int nodesize, int cacheslots,
char fillChar) throws IOException {
declareMaps(tablename, keysize, nodesize, cacheslots, null, null, fillChar);
declareMaps(tablename, keysize, nodesize, cacheslots, null, null, null, fillChar);
}
public void declareMaps(
String tablename, int keysize, int nodesize, int cacheslots,
String[] sortfields, String[] accfields, char fillChar) throws IOException {
declareMaps(tablename, keysize, nodesize, cacheslots, sortfields, accfields, fillChar, 0x800, 0);
String[] sortfields, String[] longaccfields, String[] doubleaccfields, char fillChar) throws IOException {
declareMaps(tablename, keysize, nodesize, cacheslots, sortfields, longaccfields, doubleaccfields, fillChar, 0x800, 0);
}
public void declareMaps(
String tablename, int keysize, int nodesize, int cacheslots,
String[] sortfields, String[] accfields, char fillChar,
String[] sortfields, String[] longaccfields, String[] doubleaccfields, char fillChar,
long buffersize /*bytes*/, long preloadTime) throws IOException {
if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared twice.");
if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared already in other context.");
@ -83,7 +83,7 @@ public class kelondroMapTable {
kelondroDyn dyn;
if (!(tablefile.exists())) tablefile.getParentFile().mkdirs();
dyn = new kelondroDyn(tablefile, buffersize, preloadTime, keysize, nodesize, fillChar, true, false);
kelondroMapObjects map = new kelondroMapObjects(dyn, cacheslots, sortfields, accfields);
kelondroMapObjects map = new kelondroMapObjects(dyn, cacheslots, sortfields, longaccfields, doubleaccfields);
mTables.put(tablename, map);
}
@ -160,10 +160,16 @@ public class kelondroMapTable {
throw new RuntimeException("kelondroTables.delete: table '" + tablename + "' does not exist.");
}
public synchronized long accumulator(String tablename, String field) {
public synchronized long longAccumulator(String tablename, String field) {
kelondroMapObjects table = (kelondroMapObjects) mTables.get(tablename);
if (table == null) throw new RuntimeException("kelondroTables.accumulator: map table '" + tablename + "' does not exist.");
return table.getAcc(field);
return table.getLongAcc(field);
}
public synchronized double doubleAccumulator(String tablename, String field) {
kelondroMapObjects table = (kelondroMapObjects) mTables.get(tablename);
if (table == null) throw new RuntimeException("kelondroTables.accumulator: map table '" + tablename + "' does not exist.");
return table.getDoubleAcc(field);
}
public synchronized int size(String tablename) {

25
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -87,8 +87,9 @@ public final class yacySeedDB {
*/
public static final int commonHashLength = 12;
public static final String[] sortFields = new String[] {yacySeed.LCOUNT, yacySeed.ICOUNT, yacySeed.UPTIME, yacySeed.VERSION, yacySeed.LASTSEEN};
public static final String[] accFields = new String[] {yacySeed.LCOUNT, yacySeed.ICOUNT, yacySeed.ISPEED, yacySeed.RSPEED};
public static final String[] sortFields = new String[] {yacySeed.LCOUNT, yacySeed.ICOUNT, yacySeed.UPTIME, yacySeed.VERSION, yacySeed.LASTSEEN};
public static final String[] longaccFields = new String[] {yacySeed.LCOUNT, yacySeed.ICOUNT, yacySeed.ISPEED};
public static final String[] doubleaccFields = new String[] {yacySeed.RSPEED};
// class objects
protected File seedActiveDBFile, seedPassiveDBFile, seedPotentialDBFile;
@ -206,11 +207,11 @@ public final class yacySeedDB {
private synchronized kelondroMapObjects openSeedTable(File seedDBFile) {
new File(seedDBFile.getParent()).mkdirs();
try {
return new kelondroMapObjects(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, accFields);
return new kelondroMapObjects(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, longaccFields, doubleaccFields);
} catch (Exception e) {
seedDBFile.delete();
// try again
return new kelondroMapObjects(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, accFields);
return new kelondroMapObjects(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, longaccFields, doubleaccFields);
}
}
@ -337,14 +338,14 @@ public final class yacySeedDB {
*/
}
public long countActiveURL() { return seedActiveDB.getAcc(yacySeed.LCOUNT); }
public long countActiveRWI() { return seedActiveDB.getAcc(yacySeed.ICOUNT); }
public long countActivePPM() { return seedActiveDB.getAcc(yacySeed.ISPEED); }
public long countActiveQPM() { return seedActiveDB.getAcc(yacySeed.RSPEED); }
public long countPassiveURL() { return seedPassiveDB.getAcc(yacySeed.LCOUNT); }
public long countPassiveRWI() { return seedPassiveDB.getAcc(yacySeed.ICOUNT); }
public long countPotentialURL() { return seedPotentialDB.getAcc(yacySeed.LCOUNT); }
public long countPotentialRWI() { return seedPotentialDB.getAcc(yacySeed.ICOUNT); }
public long countActiveURL() { return seedActiveDB.getLongAcc(yacySeed.LCOUNT); }
public long countActiveRWI() { return seedActiveDB.getLongAcc(yacySeed.ICOUNT); }
public long countActivePPM() { return seedActiveDB.getLongAcc(yacySeed.ISPEED); }
public double countActiveQPM() { return seedActiveDB.getDoubleAcc(yacySeed.RSPEED); }
public long countPassiveURL() { return seedPassiveDB.getLongAcc(yacySeed.LCOUNT); }
public long countPassiveRWI() { return seedPassiveDB.getLongAcc(yacySeed.ICOUNT); }
public long countPotentialURL() { return seedPotentialDB.getLongAcc(yacySeed.LCOUNT); }
public long countPotentialRWI() { return seedPotentialDB.getLongAcc(yacySeed.ICOUNT); }
public synchronized void addConnected(yacySeed seed) {
if ((seed == null) || (seed.isProper() != null)) return;

2
source/yacy.java
Unescape View File

@ -1327,7 +1327,7 @@ public final class yacy {
String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"};
for (int i=0; i < dbFileNames.length; i++) {
File dbFile = new File(yacyDBPath,dbFileNames[i]);
kelondroMapObjects db = new kelondroMapObjects(new kelondroDyn(dbFile, (1024 * 0x400) / 3, 3000, yacySeedDB.commonHashLength, 480, '#', true, false), 500, yacySeedDB.sortFields, yacySeedDB.accFields);
kelondroMapObjects db = new kelondroMapObjects(new kelondroDyn(dbFile, (1024 * 0x400) / 3, 3000, yacySeedDB.commonHashLength, 480, '#', true, false), 500, yacySeedDB.sortFields, yacySeedDB.longaccFields, yacySeedDB.doubleaccFields);
kelondroMapObjects.mapIterator it;
it = db.maps(true, false);

Write Preview
Loading…
Cancel
Save