boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- extended dbtest for comparisment tests

Browse Source 
- added initial space option for eco tables
- used initial space value in initialization of collectionIndex, this should avoid OOM failures" /Volumes/Magneto/dev/workspace/trunk/source/dbtest.java /Volumes/Magneto/dev/workspace/trunk/source/de/anomic/kelondro/kelondroCollectionIndex.java /Volumes/Magneto/dev/workspace/trunk/source/de/anomic/kelondro/kelondroDyn.java /Volumes/Magneto/dev/workspace/trunk/source/de/anomic/kelondro/kelondroEcoTable.java /Volumes/Magneto/dev/workspace/trunk/source/de/anomic/kelondro/kelondroRow.java /Volumes/Magneto/dev/workspace/trunk/source/de/anomic/kelondro/kelondroSplitTable.java /Volumes/Magneto/dev/workspace/trunk/source/de/anomic/plasma/plasmaCrawlBalancer.java /Volumes/Magneto/dev/workspace/trunk/source/de/anomic/plasma/plasmaCrawlStacker.java /Volumes/Magneto/dev/workspace/trunk/source/de/anomic/plasma/plasmaCrawlZURL.java
- added index consistency check (checks for double-occurrences of primary keys in file)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4349 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent 9eb746863d
commit 2f3b2f3481
10 changed files with 229 additions and 140 deletions
Show Stats Download Patch File Download Diff File

301
source/dbtest.java
Unescape View File

@ -7,6 +7,7 @@ import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator;
import java.util.Random; import java.util.Random;
import javax.imageio.ImageIO; import javax.imageio.ImageIO;
@ -93,17 +94,22 @@ public class dbtest {
} }
public static abstract class STJob implements Runnable { public static abstract class STJob implements Runnable {
private final kelondroIndex table; private final kelondroIndex table_test, table_reference;
private final long source; private final long source;
public STJob(final kelondroIndex aTable, final long aSource) { public STJob(final kelondroIndex table_test, final kelondroIndex table_reference, final long aSource) {
this.table = aTable; this.table_test = table_test;
this.table_reference = table_reference;
this.source = aSource; this.source = aSource;
} }
public kelondroIndex getTable() { public kelondroIndex getTable_test() {
return this.table; return this.table_test;
}
public kelondroIndex getTable_reference() {
return this.table_reference;
} }
public abstract void run(); public abstract void run();
@ -114,15 +120,16 @@ public class dbtest {
} }
public static final class WriteJob extends STJob { public static final class WriteJob extends STJob {
public WriteJob(final kelondroIndex aTable, final long aSource) { public WriteJob(final kelondroIndex table_test, final kelondroIndex table_reference, final long aSource) {
super(aTable, aSource); super(table_test, table_reference, aSource);
} }
public void run() { public void run() {
final STEntry entry = new STEntry(this.getSource()); final STEntry entry = new STEntry(this.getSource());
System.out.println("write: " + serverLog.arrayList(entry.getKey(), 0, entry.getKey().length)); System.out.println("write: " + serverLog.arrayList(entry.getKey(), 0, entry.getKey().length));
try { try {
getTable().put(getTable().row().newEntry(new byte[][] { entry.getKey(), entry.getValue() , entry.getValue() })); getTable_test().put(getTable_test().row().newEntry(new byte[][] { entry.getKey(), entry.getValue() , entry.getValue() }));
if (getTable_reference() != null) getTable_reference().put(getTable_test().row().newEntry(new byte[][] { entry.getKey(), entry.getValue() , entry.getValue() }));
} catch (IOException e) { } catch (IOException e) {
System.err.println(e); System.err.println(e);
e.printStackTrace(); e.printStackTrace();
@ -132,15 +139,16 @@ public class dbtest {
} }
public static final class RemoveJob extends STJob { public static final class RemoveJob extends STJob {
public RemoveJob(final kelondroIndex aTable, final long aSource) { public RemoveJob(final kelondroIndex table_test, final kelondroIndex table_reference, final long aSource) {
super(aTable, aSource); super(table_test, table_reference, aSource);
} }
public void run() { public void run() {
final STEntry entry = new STEntry(this.getSource()); final STEntry entry = new STEntry(this.getSource());
System.out.println("remove: " + serverLog.arrayList(entry.getKey(), 0, entry.getKey().length)); System.out.println("remove: " + serverLog.arrayList(entry.getKey(), 0, entry.getKey().length));
try { try {
getTable().remove(entry.getKey(), false); getTable_test().remove(entry.getKey(), false);
if (getTable_reference() != null) getTable_reference().remove(entry.getKey(), false);
} catch (IOException e) { } catch (IOException e) {
System.err.println(e); System.err.println(e);
e.printStackTrace(); e.printStackTrace();
@ -150,24 +158,35 @@ public class dbtest {
} }
public static final class ReadJob extends STJob { public static final class ReadJob extends STJob {
public ReadJob(final kelondroIndex aTable, final long aSource) { public ReadJob(final kelondroIndex table_test, final kelondroIndex table_reference, final long aSource) {
super(aTable, aSource); super(table_test, table_reference, aSource);
} }
public void run() { public void run() {
final STEntry entry = new STEntry(this.getSource()); final STEntry entry = new STEntry(this.getSource());
try { try {
final kelondroRow.Entry entryBytes = getTable().get(entry.getKey()); kelondroRow.Entry entryBytes = getTable_test().get(entry.getKey());
if (entryBytes != null) {
final STEntry dbEntry = new STEntry(entryBytes.getColBytes(0), entryBytes.getColBytes(1));
if (!dbEntry.isValid()) {
System.out.println("INVALID table_test: " + dbEntry);
} /* else {
System.out.println("_VALID_ table_test: " + dbEntry);
getTable().remove(entry.getKey(), true);
} */
}
if (getTable_reference() != null) {
entryBytes = getTable_reference().get(entry.getKey());
if (entryBytes != null) { if (entryBytes != null) {
//System.out.println("ENTRY=" + entryBytes.getColString(1, null));
final STEntry dbEntry = new STEntry(entryBytes.getColBytes(0), entryBytes.getColBytes(1)); final STEntry dbEntry = new STEntry(entryBytes.getColBytes(0), entryBytes.getColBytes(1));
if (!dbEntry.isValid()) { if (!dbEntry.isValid()) {
System.out.println("INVALID: " + dbEntry); System.out.println("INVALID table_reference: " + dbEntry);
} /* else { } /* else {
System.out.println("_VALID_: " + dbEntry); System.out.println("_VALID_ table_reference: " + dbEntry);
getTable().remove(entry.getKey(), true); getTable().remove(entry.getKey(), true);
} */ } */
} }
}
} catch (IOException e) { } catch (IOException e) {
System.err.println(e); System.err.println(e);
e.printStackTrace(); e.printStackTrace();
@ -176,50 +195,114 @@ public class dbtest {
} }
} }
public static kelondroIndex selectTableType(String dbe, String tablename, kelondroRow testRow) throws Exception {
if (dbe.equals("kelondroRowSet")) {
return new kelondroRowSet(testRow, 0);
}
if (dbe.equals("kelondroTree")) {
File tablefile = new File(tablename + ".kelondro.db");
return new kelondroCache(new kelondroTree(tablefile, true, preload, testRow));
}
if (dbe.equals("kelondroFlexTable")) {
File tablepath = new File(tablename).getParentFile();
return new kelondroFlexTable(tablepath, new File(tablename).getName(), preload, testRow, 0, true);
}
if (dbe.equals("kelondroSplitTable")) {
File tablepath = new File(tablename).getParentFile();
return new kelondroSplitTable(tablepath, new File(tablename).getName(), preload, testRow, true);
}
if (dbe.equals("kelondroEcoTable")) {
return new kelondroEcoTable(new File(tablename), testRow, kelondroEcoTable.tailCacheForceUsage, 1000, 0);
}
if (dbe.equals("mysql")) {
return new kelondroSQLTable("mysql", testRow);
}
if (dbe.equals("pgsql")) {
return new kelondroSQLTable("pgsql", testRow);
}
return null;
}
public static boolean checkEquivalence(kelondroIndex test, kelondroIndex reference) throws IOException {
if (reference == null) return true;
if (test.size() == reference.size()) {
System.out.println("* Testing equivalence of test table to reference table, " + test.size() + " entries");
} else {
System.out.println("* Testing equivalence of test table to reference table: FAILED! the tables have different sizes: test.size() = " + test.size() + ", reference.size() = " + reference.size());
return false;
}
boolean eq = true;
kelondroRow.Entry test_entry, reference_entry;
Iterator<kelondroRow.Entry> i = test.rows(true, null);
System.out.println("* Testing now by enumeration over test table");
long ts = System.currentTimeMillis();
while (i.hasNext()) {
test_entry = i.next();
reference_entry = reference.get(test_entry.getPrimaryKeyBytes());
if (!test_entry.equals(reference_entry)) {
System.out.println("* FAILED: test entry with key '" + new String(test_entry.getPrimaryKeyBytes()) + "' has no equivalent entry in reference");
eq = false;
}
}
i = reference.rows(true, null);
System.out.println("* Testing now by enumeration over reference table");
long rs = System.currentTimeMillis();
while (i.hasNext()) {
reference_entry = i.next();
test_entry = test.get(reference_entry.getPrimaryKeyBytes());
if (!test_entry.equals(reference_entry)) {
System.out.println("* FAILED: reference entry with key '" + new String(test_entry.getPrimaryKeyBytes()) + "' has no equivalent entry in test");
eq = false;
}
}
System.out.println("* finished checkEquivalence; test-enumeration = " + ((rs - ts) / 1000) + " seconds, reference-enumeration = "+ ((System.currentTimeMillis() - rs) / 1000) + " seconds");
if (eq) System.out.println("* SUCCESS! the tables are equal."); else System.out.println("* FAILED! the tables are not equal.");
return eq;
}
public static void main(String[] args) { public static void main(String[] args) {
System.setProperty("java.awt.headless", "true"); System.setProperty("java.awt.headless", "true");
String dbe = args[0]; // the database engine System.out.println("*** YaCy Database Test");
String command = args[1]; // test command // print out command line
String tablename = args[2]; // name of test-table boolean assertionenabled = false;
long startup = System.currentTimeMillis(); assert assertionenabled = true;
if (assertionenabled) System.out.println("*** Asserts are enabled"); else System.out.println("*** HINT: YOU SHOULD ENABLE ASSERTS! (include -ea in start arguments");
Runtime runtime = Runtime.getRuntime();
long mb = (runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory()) / 1024 / 1024;
System.out.println("*** RAM = " + mb + " MB");
System.out.print(">java " +
((assertionenabled) ? "-ea " : "") +
((mb > 100) ? ("-Xmx" + (mb + 11) + "m ") : " ") +
"-classpath classes dbtest ");
for (int i = 0; i < args.length; i++) System.out.print(args[i] + " ");
System.out.println();
// read command options
String command = args[0]; // test command
String dbe_test = args[1]; // the database engine that shall be tested
String dbe_reference = args[2]; // the reference database engine (may be null)
String tablename_test = args[3]; // name of test-table
if (dbe_reference.toLowerCase().equals("null")) dbe_reference = null;
String tablename_reference = (dbe_reference == null) ? null : (tablename_test + ".reference");
// start test
long startup = System.currentTimeMillis();
try { try {
kelondroIndex table = null;
// create a memory profiler // create a memory profiler
memprofiler profiler = new memprofiler(1024, 320, 120, new File(tablename + ".profile.png")); memprofiler profiler = new memprofiler(1024, 320, 120, new File(tablename_test + ".profile.png"));
profiler.start(); profiler.start();
// create the database access // create the database access
kelondroRow testRow = new kelondroRow("byte[] key-" + keylength + ", byte[] dummy-" + keylength + ", value-" + valuelength, kelondroBase64Order.enhancedCoder, 0); kelondroRow testRow = new kelondroRow("byte[] key-" + keylength + ", byte[] dummy-" + keylength + ", value-" + valuelength, kelondroBase64Order.enhancedCoder, 0);
if (dbe.equals("kelondroRowSet")) { kelondroIndex table_test = selectTableType(dbe_test, tablename_test, testRow);
table = new kelondroRowSet(testRow, 0); kelondroIndex table_reference = (dbe_reference == null) ? null : selectTableType(dbe_reference, tablename_reference, testRow);
}
if (dbe.equals("kelondroTree")) {
File tablefile = new File(tablename + ".kelondro.db");
table = new kelondroCache(new kelondroTree(tablefile, true, preload, testRow));
}
if (dbe.equals("kelondroFlexTable")) {
File tablepath = new File(tablename).getParentFile();
table = new kelondroFlexTable(tablepath, new File(tablename).getName(), preload, testRow, 0, true);
}
if (dbe.equals("kelondroFlexSplitTable")) {
File tablepath = new File(tablename).getParentFile();
table = new kelondroSplitTable(tablepath, new File(tablename).getName(), preload, testRow, true);
}
if (dbe.equals("kelondroEcoTable")) {
table = new kelondroEcoTable(new File(tablename), testRow, kelondroEcoTable.tailCacheDenyUsage /*kelondroEcoTable.tailCacheForceUsage*/, 100);
}
if (dbe.equals("mysql")) {
table = new kelondroSQLTable("mysql", testRow);
}
if (dbe.equals("pgsql")) {
table = new kelondroSQLTable("pgsql", testRow);
}
long afterinit = System.currentTimeMillis(); long afterinit = System.currentTimeMillis();
System.out.println("Test for db-engine " + dbe + " started to create file " + tablename + " with test " + command); System.out.println("Test for db-engine " + dbe_test + " started to create file " + tablename_test + " with test " + command);
// execute command // execute command
if (command.equals("create")) { if (command.equals("create")) {
@ -230,8 +313,9 @@ public class dbtest {
if (command.equals("fill")) { if (command.equals("fill")) {
// fill database with random entries; // fill database with random entries;
// args: <number-of-entries> <random-startpoint> // args: <number-of-entries> <random-startpoint>
long count = Long.parseLong(args[3]); // example: java -ea -Xmx200m fill kelondroEcoTable kelondroFlexTable filldbtest 50000 0
long randomstart = Long.parseLong(args[4]); long count = Long.parseLong(args[4]);
long randomstart = Long.parseLong(args[5]);
Random random = new Random(randomstart); Random random = new Random(randomstart);
byte[] key; byte[] key;
kelondroProfile ioProfileAcc = new kelondroProfile(); kelondroProfile ioProfileAcc = new kelondroProfile();
@ -239,11 +323,12 @@ public class dbtest {
kelondroProfile[] profiles; kelondroProfile[] profiles;
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
key = randomHash(random); key = randomHash(random);
table.put(table.row().newEntry(new byte[][]{key, key, dummyvalue2})); table_test.put(testRow.newEntry(new byte[][]{key, key, dummyvalue2}));
if (table_reference != null) table_reference.put(testRow.newEntry(new byte[][]{key, key, dummyvalue2}));
if (i % 1000 == 0) { if (i % 1000 == 0) {
System.out.println(i + " entries. "); System.out.println(i + " entries. ");
if (table instanceof kelondroTree) { if (table_test instanceof kelondroTree) {
profiles = ((kelondroTree) table).profiles(); profiles = ((kelondroTree) table_test).profiles();
System.out.println("Cache Delta: " + kelondroProfile.delta(profiles[0], cacheProfileAcc).toString()); System.out.println("Cache Delta: " + kelondroProfile.delta(profiles[0], cacheProfileAcc).toString());
System.out.println("IO Delta: " + kelondroProfile.delta(profiles[1], ioProfileAcc).toString()); System.out.println("IO Delta: " + kelondroProfile.delta(profiles[1], ioProfileAcc).toString());
cacheProfileAcc = (kelondroProfile) profiles[0].clone(); cacheProfileAcc = (kelondroProfile) profiles[0].clone();
@ -253,69 +338,42 @@ public class dbtest {
} }
} }
if (command.equals("benchfill")) {
// fill database with random entries;
// args: <number-of-entries> <random-startpoint>
long count = Long.parseLong(args[3]);
long time = System.currentTimeMillis();
long randomstart = Long.parseLong(args[4]);
Random random = new Random(randomstart);
byte[] key;
for (int i = 0; i < count; i++) {
key = randomHash(random);
table.put(table.row().newEntry(new byte[][]{key, key, dummyvalue2}));
if (i % 10000 == 0) {
System.out.println(System.currentTimeMillis() - time);
}
}
}
if (command.equals("read")) { if (command.equals("read")) {
// read the database and compare with random entries; // read the database and compare with random entries;
// args: <number-of-entries> <random-startpoint> // args: <number-of-entries> <random-startpoint>
long count = Long.parseLong(args[3]); long count = Long.parseLong(args[4]);
long randomstart = Long.parseLong(args[4]); long randomstart = Long.parseLong(args[5]);
Random random = new Random(randomstart); Random random = new Random(randomstart);
kelondroRow.Entry entry; kelondroRow.Entry entry;
byte[] key; byte[] key;
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
key = randomHash(random); key = randomHash(random);
entry = table.get(key); entry = table_test.get(key);
if (entry == null) System.out.println("missing value for entry " + new String(key)); else if (entry == null)
if (!(new String(entry.getColBytes(1)).equals(new String(key)))) System.out.println("wrong value for entry " + new String(key) + ": " + new String(entry.getColBytes(1))); System.out.println("missing value for entry " + new String(key) + " in test table");
else
if (!(new String(entry.getColBytes(1)).equals(new String(key)))) System.out.println("wrong value for entry " + new String(key) + ": " + new String(entry.getColBytes(1)) + " in test table");
if (table_reference != null) {
entry = table_reference.get(key);
if (entry == null)
System.out.println("missing value for entry " + new String(key) + " in reference table");
else
if (!(new String(entry.getColBytes(1)).equals(new String(key)))) System.out.println("wrong value for entry " + new String(key) + ": " + new String(entry.getColBytes(1)) + " in reference table");
}
if (i % 1000 == 0) { if (i % 1000 == 0) {
System.out.println(i + " entries processed so far."); System.out.println(i + " entries processed so far.");
} }
} }
} }
if (command.equals("benchread")) {
// read the database and compare with random entries;
// args: <number-of-entries> <random-startpoint>
long count = Long.parseLong(args[3]);
long time = System.currentTimeMillis();
long randomstart = Long.parseLong(args[4]);
Random random = new Random(randomstart);
kelondroRow.Entry entry;
byte[] key;
for (int i = 0; i < count; i++) {
key = randomHash(random);
entry = table.get(key);
if (entry == null) System.out.println("missing value for entry " + new String(key)); else
if (!(new String(entry.getColBytes(1)).equals(new String(key)))) System.out.println("wrong value for entry " + new String(key) + ": " + new String(entry.getColBytes(1)));
if (i % 10000 == 0) {
System.out.println(System.currentTimeMillis() - time);
}
}
}
if (command.equals("ramtest")) { if (command.equals("ramtest")) {
// fill database with random entries and delete them again; // fill database with random entries and delete them again;
// this is repeated without termination; after each loop // this is repeated without termination; after each loop
// the current ram is printed out // the current ram is printed out
// args: <number-of-entries> <random-startpoint> // args: <number-of-entries> <random-startpoint>
long count = Long.parseLong(args[3]); long count = Long.parseLong(args[4]);
long randomstart = Long.parseLong(args[4]); long randomstart = Long.parseLong(args[5]);
byte[] key; byte[] key;
Random random; Random random;
long start, write, remove; long start, write, remove;
@ -326,7 +384,7 @@ public class dbtest {
start = System.currentTimeMillis(); start = System.currentTimeMillis();
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
key = randomHash(random); key = randomHash(random);
table.put(table.row().newEntry(new byte[][]{key, key, dummyvalue2})); table_test.put(table_test.row().newEntry(new byte[][]{key, key, dummyvalue2}));
} }
write = System.currentTimeMillis() - start; write = System.currentTimeMillis() - start;
@ -335,7 +393,7 @@ public class dbtest {
start = System.currentTimeMillis(); start = System.currentTimeMillis();
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
key = randomHash(random); key = randomHash(random);
table.remove(key, false); table_test.remove(key, false);
} }
remove = System.currentTimeMillis() - start; remove = System.currentTimeMillis() - start;
@ -355,8 +413,8 @@ public class dbtest {
if (command.equals("list")) { if (command.equals("list")) {
kelondroCloneableIterator<kelondroRow.Entry> i = null; kelondroCloneableIterator<kelondroRow.Entry> i = null;
if (table instanceof kelondroTree) i = ((kelondroTree) table).rows(true, null); if (table_test instanceof kelondroTree) i = ((kelondroTree) table_test).rows(true, null);
if (table instanceof kelondroSQLTable) i = ((kelondroSQLTable) table).rows(true, null); if (table_test instanceof kelondroSQLTable) i = ((kelondroSQLTable) table_test).rows(true, null);
kelondroRow.Entry row; kelondroRow.Entry row;
while (i.hasNext()) { while (i.hasNext()) {
row = i.next(); row = i.next();
@ -368,7 +426,7 @@ public class dbtest {
if (command.equals("stressThreaded")) { if (command.equals("stressThreaded")) {
// //
// args: <number-of-writes> <number-of-reads-per-write> <random-startpoint> // args: <number-of-writes> <number-of-reads-per-write> <random-startpoint>
// example: kelondroFlexTable stressThreaded /Users/admin/dbtest 500 50 0 // example: java -ea -classpath classes dbtest stressThreaded kelondroEcoTable kelondroFlexTable stressthreadedtest 500 50 0
/* result with svn 4346 /* result with svn 4346
kelondroFlex: kelondroFlex:
removed: 70, size of jcontrol set: 354, size of kcontrol set: 354 removed: 70, size of jcontrol set: 354, size of kcontrol set: 354
@ -379,9 +437,9 @@ public class dbtest {
Database size = 354 unique entries. Database size = 354 unique entries.
Execution time: open=1324, command=34032, close=1, total=35357 Execution time: open=1324, command=34032, close=1, total=35357
*/ */
long writeCount = Long.parseLong(args[3]); long writeCount = Long.parseLong(args[4]);
long readCount = Long.parseLong(args[4]); long readCount = Long.parseLong(args[5]);
long randomstart = Long.parseLong(args[5]); long randomstart = Long.parseLong(args[6]);
final Random random = new Random(randomstart); final Random random = new Random(randomstart);
long r; long r;
Long R; Long R;
@ -393,10 +451,10 @@ public class dbtest {
r = Math.abs(random.nextLong() % 1000); r = Math.abs(random.nextLong() % 1000);
jcontrol.add(new Long(r)); jcontrol.add(new Long(r));
kcontrol.putb((int) r, "x".getBytes()); kcontrol.putb((int) r, "x".getBytes());
serverInstantThread.oneTimeJob(new WriteJob(table, r), 0, 50); serverInstantThread.oneTimeJob(new WriteJob(table_test, table_reference, r), 0, 50);
if (random.nextLong() % 5 == 0) ra.add(new Long(r)); if (random.nextLong() % 5 == 0) ra.add(new Long(r));
for (int j = 0; j < readCount; j++) { for (int j = 0; j < readCount; j++) {
serverInstantThread.oneTimeJob(new ReadJob(table, random.nextLong() % writeCount), random.nextLong() % 1000, 20); serverInstantThread.oneTimeJob(new ReadJob(table_test, table_reference, random.nextLong() % writeCount), random.nextLong() % 1000, 20);
} }
if ((ra.size() > 0) && (random.nextLong() % 7 == 0)) { if ((ra.size() > 0) && (random.nextLong() % 7 == 0)) {
rc++; rc++;
@ -405,7 +463,7 @@ public class dbtest {
jcontrol.remove(R); jcontrol.remove(R);
kcontrol.removeb((int) R.longValue()); kcontrol.removeb((int) R.longValue());
System.out.println("remove: " + R.longValue()); System.out.println("remove: " + R.longValue());
serverInstantThread.oneTimeJob(new RemoveJob(table, ((Long) ra.remove(p)).longValue()), 0, 50); serverInstantThread.oneTimeJob(new RemoveJob(table_test, table_reference, ((Long) ra.remove(p)).longValue()), 0, 50);
} }
} }
System.out.println("removed: " + rc + ", size of jcontrol set: " + jcontrol.size() + ", size of kcontrol set: " + kcontrol.size()); System.out.println("removed: " + rc + ", size of jcontrol set: " + jcontrol.size() + ", size of kcontrol set: " + kcontrol.size());
@ -419,9 +477,9 @@ public class dbtest {
if (command.equals("stressSequential")) { if (command.equals("stressSequential")) {
// //
// args: <number-of-writes> <number-of-reads> <random-startpoint> // args: <number-of-writes> <number-of-reads> <random-startpoint>
long writeCount = Long.parseLong(args[3]); long writeCount = Long.parseLong(args[4]);
long readCount = Long.parseLong(args[4]); long readCount = Long.parseLong(args[5]);
long randomstart = Long.parseLong(args[5]); long randomstart = Long.parseLong(args[6]);
Random random = new Random(randomstart); Random random = new Random(randomstart);
long r; long r;
Long R; Long R;
@ -434,10 +492,10 @@ public class dbtest {
r = Math.abs(random.nextLong() % 1000); r = Math.abs(random.nextLong() % 1000);
jcontrol.add(new Long(r)); jcontrol.add(new Long(r));
kcontrol.putb((int) r, "x".getBytes()); kcontrol.putb((int) r, "x".getBytes());
new WriteJob(table, r).run(); new WriteJob(table_test, table_reference, r).run();
if (random.nextLong() % 5 == 0) ra.add(new Long(r)); if (random.nextLong() % 5 == 0) ra.add(new Long(r));
for (int j = 0; j < readCount; j++) { for (int j = 0; j < readCount; j++) {
new ReadJob(table, random.nextLong() % writeCount).run(); new ReadJob(table_test, table_reference, random.nextLong() % writeCount).run();
} }
if ((ra.size() > 0) && (random.nextLong() % 7 == 0)) { if ((ra.size() > 0) && (random.nextLong() % 7 == 0)) {
rc++; rc++;
@ -445,7 +503,7 @@ public class dbtest {
R = (Long) ra.get(p); R = (Long) ra.get(p);
jcontrol.remove(R); jcontrol.remove(R);
kcontrol.removeb((int) R.longValue()); kcontrol.removeb((int) R.longValue());
new RemoveJob(table, ((Long) ra.remove(p)).longValue()).run(); new RemoveJob(table_test, table_reference, ((Long) ra.remove(p)).longValue()).run();
} }
} }
try {Thread.sleep(1000);} catch (InterruptedException e) {} try {Thread.sleep(1000);} catch (InterruptedException e) {}
@ -453,15 +511,22 @@ public class dbtest {
} }
long aftercommand = System.currentTimeMillis(); long aftercommand = System.currentTimeMillis();
checkEquivalence(table_test, table_reference);
long afterequiv = System.currentTimeMillis();
// final report // final report
System.out.println("Database size = " + table.size() + " unique entries."); System.out.println("Database size = " + table_test.size() + " unique entries.");
// finally close the database/table // finally close the database/table
table.close(); table_test.close();
if (table_reference != null) table_reference.close();
long afterclose = System.currentTimeMillis(); long afterclose = System.currentTimeMillis();
System.out.println("Execution time: open=" + (afterinit - startup) + ", command=" + (aftercommand - afterinit) + ", close=" + (afterclose - aftercommand) + ", total=" + (afterclose - startup)); System.out.println("Execution time: open=" + (afterinit - startup) +
", command=" + (aftercommand - afterinit) +
", equiv=" + (afterequiv - aftercommand) +
", close=" + (afterclose - afterequiv) +
", total=" + (afterclose - startup));
profiler.terminate(); profiler.terminate();
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();

6
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -160,9 +160,9 @@ public class kelondroCollectionIndex {
// initialize (new generation) index table from file // initialize (new generation) index table from file
if (serverMemory.request(necessaryRAM4fullTable, false)) { if (serverMemory.request(necessaryRAM4fullTable, false)) {
index = new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize); index = new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize, initialSpace);
} else if (serverMemory.request(necessaryRAM4fullIndex, false)) { } else if (serverMemory.request(necessaryRAM4fullIndex, false)) {
index = new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize); index = new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, initialSpace);
} else { } else {
index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRowdef, initialSpace, true); index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRowdef, initialSpace, true);
} }
@ -293,7 +293,7 @@ public class kelondroCollectionIndex {
// open a ecotable // open a ecotable
long records = f.length() / indexRowdef.objectsize; long records = f.length() / indexRowdef.objectsize;
long necessaryRAM4fullTable = minimumRAM4Eco + (indexRowdef.objectsize + 4) * records * 3 / 2; long necessaryRAM4fullTable = minimumRAM4Eco + (indexRowdef.objectsize + 4) * records * 3 / 2;
return new kelondroEcoTable(f, indexRowdef, (serverMemory.request(necessaryRAM4fullTable, false)) ? kelondroEcoTable.tailCacheUsageAuto : kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize); return new kelondroEcoTable(f, indexRowdef, (serverMemory.request(necessaryRAM4fullTable, false)) ? kelondroEcoTable.tailCacheUsageAuto : kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, initialSpace);
} }
} }

4
source/de/anomic/kelondro/kelondroDyn.java
Unescape View File

@ -99,10 +99,10 @@ public class kelondroDyn {
if (file.isDirectory()) { if (file.isDirectory()) {
fbi = new kelondroFlexTable(file.getParentFile(), file.getName(), 10000, rowdef, 0, resetOnFail); fbi = new kelondroFlexTable(file.getParentFile(), file.getName(), 10000, rowdef, 0, resetOnFail);
} else { } else {
fbi = new kelondroEcoTable(file, rowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize); fbi = new kelondroEcoTable(file, rowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize, 0);
} }
} else { } else {
fbi = new kelondroEcoTable(file, rowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize); fbi = new kelondroEcoTable(file, rowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize, 0);
} }
} }
this.index = (useObjectCache) ? (kelondroIndex) new kelondroCache(fbi) : fbi; this.index = (useObjectCache) ? (kelondroIndex) new kelondroCache(fbi) : fbi;

23
source/de/anomic/kelondro/kelondroEcoTable.java
Unescape View File

@ -48,7 +48,6 @@ import de.anomic.server.serverMemory;
* - the access index can be either completely in RAM (kelondroRAMIndex) or it is file-based (kelondroTree) * - the access index can be either completely in RAM (kelondroRAMIndex) or it is file-based (kelondroTree)
* - the content cache can be either a complete RAM-based shadow of the File, or empty. * - the content cache can be either a complete RAM-based shadow of the File, or empty.
* The content cache can also be deleted during run-time, if the available RAM gets too low. * The content cache can also be deleted during run-time, if the available RAM gets too low.
*
*/ */
public class kelondroEcoTable implements kelondroIndex { public class kelondroEcoTable implements kelondroIndex {
@ -66,7 +65,7 @@ public class kelondroEcoTable implements kelondroIndex {
private kelondroRow rowdef, taildef; private kelondroRow rowdef, taildef;
private int buffersize; private int buffersize;
public kelondroEcoTable(File tablefile, kelondroRow rowdef, int useTailCache, int buffersize) { public kelondroEcoTable(File tablefile, kelondroRow rowdef, int useTailCache, int buffersize, int initialSpace) {
this.rowdef = rowdef; this.rowdef = rowdef;
this.buffersize = buffersize; this.buffersize = buffersize;
assert rowdef.primaryKeyIndex == 0; assert rowdef.primaryKeyIndex == 0;
@ -95,28 +94,34 @@ public class kelondroEcoTable implements kelondroIndex {
this.file = new kelondroBufferedEcoFS(new kelondroEcoFS(tablefile, rowdef.objectsize), this.buffersize); this.file = new kelondroBufferedEcoFS(new kelondroEcoFS(tablefile, rowdef.objectsize), this.buffersize);
// initialize index and copy table // initialize index and copy table
int records = file.size(); int records = Math.max(file.size(), initialSpace);
long neededRAM4table = 10 * 1024 * 1024 + records * (rowdef.objectsize + 4) * 3 / 2; long neededRAM4table = 10 * 1024 * 1024 + records * (rowdef.objectsize + 4) * 3 / 2;
table = ((useTailCache == tailCacheForceUsage) || table = ((useTailCache == tailCacheForceUsage) ||
((useTailCache == tailCacheUsageAuto) && (serverMemory.request(neededRAM4table, true)))) ? ((useTailCache == tailCacheUsageAuto) && (serverMemory.request(neededRAM4table, true)))) ?
new kelondroRowSet(taildef, records + 1) : null; new kelondroRowSet(taildef, records) : null;
index = new kelondroBytesIntMap(rowdef.primaryKeyLength, rowdef.objectOrder, records + 1); index = new kelondroBytesIntMap(rowdef.primaryKeyLength, rowdef.objectOrder, records);
System.out.println("*** DEBUG: EcoTable " + tablefile.toString() + " has table copy " + ((table == null) ? "DISABLED" : "ENABLED"));
// read all elements from the file into the copy table // read all elements from the file into the copy table
byte[] record = new byte[rowdef.objectsize]; byte[] record = new byte[rowdef.objectsize];
byte[] key = new byte[rowdef.primaryKeyLength]; byte[] key = new byte[rowdef.primaryKeyLength];
for (int i = 0; i < records; i++) { int fs = file.size();
for (int i = 0; i < fs; i++) {
// read entry // read entry
file.get(i, record, 0); file.get(i, record, 0);
// write the key into the index table // write the key into the index table
System.arraycopy(record, 0, key, 0, rowdef.primaryKeyLength); System.arraycopy(record, 0, key, 0, rowdef.primaryKeyLength);
index.addi(key, i); index.addi(key, i);
//index.puti(key, i);
// write the tail into the table // write the tail into the table
if (table != null) table.addUnique(taildef.newEntry(record, rowdef.primaryKeyLength, true)); if (table != null) table.addUnique(taildef.newEntry(record, rowdef.primaryKeyLength, true));
} }
// check consistency
ArrayList<Integer[]> doubles = index.removeDoubles();
if (doubles.size() > 0) {
System.out.println("DEBUG: WARNING - EcoTable " + tablefile + " has " + doubles.size() + " doubles");
}
} catch (FileNotFoundException e) { } catch (FileNotFoundException e) {
// should never happen // should never happen
e.printStackTrace(); e.printStackTrace();
@ -507,7 +512,7 @@ public class kelondroEcoTable implements kelondroIndex {
public static kelondroIndex testTable(File f, String testentities, int testcase) throws IOException { public static kelondroIndex testTable(File f, String testentities, int testcase) throws IOException {
if (f.exists()) f.delete(); if (f.exists()) f.delete();
kelondroRow rowdef = new kelondroRow("byte[] a-4, byte[] b-4", kelondroNaturalOrder.naturalOrder, 0); kelondroRow rowdef = new kelondroRow("byte[] a-4, byte[] b-4", kelondroNaturalOrder.naturalOrder, 0);
kelondroIndex tt = new kelondroEcoTable(f, rowdef, testcase, 100); kelondroIndex tt = new kelondroEcoTable(f, rowdef, testcase, 100, 0);
byte[] b; byte[] b;
kelondroRow.Entry row = rowdef.newEntry(); kelondroRow.Entry row = rowdef.newEntry();
for (int i = 0; i < testentities.length(); i++) { for (int i = 0; i < testentities.length(); i++) {

6
source/de/anomic/kelondro/kelondroFlexTable.java
Unescape View File

@ -109,6 +109,12 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
index = new kelondroBytesIntMap(ki); index = new kelondroBytesIntMap(ki);
assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
} }
// check consistency
ArrayList<Integer[]> doubles = index.removeDoubles();
if (doubles.size() > 0) {
System.out.println("DEBUG: WARNING - FlexTable " + newpath.toString() + " has " + doubles.size() + " doubles");
}
// assign index to wrapper // assign index to wrapper
description = "stt=" + Long.toString(System.currentTimeMillis() - start) + ";"; description = "stt=" + Long.toString(System.currentTimeMillis() - start) + ";";

13
source/de/anomic/kelondro/kelondroRow.java
Unescape View File

@ -321,10 +321,22 @@ public final class kelondroRow {
} }
public final int compareTo(Entry o) { public final int compareTo(Entry o) {
// compares only the content of the primary key
if (objectOrder == null) throw new kelondroException("objects cannot be compared, no order given"); if (objectOrder == null) throw new kelondroException("objects cannot be compared, no order given");
return objectOrder.compare(this.getPrimaryKeyBytes(), ((Entry) o).getPrimaryKeyBytes()); return objectOrder.compare(this.getPrimaryKeyBytes(), ((Entry) o).getPrimaryKeyBytes());
} }
public final boolean equals(Entry otherEntry) {
// compares the content of the complete entry
byte[] t = this.bytes();
byte[] o = otherEntry.bytes();
if (o.length != t.length) return false;
for (int i = 0; i < t.length; i++) {
if (t[i] != o[i]) return false;
}
return true;
}
public final byte[] bytes() { public final byte[] bytes() {
if ((offset == 0) && (rowinstance.length == objectsize)) { if ((offset == 0) && (rowinstance.length == objectsize)) {
return rowinstance; return rowinstance;
@ -630,6 +642,7 @@ public final class kelondroRow {
public final boolean equals(kelondroRow otherRow) { public final boolean equals(kelondroRow otherRow) {
if (this.objectsize != otherRow.objectsize) return false; if (this.objectsize != otherRow.objectsize) return false;
if (this.columns() != otherRow.columns()) return false;
for (int i = 0; i < otherRow.row.length; i++) { for (int i = 0; i < otherRow.row.length; i++) {
if (!(this.row[i].equals(otherRow.row[i]))) return false; if (!(this.row[i].equals(otherRow.row[i]))) return false;
} }

8
source/de/anomic/kelondro/kelondroSplitTable.java
Unescape View File

@ -119,7 +119,7 @@ public class kelondroSplitTable implements kelondroIndex {
// this is a kelonodroFlex table // this is a kelonodroFlex table
table = new kelondroCache(new kelondroFlexTable(path, maxf, preloadTime, rowdef, 0, resetOnFail)); table = new kelondroCache(new kelondroFlexTable(path, maxf, preloadTime, rowdef, 0, resetOnFail));
} else { } else {
table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize); table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0);
} }
tables.put(date, table); tables.put(date, table);
} }
@ -229,13 +229,13 @@ public class kelondroSplitTable implements kelondroIndex {
table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef, 0, true); table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef, 0, true);
} else { } else {
// open a eco table // open a eco table
table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize); table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0);
} }
} else { } else {
// make new table // make new table
if (serverMemory.request(minimumRAM4Eco, true)) { if (serverMemory.request(minimumRAM4Eco, true)) {
// enough memory for a ecoTable // enough memory for a ecoTable
table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize); table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0);
} else { } else {
// use the flex table // use the flex table
table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef, 0, true); table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef, 0, true);
@ -273,7 +273,7 @@ public class kelondroSplitTable implements kelondroIndex {
// make new table // make new table
if (serverMemory.request(minimumRAM4Eco, true)) { if (serverMemory.request(minimumRAM4Eco, true)) {
// enough memory for a ecoTable // enough memory for a ecoTable
table = new kelondroEcoTable(new File(path, tablename + "." + suffix), rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize); table = new kelondroEcoTable(new File(path, tablename + "." + suffix), rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0);
} else { } else {
// use the flex table // use the flex table
table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef, 0, true); table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef, 0, true);

2
source/de/anomic/plasma/plasmaCrawlBalancer.java
Unescape View File

@ -142,7 +142,7 @@ public class plasmaCrawlBalancer {
private void openFileIndex() { private void openFileIndex() {
cacheStacksPath.mkdirs(); cacheStacksPath.mkdirs();
urlFileIndex = new kelondroEcoTable(new File(cacheStacksPath, stackname + indexSuffix), plasmaCrawlEntry.rowdef, (fullram) ? kelondroEcoTable.tailCacheUsageAuto : kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize); urlFileIndex = new kelondroEcoTable(new File(cacheStacksPath, stackname + indexSuffix), plasmaCrawlEntry.rowdef, (fullram) ? kelondroEcoTable.tailCacheUsageAuto : kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0);
} }
private void resetFileIndex() { private void resetFileIndex() {

4
source/de/anomic/plasma/plasmaCrawlStacker.java
Unescape View File

@ -302,7 +302,7 @@ public final class plasmaCrawlStacker extends Thread {
cacheStacksPath.mkdirs(); cacheStacksPath.mkdirs();
File f = new File(cacheStacksPath, stackfile); File f = new File(cacheStacksPath, stackfile);
try { try {
this.urlEntryCache = new kelondroEcoTable(f, plasmaCrawlEntry.rowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize); this.urlEntryCache = new kelondroEcoTable(f, plasmaCrawlEntry.rowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize, 0);
//this.urlEntryCache = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime, plasmaCrawlEntry.rowdef, 0, true)); //this.urlEntryCache = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime, plasmaCrawlEntry.rowdef, 0, true));
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
@ -310,7 +310,7 @@ public final class plasmaCrawlStacker extends Thread {
f.delete(); f.delete();
//kelondroFlexTable.delete(cacheStacksPath, newCacheName); //kelondroFlexTable.delete(cacheStacksPath, newCacheName);
try { try {
this.urlEntryCache = new kelondroEcoTable(f, plasmaCrawlEntry.rowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize); this.urlEntryCache = new kelondroEcoTable(f, plasmaCrawlEntry.rowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize, 0);
//this.urlEntryCache = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime, plasmaCrawlEntry.rowdef, 0, true)); //this.urlEntryCache = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime, plasmaCrawlEntry.rowdef, 0, true));
} catch (Exception ee) { } catch (Exception ee) {
ee.printStackTrace(); ee.printStackTrace();

2
source/de/anomic/plasma/plasmaCrawlZURL.java
Unescape View File

@ -69,7 +69,7 @@ public class plasmaCrawlZURL {
if (f.isDirectory()) kelondroFlexTable.delete(cachePath, tablename); else f.delete(); if (f.isDirectory()) kelondroFlexTable.delete(cachePath, tablename); else f.delete();
} }
} }
urlIndex = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize); urlIndex = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize, 0);
//urlIndex = new kelondroFlexTable(cachePath, tablename, -1, rowdef, 0, true); //urlIndex = new kelondroFlexTable(cachePath, tablename, -1, rowdef, 0, true);
} }

Write Preview
Loading…
Cancel
Save