boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fixed datatabase inconsistency bugs

Browse Source 
inserted many debug lines
added a huge number of asserts
extended database test methods


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3579 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 18 years ago
parent ca79362b9d
commit 595ee10468
19 changed files with 323 additions and 218 deletions
Show Stats Download Patch File Download Diff File

65
source/dbtest.java
Unescape View File

@ -11,6 +11,7 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
@ -23,15 +24,18 @@ import de.anomic.kelondro.kelondroCloneableIterator;
import de.anomic.kelondro.kelondroFlexSplitTable;
import de.anomic.kelondro.kelondroFlexTable;
import de.anomic.kelondro.kelondroIndex;
import de.anomic.kelondro.kelondroIntBytesMap;
import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.kelondro.kelondroOrder;
import de.anomic.kelondro.kelondroProfile;
import de.anomic.kelondro.kelondroRow;
import de.anomic.kelondro.kelondroRowSet;
import de.anomic.kelondro.kelondroSplittedTree;
import de.anomic.kelondro.kelondroTree;
import de.anomic.kelondro.kelondroRow.Entry;
import de.anomic.server.serverInstantThread;
import de.anomic.server.serverMemory;
import de.anomic.server.logging.serverLog;
import de.anomic.ymage.ymageChart;
public class dbtest {
@ -50,8 +54,9 @@ public class dbtest {
public static byte[] randomHash(final long r0, final long r1) {
// a long can have 64 bit, but a 12-byte hash can have 6 * 12 = 72 bits
// so we construct a generic Hash using two long values
return (kelondroBase64Order.enhancedCoder.encodeLong(Math.abs(r0), 11).substring(5) +
kelondroBase64Order.enhancedCoder.encodeLong(Math.abs(r1), 11).substring(5)).getBytes();
String s = (kelondroBase64Order.enhancedCoder.encodeLong(Math.abs(r0), 6) +
kelondroBase64Order.enhancedCoder.encodeLong(Math.abs(r1), 6));
return s.getBytes();
}
public static byte[] randomHash(Random r) {
@ -124,6 +129,7 @@ public class dbtest {
public void run() {
final STEntry entry = new STEntry(this.getSource());
System.out.println("write: " + serverLog.arrayList(entry.getKey(), 0, entry.getKey().length));
try {
getTable().put(getTable().row().newEntry(new byte[][] { entry.getKey(), entry.getValue() , entry.getValue() }));
} catch (IOException e) {
@ -141,6 +147,7 @@ public class dbtest {
public void run() {
final STEntry entry = new STEntry(this.getSource());
System.out.println("remove: " + serverLog.arrayList(entry.getKey(), 0, entry.getKey().length));
try {
getTable().remove(entry.getKey());
} catch (IOException e) {
@ -194,6 +201,9 @@ public class dbtest {
// create the database access
kelondroRow testRow = new kelondroRow("byte[] key-" + keylength + ", byte[] dummy-" + keylength + ", value-" + valuelength, kelondroBase64Order.enhancedCoder, 0);
if (dbe.equals("kelondroRowSet")) {
table = new kelondroRowSet(testRow, 0);
}
if (dbe.equals("kelondroTree")) {
File tablefile = new File(tablename + ".kelondro.db");
table = new kelondroCache(new kelondroTree(tablefile, true, preload, testRow), true, false);
@ -376,21 +386,31 @@ public class dbtest {
long randomstart = Long.parseLong(args[5]);
final Random random = new Random(randomstart);
long r;
int p;
Long R;
int p, rc=0;
ArrayList ra = new ArrayList();
HashSet jcontrol = new HashSet();
kelondroIntBytesMap kcontrol = new kelondroIntBytesMap(1, 0);
for (int i = 0; i < writeCount; i++) {
r = random.nextLong() % 1000;
r = Math.abs(random.nextLong() % 1000);
jcontrol.add(new Long(r));
kcontrol.putb((int) r, "x".getBytes());
serverInstantThread.oneTimeJob(new WriteJob(table, r), 0, 50);
if (random.nextLong() % 5 == 0) ra.add(new Long(r));
for (int j = 0; j < readCount; j++) {
serverInstantThread.oneTimeJob(new ReadJob(table, random.nextLong() % writeCount), random.nextLong() % 1000, 20);
}
if ((ra.size() > 0) && (random.nextLong() % 7 == 0)) {
rc++;
p = Math.abs(random.nextInt()) % ra.size();
System.out.println("remove: " + ((Long) ra.get(p)).longValue());
R = (Long) ra.get(p);
jcontrol.remove(R);
kcontrol.removeb((int) R.longValue());
System.out.println("remove: " + R.longValue());
serverInstantThread.oneTimeJob(new RemoveJob(table, ((Long) ra.remove(p)).longValue()), 0, 50);
}
}
System.out.println("removed: " + rc + ", size of jcontrol set: " + jcontrol.size() + ", size of kcontrol set: " + kcontrol.size());
while (serverInstantThread.instantThreadCounter > 0) {
try {Thread.sleep(1000);} catch (InterruptedException e) {} // wait for all tasks to finish
System.out.println("count: " + serverInstantThread.instantThreadCounter + ", jobs: " + serverInstantThread.jobs.toString());
@ -404,13 +424,34 @@ public class dbtest {
long writeCount = Long.parseLong(args[3]);
long readCount = Long.parseLong(args[4]);
long randomstart = Long.parseLong(args[5]);
final Random random = new Random(randomstart);
Random random = new Random(randomstart);
long r;
Long R;
int p, rc=0;
ArrayList ra = new ArrayList();
HashSet jcontrol = new HashSet();
kelondroIntBytesMap kcontrol = new kelondroIntBytesMap(1, 0);
for (int i = 0; i < writeCount; i++) {
new WriteJob(table, i).run();
//if (i == 30) random = new Random(randomstart);
r = Math.abs(random.nextLong() % 1000);
jcontrol.add(new Long(r));
kcontrol.putb((int) r, "x".getBytes());
new WriteJob(table, r).run();
if (random.nextLong() % 5 == 0) ra.add(new Long(r));
for (int j = 0; j < readCount; j++) {
new ReadJob(table, random.nextLong() % writeCount).run();
}
if ((ra.size() > 0) && (random.nextLong() % 7 == 0)) {
rc++;
p = Math.abs(random.nextInt()) % ra.size();
R = (Long) ra.get(p);
jcontrol.remove(R);
kcontrol.removeb((int) R.longValue());
new RemoveJob(table, ((Long) ra.remove(p)).longValue()).run();
}
}
try {Thread.sleep(1000);} catch (InterruptedException e) {}
System.out.println("removed: " + rc + ", size of jcontrol set: " + jcontrol.size() + ", size of kcontrol set: " + kcontrol.size());
}
long aftercommand = System.currentTimeMillis();
@ -418,10 +459,7 @@ public class dbtest {
System.out.println("Database size = " + table.size() + " unique entries.");
// finally close the database/table
if (table instanceof kelondroTree) ((kelondroTree) table).close();
if (table instanceof kelondroFlexTable) ((kelondroFlexTable) table).close();
if (table instanceof kelondroSplittedTree) ((kelondroSplittedTree) table).close();
if (table instanceof dbTable) ((dbTable)table).close();
table.close();
long afterclose = System.currentTimeMillis();
@ -501,7 +539,7 @@ final class dbTable implements kelondroIndex {
this.theDBConnection = null;
}
public int size() throws IOException {
public int size() {
int size = -1;
try {
String sqlQuery = new String
@ -521,7 +559,8 @@ final class dbTable implements kelondroIndex {
return size;
} catch (Exception e) {
throw new IOException(e.getMessage());
e.printStackTrace();
return -1;
}
}

30
source/de/anomic/kelondro/kelondroBase64Order.java
Unescape View File

@ -299,8 +299,8 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
public final int compare0(byte[] a, int aoffset, int alength, byte[] b, int boffset, int blength) {
if (zero == null) return compares(a, aoffset, alength, b, boffset, blength);
// we have an artificial start point. check all combinations
int az = compares(a, aoffset, alength, zero, 0, zero.length); // -1 if a < z; 0 if a == z; 1 if a > z
int bz = compares(b, boffset, blength, zero, 0, zero.length); // -1 if b < z; 0 if b == z; 1 if b > z
int az = compares(a, aoffset, alength, zero, 0, Math.min(alength, zero.length)); // -1 if a < z; 0 if a == z; 1 if a > z
int bz = compares(b, boffset, blength, zero, 0, Math.min(blength, zero.length)); // -1 if b < z; 0 if b == z; 1 if b > z
if ((az == 0) && (bz == 0)) return 0;
if (az == 0) return -1;
if (bz == 0) return 1;
@ -315,32 +315,28 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
int i = 0;
final int al = Math.min(alength, a.length - aoffset);
final int bl = Math.min(blength, b.length - boffset);
final int len = (al > bl) ? bl : al;
if (al > bl) return 1;
if (al < bl) return -1;
byte ac, bc;
byte acc, bcc;
while (i < len) {
assert (i + aoffset < a.length) : "i = " + i + ", aoffset = " + aoffset + ", a.length = " + a.length + ", a = " + serverLog.arrayList(a, aoffset, len);
assert (i + boffset < b.length) : "i = " + i + ", boffset = " + boffset + ", b.length = " + b.length + ", b = " + serverLog.arrayList(b, boffset, len);
while (i < al) {
assert (i + aoffset < a.length) : "i = " + i + ", aoffset = " + aoffset + ", a.length = " + a.length + ", a = " + serverLog.arrayList(a, aoffset, al);
assert (i + boffset < b.length) : "i = " + i + ", boffset = " + boffset + ", b.length = " + b.length + ", b = " + serverLog.arrayList(b, boffset, al);
ac = a[aoffset + i];
assert (ac >= 0) && (ac < 128) : "ac = " + ac + ", a = " + serverLog.arrayList(a, aoffset, len);
assert (ac >= 0) && (ac < 128) : "ac = " + ac + ", a = " + serverLog.arrayList(a, aoffset, al);
bc = b[boffset + i];
assert (bc >= 0) && (bc < 128) : "bc = " + bc + ", b = " + serverLog.arrayList(b, boffset, len);
if ((ac == 0) && (bc == 0)) return 0; // zero-terminated length
assert (bc >= 0) && (bc < 128) : "bc = " + bc + ", b = " + serverLog.arrayList(b, boffset, al);
acc = ahpla[ac];
assert (acc >= 0) : "acc = " + acc + ", a = " + serverLog.arrayList(a, aoffset, len) + "/" + new String(a, aoffset, len) + ", aoffset = 0x" + Integer.toHexString(aoffset) + ", i = " + i + "\n" + serverLog.table(a, 16, aoffset);
assert (acc >= 0) : "acc = " + acc + ", a = " + serverLog.arrayList(a, aoffset, al) + "/" + new String(a, aoffset, al) + ", aoffset = 0x" + Integer.toHexString(aoffset) + ", i = " + i + "\n" + serverLog.table(a, 16, aoffset);
bcc = ahpla[bc];
assert (bcc >= 0) : "bcc = " + bcc + ", b = " + serverLog.arrayList(b, boffset, len) + "/" + new String(b, boffset, len) + ", boffset = 0x" + Integer.toHexString(boffset) + ", i = " + i + "\n" + serverLog.table(b, 16, boffset);
assert (bcc >= 0) : "bcc = " + bcc + ", b = " + serverLog.arrayList(b, boffset, al) + "/" + new String(b, boffset, al) + ", boffset = 0x" + Integer.toHexString(boffset) + ", i = " + i + "\n" + serverLog.table(b, 16, boffset);
if (acc > bcc) return 1;
if (acc < bcc) return -1;
// else the bytes are equal and it may go on yet undecided
i++;
}
// check if we have a zero-terminated equality
if ((i == al) && (i < bl) && (b[i + boffset] == 0)) return 0;
if ((i == bl) && (i < al) && (a[i + aoffset] == 0)) return 0;
// no, decide by length
if (al > bl) return 1;
if (al < bl) return -1;
// no, they are equal
// they are equal
return 0;
}

4
source/de/anomic/kelondro/kelondroBytesIntMap.java
Unescape View File

@ -49,6 +49,7 @@ public class kelondroBytesIntMap {
}
public synchronized int puti(byte[] key, int i) throws IOException {
assert i >= 0 : "i = " + i;
assert (key != null);
//assert (!(serverLog.allZero(key)));
kelondroRow.Entry newentry = ki.row().newEntry();
@ -60,6 +61,7 @@ public class kelondroBytesIntMap {
}
public synchronized void addi(byte[] key, int i) throws IOException {
assert i >= 0 : "i = " + i;
assert (key != null);
//assert (!(serverLog.allZero(key)));
kelondroRow.Entry newentry = ki.row().newEntry();
@ -87,7 +89,7 @@ public class kelondroBytesIntMap {
return (int) indexentry.getColLong(1);
}
public synchronized int size() throws IOException {
public synchronized int size() {
return ki.size();
}

2
source/de/anomic/kelondro/kelondroCache.java
Unescape View File

@ -638,7 +638,7 @@ public class kelondroCache implements kelondroIndex {
return index.rows(up, firstKey);
}
public int size() throws IOException {
public int size() {
return index.size() + ((writeBufferUnique == null) ? 0 : writeBufferUnique.size());
}

2
source/de/anomic/kelondro/kelondroFlexSplitTable.java
Unescape View File

@ -137,7 +137,7 @@ public class kelondroFlexSplitTable implements kelondroIndex {
return new String(suffix);
}
public int size() throws IOException {
public int size() {
Iterator i = tables.values().iterator();
int s = 0;
while (i.hasNext()) {

155
source/de/anomic/kelondro/kelondroFlexTable.java
Unescape View File

@ -44,7 +44,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
private static TreeMap tableTracker = new TreeMap();
// class objects
protected kelondroBytesIntMap ROindex, RWindex;
protected kelondroBytesIntMap index;
private boolean RAMIndex;
public kelondroFlexTable(File path, String tablename, long preloadTime, kelondroRow rowdef, boolean resetOnFail) {
@ -91,8 +91,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
+ " index entries initialized and sorted from "
+ super.col[0].size() + " keys.");
RAMIndex = true;
ROindex = new kelondroBytesIntMap(ki);
RWindex = new kelondroBytesIntMap(new kelondroRowSet(new kelondroRow(new kelondroColumn[]{super.row().column(0), new kelondroColumn("int c-4 {b256}")}, super.rowdef.objectOrder, super.rowdef.primaryKey), 100));
index = new kelondroBytesIntMap(ki);
tableTracker.put(this.filename(), this);
} else {
// too less ram for a ram index
@ -111,8 +110,9 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
System.out.println(ki.size() + " entries indexed from " + super.col[0].size() + " keys.");
RAMIndex = false;
}
ROindex = null;
RWindex = new kelondroBytesIntMap(ki);
index = new kelondroBytesIntMap(ki);
assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
}
// assign index to wrapper
description = "stt=" + Long.toString(System.currentTimeMillis() - start) + ";";
@ -120,9 +120,8 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
} catch (IOException e) {
if (resetOnFail) {
RAMIndex = true;
ROindex = null;
try {
RWindex = new kelondroBytesIntMap(new kelondroRowSet(new kelondroRow(new kelondroColumn[]{super.row().column(0), new kelondroColumn("int c-4 {b256}")}, super.rowdef.objectOrder, super.rowdef.primaryKey), 100));
index = new kelondroBytesIntMap(new kelondroRowSet(new kelondroRow(new kelondroColumn[]{super.row().column(0), new kelondroColumn("int c-4 {b256}")}, super.rowdef.objectOrder, super.rowdef.primaryKey), 100));
} catch (IOException e1) {
throw new kelondroException(e1.getMessage());
}
@ -135,8 +134,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
public void reset() throws IOException {
super.reset();
RAMIndex = true;
ROindex = null;
RWindex = new kelondroBytesIntMap(new kelondroRowSet(new kelondroRow(new kelondroColumn[]{super.row().column(0), new kelondroColumn("int c-4 {b256}")}, super.rowdef.objectOrder, super.rowdef.primaryKey), 100));
index = new kelondroBytesIntMap(new kelondroRowSet(new kelondroRow(new kelondroColumn[]{super.row().column(0), new kelondroColumn("int c-4 {b256}")}, super.rowdef.objectOrder, super.rowdef.primaryKey), 100));
}
public static int staticSize(File path, String tablename) {
@ -155,7 +153,8 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
// it is not recommended to implement or use a has predicate unless
// it can be ensured that it causes no IO
if ((kelondroRecords.debugmode) && (RAMIndex != true)) serverLog.logWarning("kelondroFlexTable", "RAM index warning in file " + super.tablename);
return (RWindex.geti(key) >= 0) || ((ROindex != null) && (ROindex.geti(key) >= 0));
assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
return index.geti(key) >= 0;
}
private kelondroIndex initializeRamIndex() {
@ -175,6 +174,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
indexentry = ri.row().newEntry();
indexentry.setCol(0, key);
indexentry.setCol(1, i);
//System.out.println("ENTRY: " + serverLog.arrayList(indexentry.bytes(), 0, indexentry.objectsize()));
ri.addUnique(indexentry);
if ((i % 10000) == 0) {
System.out.print('.');
@ -184,6 +184,9 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
System.out.print(" -ordering- ");
System.out.flush();
ri.sort();
int sbu = ri.size();
ri.uniq();
if (ri.size() != sbu) serverLog.logSevere("kelondroFlexTable.initializeRamIndex: " + tablename, "; size before uniq = " + sbu + ", after uniq = " + ri.size());
return ri;
}
@ -217,20 +220,17 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
}
public synchronized kelondroRow.Entry get(byte[] key) throws IOException {
int pos = RWindex.geti(key);
if (ROindex != null) {
if (pos < 0) {
pos = ROindex.geti(key);
} else {
assert ROindex.geti(key) < 0;
}
}
if (pos < 0) return null;
// i may be greater than this.size(), because this table may have deleted entries
// the deleted entries are subtracted from the 'real' tablesize, so the size may be
// smaller than an index to a row entry
return super.get(pos);
}
int pos = index.geti(key);
assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
if (pos < 0) return null;
// i may be greater than this.size(), because this table may have deleted entries
// the deleted entries are subtracted from the 'real' tablesize,
// so the size may be smaller than an index to a row entry
kelondroRow.Entry result = super.get(pos);
assert result != null;
assert rowdef.objectOrder.compare(result.getColBytes(rowdef.primaryKey), key) == 0 : "key and row does not match; key = " + serverLog.arrayList(key, 0, key.length) + " row.key = " + serverLog.arrayList(result.getColBytes(rowdef.primaryKey), 0, rowdef.width(rowdef.primaryKey));
return result;
}
public synchronized void putMultiple(List rows, Date entryDate) throws IOException {
// put a list of entries in a ordered way.
@ -241,11 +241,11 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
byte[] key;
TreeMap old_rows_ordered = new TreeMap();
ArrayList new_rows_sequential = new ArrayList();
assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
while (i.hasNext()) {
row = (kelondroRow.Entry) i.next();
key = row.getColBytes(0);
pos = RWindex.geti(key);
if ((pos < 0) && (ROindex != null)) pos = ROindex.geti(key);
pos = index.geti(key);
if (pos < 0) {
new_rows_sequential.add(row);
} else {
@ -257,10 +257,12 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
// write new entries to index
addUniqueMultiple(new_rows_sequential, entryDate);
assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
}
public synchronized kelondroRow.Entry put(kelondroRow.Entry row, Date entryDate) throws IOException {
return put(row);
assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
return put(row);
}
public synchronized kelondroRow.Entry put(kelondroRow.Entry row) throws IOException {
@ -268,24 +270,43 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
assert (!(serverLog.allZero(row.getColBytes(0))));
assert row.objectsize() <= this.rowdef.objectsize;
byte[] key = row.getColBytes(0);
int pos = RWindex.geti(key);
if ((pos < 0) && (ROindex != null)) pos = ROindex.geti(key);
int pos = index.geti(key);
if (pos < 0) {
RWindex.puti(key, super.add(row));
pos = super.add(row);
index.puti(key, pos);
assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
return null;
}
//System.out.println("row.key=" + serverLog.arrayList(row.bytes(), 0, row.objectsize()));
kelondroRow.Entry oldentry = super.get(pos);
assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
if (oldentry == null) {
serverLog.logSevere("kelondroFlexTable", "put(): index failure; the index pointed to a cell which is empty. content.size() = " + this.size() + ", index.size() = " + ((index == null) ? 0 : index.size()));
// patch bug ***** FIND CAUSE! (see also: remove)
int oldindex = index.removei(key);
assert oldindex >= 0;
assert index.geti(key) == -1;
// here is this.size() > index.size() because of remove operation above
index.puti(key, super.add(row));
assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
return null;
}
assert oldentry != null : "overwrite of empty position " + pos + ", index management must have failed before";
assert rowdef.objectOrder.compare(oldentry.getColBytes(rowdef.primaryKey), key) == 0 : "key and row does not match; key = " + serverLog.arrayList(key, 0, key.length) + " row.key = " + serverLog.arrayList(oldentry.getColBytes(rowdef.primaryKey), 0, rowdef.width(rowdef.primaryKey));
super.set(pos, row);
assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
return oldentry;
}
public synchronized void addUnique(kelondroRow.Entry row, Date entryDate) throws IOException {
addUnique(row);
assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
addUnique(row);
}
public synchronized void addUnique(kelondroRow.Entry row) throws IOException {
assert row.objectsize() == this.rowdef.objectsize;
RWindex.addi(row.getColBytes(0), super.add(row));
assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
index.addi(row.getColBytes(0), super.add(row));
}
public synchronized void addUniqueMultiple(List rows, Date entryDate) throws IOException {
@ -298,49 +319,48 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
Map.Entry entry;
while (i.hasNext()) {
entry = (Map.Entry) i.next();
RWindex.puti((byte[]) entry.getValue(), ((Integer) entry.getKey()).intValue());
index.puti((byte[]) entry.getValue(), ((Integer) entry.getKey()).intValue());
}
assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
}
public synchronized kelondroRow.Entry remove(byte[] key) throws IOException {
int i = RWindex.removei(key);
if (ROindex != null) {
if (i < 0) {
i = ROindex.removei(key); // yes, we are allowed to remove entries from RO partition of the index
} else {
assert ROindex.removei(key) < 0;
}
int i = index.removei(key);
assert (index.geti(key) < 0); // must be deleted
if (i < 0) {
assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
return null;
}
assert (RWindex.removei(key) < 0);
assert (ROindex == null) || (ROindex.removei(key) < 0);
if (i < 0) return null;
kelondroRow.Entry r = super.get(i);
assert r != null; // error
if (r == null) {
serverLog.logSevere("kelondroFlexTable", "remove(): index failure; the index pointed to a cell which is empty. content.size() = " + this.size() + ", index.size() = " + ((index == null) ? 0 : index.size()));
// patch bug ***** FIND CAUSE! (see also: put)
assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
return null;
}
assert r != null : "r == null"; // should be avoided with path above
assert rowdef.objectOrder.compare(r.getColBytes(rowdef.primaryKey), key) == 0 : "key and row does not match; key = " + serverLog.arrayList(key, 0, key.length) + " row.key = " + serverLog.arrayList(r.getColBytes(rowdef.primaryKey), 0, rowdef.width(rowdef.primaryKey));
super.remove(i);
assert super.get(i) == null : "i = " + i + ", get(i) = " + serverLog.arrayList(super.get(i).bytes(), 0, 12);
return r;
assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
return r;
}
public synchronized kelondroRow.Entry removeOne() throws IOException {
int i = RWindex.removeonei();
if ((i < 0) && (ROindex != null)) i = ROindex.removeonei();
int i = index.removeonei();
if (i < 0) return null;
kelondroRow.Entry r;
r = super.get(i);
super.remove(i);
return r;
assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
return r;
}
public synchronized kelondroCloneableIterator rows(boolean up, byte[] firstKey) throws IOException {
if (ROindex == null) return new rowIterator(RWindex, up, firstKey);
if (RWindex == null) return new rowIterator(ROindex, up, firstKey);
return new kelondroMergeIterator(
new rowIterator(ROindex, up, firstKey),
new rowIterator(RWindex, up, firstKey),
row().objectOrder,
kelondroMergeIterator.simpleMerge,
up
);
if (index == null) return new rowIterator(index, up, firstKey);
assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
return new rowIterator(index, up, firstKey);
}
public class rowIterator implements kelondroCloneableIterator {
@ -392,11 +412,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
}
public kelondroProfile profile() {
if (ROindex == null) {
return RWindex.profile();
} else {
return kelondroProfile.consolidate(ROindex.profile(), RWindex.profile());
}
return index.profile();
}
public static final Iterator filenames() {
@ -424,11 +440,9 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
// returns statistical data about this object
HashMap map = new HashMap();
try {
map.put("tableIndexChunkSize", (!RAMIndex) ? "0" : Integer.toString(RWindex.row().objectsize));
map.put("tableROIndexCount", ((!RAMIndex) || (ROindex == null)) ? "0" : Integer.toString(ROindex.size()));
map.put("tableROIndexMem", ((!RAMIndex) || (ROindex == null)) ? "0" : Integer.toString((int) (ROindex.row().objectsize * ROindex.size())));
map.put("tableRWIndexCount", (!RAMIndex) ? "0" : Integer.toString(RWindex.size()));
map.put("tableRWIndexMem", (!RAMIndex) ? "0" : Integer.toString((int) (RWindex.row().objectsize * RWindex.size() * kelondroRowCollection.growfactor)));
map.put("tableIndexChunkSize", (!RAMIndex) ? "0" : Integer.toString(index.row().objectsize));
map.put("tableIndexCount", (!RAMIndex) ? "0" : Integer.toString(index.size()));
map.put("tableIndexMem", (!RAMIndex) ? "0" : Integer.toString((int) (index.row().objectsize * index.size() * kelondroRowCollection.growfactor)));
} catch (IOException e) {
}
return map;
@ -438,8 +452,11 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
if (tableTracker.remove(this.filename) == null) {
serverLog.logWarning("kelondroFlexTable", "close(): file '" + this.filename + "' was not tracked with record tracker.");
}
if (ROindex != null) {ROindex.close(); ROindex = null;}
if (RWindex != null) {RWindex.close(); RWindex = null;}
if ((index != null) && (this.size() != ((index == null) ? 0 : index.size()))) {
serverLog.logSevere("kelondroFlexTable", "close(): inconsistent content/index size. content.size() = " + this.size() + ", index.size() = " + ((index == null) ? 0 : index.size()));
}
if (index != null) {index.close(); index = null;}
super.close();
}

15
source/de/anomic/kelondro/kelondroFlexWidthArray.java
Unescape View File

@ -231,7 +231,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
rowentry = (kelondroRow.Entry) entry.getValue();
assert rowentry.objectsize() == this.rowdef.objectsize;
e = col[c].row().newEntry(rowentry.bytes(), rowdef.colstart[c]);
e = col[c].row().newEntry(rowentry.bytes(), rowdef.colstart[c], false);
col[c].set(index, e);
}
c = c + col[c].row().columns();
@ -244,7 +244,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
kelondroRow.Entry e;
byte[] reb = rowentry.bytes();
while (c < rowdef.columns()) {
e = col[c].row().newEntry(reb, rowdef.colstart[c]);
e = col[c].row().newEntry(reb, rowdef.colstart[c], false);
col[c].set(index, e);
c = c + col[c].row().columns();
}
@ -252,16 +252,13 @@ public class kelondroFlexWidthArray implements kelondroArray {
public synchronized int add(kelondroRow.Entry rowentry) throws IOException {
assert rowentry.objectsize() == this.rowdef.objectsize;
kelondroRow.Entry e;
int index = -1;
byte[] reb = rowentry.bytes();
e = col[0].row().newEntry(reb, 0);
index = col[0].add(e);
index = col[0].add(col[0].row().newEntry(reb, 0, false));
int c = col[0].row().columns();
while (c < rowdef.columns()) {
e = col[c].row().newEntry(reb, rowdef.colstart[c]);
col[c].set(index, e);
col[c].set(index, col[c].row().newEntry(reb, rowdef.colstart[c], false));
c = c + col[c].row().columns();
}
return index;
@ -286,12 +283,12 @@ public class kelondroFlexWidthArray implements kelondroArray {
kelondroRow.Entry e;
int index = -1;
byte[] reb = rowentry.bytes();
e = col[0].row().newEntry(reb, 0);
e = col[0].row().newEntry(reb, 0, false);
index = col[0].add(e);
int c = col[0].row().columns();
while (c < rowdef.columns()) {
e = col[c].row().newEntry(reb, rowdef.colstart[c]);
e = col[c].row().newEntry(reb, rowdef.colstart[c], false);
// remember write to column, but do not write directly
colm[c].put(new Integer(index), e); // col[c].set(index,e);
c = c + col[c].row().columns();

2
source/de/anomic/kelondro/kelondroIndex.java
Unescape View File

@ -57,7 +57,7 @@ import java.util.List;
public interface kelondroIndex {
public String filename(); // returns a unique identified for this index; can be a real or artificial file name
public int size() throws IOException;
public int size();
public kelondroProfile profile();
public kelondroRow row();
public boolean has(byte[] key) throws IOException; // use this only if there is no get in case that has returns true

55
source/de/anomic/kelondro/kelondroIntBytesMap.java
Unescape View File

@ -24,9 +24,10 @@
package de.anomic.kelondro;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
//import java.util.Random;
import java.util.Random;
public class kelondroIntBytesMap {
@ -58,7 +59,6 @@ public class kelondroIntBytesMap {
newentry.setCol(1, value);
index.addUnique(newentry);
}
public byte[] putb(int ii, byte[] value) {
kelondroRow.Entry newentry;
@ -100,19 +100,40 @@ public class kelondroIntBytesMap {
}
public static void main(String[] args) {
long start = System.currentTimeMillis();
kelondroIntBytesMap c = new kelondroIntBytesMap(30, 0);
//Random random = new Random(0);
int x;
for (int i = 0; i < 100000; i++) {
//x = random.nextInt(100000);
x = i;
c.putb(x, new byte[30]);
//if (c.getb(x) == null) System.out.println("consistency error at " + i + " with key " + x);
if (i % 10000 == 0) System.out.println(i + " entries. ");
}
System.out.println("RESULT SIZE: " + c.size());
System.out.println("Time: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
}
boolean assertEnabled = false;
assert assertEnabled = true;
System.out.println((assertEnabled) ? "asserts are enabled" : "enable asserts with 'java -ea'; not enabled yet");
long start = System.currentTimeMillis();
long randomstart = 0;
Random random = new Random(randomstart);
long r;
Long R;
int p, rc = 0;
ArrayList ra = new ArrayList();
HashSet jcontrol = new HashSet();
kelondroIntBytesMap kcontrol = new kelondroIntBytesMap(1, 0);
for (int i = 0; i < 1000000; i++) {
r = Math.abs(random.nextLong() % 10000);
//System.out.println("add " + r);
jcontrol.add(new Long(r));
kcontrol.putb((int) r, "x".getBytes());
if (random.nextLong() % 5 == 0) ra.add(new Long(r));
if ((ra.size() > 0) && (random.nextLong() % 7 == 0)) {
rc++;
p = Math.abs(random.nextInt()) % ra.size();
R = (Long) ra.get(p);
//System.out.println("remove " + R.longValue());
jcontrol.remove(R);
kcontrol.removeb((int) R.longValue());
assert kcontrol.removeb((int) R.longValue()) == null;
}
assert jcontrol.size() == kcontrol.size();
}
System.out.println("removed: " + rc + ", size of jcontrol set: "
+ jcontrol.size() + ", size of kcontrol set: "
+ kcontrol.size());
System.out.println("Time: "
+ ((System.currentTimeMillis() - start) / 1000) + " seconds");
}
}

2
source/de/anomic/kelondro/kelondroMapTable.java
Unescape View File

@ -178,7 +178,7 @@ public class kelondroMapTable {
if (table != null) return table.size();
kelondroIndex Tree = (kelondroIndex) tTables.get(tablename);
if (Tree != null) try { return Tree.size(); } catch (IOException e) {return 0;}
if (Tree != null) return Tree.size();
throw new RuntimeException("kelondroTables.accumulator: table '" + tablename + "' does not exist.");
}

13
source/de/anomic/kelondro/kelondroNaturalOrder.java
Unescape View File

@ -168,9 +168,10 @@ public class kelondroNaturalOrder extends kelondroAbstractOrder implements kelon
int i = 0;
final int al = Math.min(alength, a.length - aoffset);
final int bl = Math.min(blength, b.length - boffset);
final int len = Math.min(al, bl);
if (al > bl) return 1;
if (al < bl) return -1;
int aa, bb;
while (i < len) {
while (i < al) {
aa = 0xff & (int) a[i + aoffset];
bb = 0xff & (int) b[i + boffset];
if (aa > bb) return 1;
@ -178,13 +179,7 @@ public class kelondroNaturalOrder extends kelondroAbstractOrder implements kelon
// else the bytes are equal and it may go on yet undecided
i++;
}
// check if we have a zero-terminated equality
if ((i == al) && (i < bl) && (b[i + boffset] == 0)) return 0;
if ((i == bl) && (i < al) && (a[i + aoffset] == 0)) return 0;
// no, decide by length
if (al > bl) return 1;
if (al < bl) return -1;
// no, they are equal
// they are equal
return 0;
}

74
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -874,6 +874,10 @@ public class kelondroRecords {
System.arraycopy(rowinstance, ROW.width(0), this.tailChunk, 0, tailchunksize);
}
if (cacheHeaders != null) synchronized (cacheHeaders) {
updateNodeCache();
}
// mark chunks as changed
// if the head/tail chunks come from a file system read, setChanged should be false
// if the chunks come from a overwrite attempt, it should be true
@ -960,7 +964,6 @@ public class kelondroRecords {
}
} else synchronized(cacheHeaders) {
byte[] cacheEntry = null;
int cp = CP_HIGH;
cacheEntry = cacheHeaders.getb(this.handle.index);
if (cacheEntry == null) {
// cache miss, we read overhead and key from file
@ -981,17 +984,8 @@ public class kelondroRecords {
entryFile.readFully(seekpos(this.handle), this.headChunk, 0, headchunksize);
}
// calculate cache priority
cp = CP_HIGH;
if (OHHANDLEC == 3) {
Handle l = getOHHandle(1);
Handle r = getOHHandle(2);
if ((l == null) && (r == null)) cp = CP_LOW;
else if ((l == null) || (r == null)) cp = CP_MEDIUM;
}
// if space left in cache, copy these value to the cache
updateNodeCache(cp);
updateNodeCache();
} else {
readHit++;
this.headChunk = cacheEntry;
@ -1116,7 +1110,7 @@ public class kelondroRecords {
//System.out.println("WRITEH(" + filename + ", " + seekpos(this.handle) + ", " + this.headChunk.length + ")");
assert (headChunk == null) || (headChunk.length == headchunksize);
entryFile.write(seekpos(this.handle), (this.headChunk == null) ? new byte[headchunksize] : this.headChunk);
updateNodeCache(cachePriority);
updateNodeCache();
this.headChanged = false;
}
@ -1177,14 +1171,11 @@ public class kelondroRecords {
return cacheGrowStatus() > 0;
}
private void updateNodeCache(int priority) {
private void updateNodeCache() {
if (this.handle == null) return; // wrong access
if (this.headChunk == null) return; // nothing there to cache
if (priority == CP_NONE) return; // it is not wanted that this shall be cached
if (cacheHeaders == null) return; // we do not use the cache
if (!(cacheSpace())) return;
synchronized (cacheHeaders) {
if (cacheSpace()) synchronized (cacheHeaders) {
// generate cache entry
//byte[] cacheEntry = new byte[headchunksize];
//System.arraycopy(headChunk, 0, cacheEntry, 0, headchunksize);
@ -1196,6 +1187,11 @@ public class kelondroRecords {
//System.out.println("kelondroRecords cache4" + filename + ": cache record size = " + (memBefore - Runtime.getRuntime().freeMemory()) + " bytes" + ((newentry) ? " new" : ""));
//printCache();
} else {
// there shall be no entry in the cache. If one exists, we remove it
boolean rem = false;
rem = (cacheHeaders.removeb(this.handle.index) != null);
if (rem) cacheDelete++;
}
}
@ -1636,8 +1632,12 @@ public class kelondroRecords {
printCache();
System.out.println("--");
System.out.println("NODES");
for (int i = 0; i < USAGE.allCount(); i++)
System.out.println("NODE: " + new Node(new Handle(i), (Node) null, 0, true).toString());
Iterator i = new contentNodeIterator(-1);
Node n;
while (i.hasNext()) {
n = (Node) i.next();
System.out.println("NODE: " + n.toString());
}
}
public String toString() {
@ -1712,4 +1712,40 @@ public class kelondroRecords {
return kelondroProfile.consolidate(profiles());
}
public static void main(String[] args) {
File testfile = new File("kelondroRecordsTest.records");
kelondroRow testRow = new kelondroRow("byte[] key-16, value-16", kelondroBase64Order.enhancedCoder, 0);
// generate
if (testfile.exists()) testfile.delete();
try {
kelondroRecords r = new kelondroRecords(testfile, true, (long) -1,
(short) 0, (short) 0, testRow, 0, 0, 0);
r.newNode("ABCDEFGHIJKLMNOPQRSTUVWXYZ012345".getBytes());
r.newNode("abcdefghijklmnopqrstuvwxyz012345".getBytes());
r.newNode("A______________________________B".getBytes());
r.newNode("C______________________________D".getBytes());
r.newNode("E______________________________F".getBytes());
r.newNode("X______________________________Y".getBytes());
r.deleteNode(r.new Handle(1));
r.deleteNode(r.new Handle(3));
r.newNode("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".getBytes());
r.newNode("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".getBytes());
r.newNode("cccccccccccccccccccccccccccccccc".getBytes());
r.close();
} catch (IOException e) {
e.printStackTrace();
}
// re-open
try {
kelondroRecords r = new kelondroRecords(testfile, true, (long) -1,
(short) 0, (short) 0, testRow, 0, 0, 0);
r.print(true);
} catch (IOException e) {
e.printStackTrace();
}
}
}

32
source/de/anomic/kelondro/kelondroRow.java
Unescape View File

@ -151,21 +151,24 @@ public class kelondroRow {
//assert (rowinstance[0] != 0);
assert (this.objectOrder.wellformed(rowinstance, 0, row[0].cellwidth())) : "rowinstance[0] = " + serverLog.arrayList(rowinstance, 0, row[0].cellwidth());
if (!(this.objectOrder.wellformed(rowinstance, 0, row[0].cellwidth()))) return null;
return new Entry(rowinstance);
return new Entry(rowinstance, false);
}
public Entry newEntry(Entry oldrow, int fromColumn) {
if (oldrow == null) return null;
assert (oldrow.getColBytes(0)[0] != 0);
assert (this.objectOrder.wellformed(oldrow.getColBytes(0), 0, row[0].cellwidth()));
return new Entry(oldrow, fromColumn);
return new Entry(oldrow, fromColumn, false);
}
public Entry newEntry(byte[] rowinstance, int start) {
public Entry newEntry(byte[] rowinstance, int start, boolean clone) {
if (rowinstance == null) return null;
//assert (rowinstance[0] != 0);
assert (this.objectOrder.wellformed(rowinstance, start, row[0].cellwidth()));
return new Entry(rowinstance, start);
// this method offers the option to clone the content
// this is necessary if it is known that the underlying byte array may change and therefore
// the reference to the byte array does not contain the original content
return new Entry(rowinstance, start, clone);
}
public Entry newEntry(byte[][] cells) {
@ -198,22 +201,23 @@ public class kelondroRow {
offset = 0;
}
public Entry(byte[] newrow) {
this(newrow, 0);
public Entry(byte[] newrow, boolean forceclone) {
this(newrow, 0, forceclone);
}
public Entry(Entry oldrow, int fromColumn) {
this(oldrow.rowinstance, oldrow.offset + oldrow.colstart(fromColumn));
public Entry(Entry oldrow, int fromColumn, boolean forceclone) {
this(oldrow.rowinstance, oldrow.offset + oldrow.colstart(fromColumn), forceclone);
}
public Entry(byte[] newrow, int start) {
if (newrow.length - start >= objectsize) {
public Entry(byte[] newrow, int start, boolean forceclone) {
if ((!forceclone) && (newrow.length - start >= objectsize)) {
this.rowinstance = newrow;
this.offset = start;
} else {
this.rowinstance = new byte[objectsize];
System.arraycopy(newrow, start, this.rowinstance, 0, newrow.length - start);
System.arraycopy(newrow, start, this.rowinstance, 0, objectsize);
this.offset = 0;
}
this.offset = start;
//for (int i = ll; i < objectsize; i++) this.rowinstance[i] = 0;
}
@ -500,6 +504,8 @@ public class kelondroRow {
}
public byte[] getColBytes(int column) {
assert offset + colstart[column] + row[column].cellwidth() <= rowinstance.length :
"column = " + column + ", offset = " + offset + ", colstart[column] = " + colstart[column] + ", row[column].cellwidth() = " + row[column].cellwidth() + ", rowinstance.length = " + rowinstance.length;
byte[] c = new byte[row[column].cellwidth()];
System.arraycopy(rowinstance, offset + colstart[column], c, 0, row[column].cellwidth());
return c;
@ -552,7 +558,7 @@ public class kelondroRow {
public final class EntryIndex extends Entry {
private int index;
public EntryIndex(byte[] row, int i) {
super(row);
super(row, false);
this.index = i;
}
public int index() {

44
source/de/anomic/kelondro/kelondroRowCollection.java
Unescape View File

@ -39,7 +39,7 @@ public class kelondroRowCollection {
public static final double growfactor = 1.4;
private byte[] chunkcache;
protected byte[] chunkcache;
protected int chunkcount;
protected long lastTimeRead, lastTimeWrote;
protected kelondroRow rowdef;
@ -222,7 +222,7 @@ public class kelondroRowCollection {
if (index >= chunkcount) return null;
if (index * rowdef.objectsize() >= chunkcache.length) return null;
this.lastTimeRead = System.currentTimeMillis();
return rowdef.newEntry(chunkcache, index * rowdef.objectsize());
return rowdef.newEntry(chunkcache, index * rowdef.objectsize(), true);
}
public synchronized final void set(int index, kelondroRow.Entry a) {
@ -235,7 +235,8 @@ public class kelondroRowCollection {
}
public synchronized void addUnique(kelondroRow.Entry row) {
addUnique(row.bytes(), 0, row.bytes().length);
byte[] r = row.bytes();
addUnique(r, 0, r.length);
}
public synchronized void addUnique(kelondroRow.Entry row, Date entryDate) {
@ -288,30 +289,27 @@ public class kelondroRowCollection {
System.arraycopy(c.chunkcache, 0, chunkcache, rowdef.objectsize() * chunkcount, rowdef.objectsize() * c.size());
chunkcount += c.size();
}
private final void removeShift(int pos, int dist, int upBound) {
assert ((pos + dist) * rowdef.objectsize() >= 0) : "pos = " + pos + ", dist = " + dist + ", rowdef.objectsize() = " + rowdef.objectsize;
assert (pos * rowdef.objectsize() >= 0) : "pos = " + pos + ", rowdef.objectsize() = " + rowdef.objectsize;
assert ((pos + dist) * rowdef.objectsize() + (upBound - pos - dist) * rowdef.objectsize() <= chunkcache.length) : "pos = " + pos + ", dist = " + dist + ", rowdef.objectsize() = " + rowdef.objectsize + ", upBound = " + upBound + ", chunkcache.length = " + chunkcache.length;
assert (pos * rowdef.objectsize() + (upBound - pos - dist) * rowdef.objectsize() <= chunkcache.length) : "pos = " + pos + ", dist = " + dist + ", rowdef.objectsize() = " + rowdef.objectsize + ", upBound = " + upBound + ", chunkcache.length = " + chunkcache.length;
System.arraycopy(chunkcache, (pos + dist) * rowdef.objectsize(),
chunkcache, pos * rowdef.objectsize(),
(upBound - pos - dist) * rowdef.objectsize());
}
private final void copytop(int i) {
// copies the topmost row element to given position
if (i == chunkcount - 1) return;
System.arraycopy(chunkcache, this.rowdef.objectsize() * (chunkcount - 1), chunkcache, this.rowdef.objectsize() * i, this.rowdef.objectsize());
}
protected synchronized final void removeRow(int p) {
assert ((p >= 0) && (p < chunkcount) && (chunkcount > 0)) : "p = " + p + ", chunkcount = " + chunkcount;
assert p >= 0 : "p = " + p;
assert p < chunkcount : "p = " + p + ", chunkcount = " + chunkcount;
assert chunkcount > 0 : "chunkcount = " + chunkcount;
assert sortBound <= chunkcount : "sortBound = " + sortBound + ", chunkcount = " + chunkcount;
if (p < sortBound) {
removeShift(p, 1, chunkcount);
// remove by shift
System.arraycopy(
chunkcache, (p + 1) * this.rowdef.objectsize(),
chunkcache, p * this.rowdef.objectsize(),
(chunkcount - p - 1) * this.rowdef.objectsize());
sortBound--;
} else {
copytop(p);
// remove by copying the top-element to the remove position
if (p != chunkcount - 1) {
System.arraycopy(
chunkcache, (chunkcount - 1) * this.rowdef.objectsize(),
chunkcache, p * this.rowdef.objectsize(),
this.rowdef.objectsize());
}
}
chunkcount--;
this.lastTimeWrote = System.currentTimeMillis();
@ -482,6 +480,8 @@ public class kelondroRowCollection {
if (chunkcount <= 1) return;
int i = 0;
while (i < chunkcount - 1) {
//System.out.println("ENTRY0: " + serverLog.arrayList(chunkcache, rowdef.objectsize*i, rowdef.objectsize));
//System.out.println("ENTRY1: " + serverLog.arrayList(chunkcache, rowdef.objectsize*(i+1), rowdef.objectsize));
if (compare(i, i + 1) == 0) {
removeRow(i);
} else {

11
source/de/anomic/kelondro/kelondroRowSet.java
Unescape View File

@ -30,6 +30,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.Random;
import de.anomic.server.logging.serverLog;
public class kelondroRowSet extends kelondroRowCollection implements kelondroIndex {
private static final int collectionReSortLimit = 90;
@ -107,9 +109,12 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
private synchronized kelondroRow.Entry remove(byte[] a, int start, int length) {
int index = find(a, start, length);
if (index < 0) return null;
//System.out.println("remove: chunk found at index position (before remove) " + index + ", inset=" + serverLog.arrayList(super.chunkcache, super.rowdef.objectsize() * index, length + 10) + ", searchkey=" + serverLog.arrayList(a, start, length));
kelondroRow.Entry entry = super.get(index);
super.removeRow(index);
assert find(a, start, length) < 0; // check if the remove worked
//System.out.println("remove: chunk found at index position (after remove) " + index + ", inset=" + serverLog.arrayList(super.chunkcache, super.rowdef.objectsize() * index, length) + ", searchkey=" + serverLog.arrayList(a, start, length));
int findagainindex = find(a, start, length);
assert findagainindex < 0 : "remove: chunk found again at index position (after remove) " + findagainindex + ", inset=" + serverLog.arrayList(super.chunkcache, super.rowdef.objectsize() * findagainindex, length) + ", searchkey=" + serverLog.arrayList(a, start, length); // check if the remove worked
return entry;
}
@ -132,7 +137,9 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
if (rowdef.objectOrder == null) return iterativeSearch(a, astart, alength, 0, this.chunkcount);
// check if a re-sorting make sense
if ((this.chunkcount - this.sortBound) > collectionReSortLimit) sort();
if ((this.chunkcount - this.sortBound) > collectionReSortLimit) {
sort();
}
// first try to find in sorted area
int p = binarySearch(a, astart, alength);

22
source/de/anomic/plasma/plasmaCrawlBalancer.java
Unescape View File

@ -192,19 +192,15 @@ public class plasmaCrawlBalancer {
public synchronized int size() {
int componentsize = urlFileStack.size() + urlRAMStack.size() + sizeDomainStacks();
try {
if (componentsize != urlFileIndex.size()) {
// here is urlIndexFile.size() always smaller. why?
if (kelondroRecords.debugmode) {
serverLog.logWarning("PLASMA BALANCER", "size operation wrong in " + stackname + " - componentsize = " + componentsize + ", urlFileIndex.size() = " + urlFileIndex.size());
}
if ((componentsize == 0) && (urlFileIndex.size() > 0)) {
resetFileIndex();
}
}
} catch (IOException e) {
e.printStackTrace();
}
if (componentsize != urlFileIndex.size()) {
// here is urlIndexFile.size() always smaller. why?
if (kelondroRecords.debugmode) {
serverLog.logWarning("PLASMA BALANCER", "size operation wrong in " + stackname + " - componentsize = " + componentsize + ", urlFileIndex.size() = " + urlFileIndex.size());
}
if ((componentsize == 0) && (urlFileIndex.size() > 0)) {
resetFileIndex();
}
}
return componentsize;
}

6
source/de/anomic/plasma/plasmaCrawlLURL.java
Unescape View File

@ -112,11 +112,7 @@ public final class plasmaCrawlLURL {
}
public int size() {
try {
return urlIndexFile.size();
} catch (IOException e) {
return 0;
}
return urlIndexFile.size();
}
public void close() {

6
source/de/anomic/plasma/plasmaCrawlZURL.java
Unescape View File

@ -62,11 +62,7 @@ public class plasmaCrawlZURL {
}
public int size() {
try {
return urlIndexFile.size() ;
} catch (IOException e) {
return 0;
}
return urlIndexFile.size() ;
}
public void close() {

1
source/de/anomic/server/logging/serverLog.java
Unescape View File

@ -221,6 +221,7 @@ public final class serverLog {
public static final String arrayList(byte[] b, int start, int length) {
if (b == null) return "NULL";
if (b.length == 0) return "[]";
length = Math.min(length, b.length - start);
StringBuffer sb = new StringBuffer(b.length * 4);
sb.append('[').append(Integer.toString((int) b[start])).append(',');
for (int i = 1; i < length; i++) sb.append(' ').append(Integer.toString((int) b[start + i])).append(',');

Write Preview
Loading…
Cancel
Save