boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

debugging of kelondroRecords ... possibly synchronization bugs found

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1188 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent a1061495d4
commit 288b2c353e
3 changed files with 432 additions and 363 deletions
Show Stats Download Patch File Download Diff File

3
source/dbtest.java
Unescape View File

@ -54,6 +54,7 @@ public class dbtest {
public STEntry(final long aSource) {
this.key = randomHash(aSource, aSource);
this.value = new byte[valuelength];
for (int i = 0; i < valuelength; i++) this.value[i] = 0;
final byte[] tempKey = String.valueOf(aSource).getBytes();
System.arraycopy(tempKey, 0, this.value, 0, tempKey.length);
}
@ -128,6 +129,7 @@ public class dbtest {
try {
final byte[][] entryBytes = getTable().get(entry.getKey());
if (entryBytes != null) {
System.out.println("ENTRY=" + new String(entryBytes[1]));
final STEntry dbEntry = new STEntry(entryBytes[0], entryBytes[1]);
if (!dbEntry.isValid()) {
System.out.println(dbEntry);
@ -139,7 +141,6 @@ public class dbtest {
System.err.println(e);
}
}
}
public static void main(String[] args) {

194
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -159,10 +159,10 @@ public class kelondroRecords {
// FHandles: number of integer properties
// txtProps: number of text properties
if (file.exists()) throw new IOException("kelondroRecords: file " + file + " already exist");
assert (!file.exists()): "file " + file + " already exist";
this.filename = file.getCanonicalPath();
kelondroRA raf = new kelondroFileRA(this.filename);
//kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename));
//kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100);
// kelondroRA raf = new kelondroNIOFileRA(this.filename, false, 10000);
init(raf, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth);
initCache(buffersize);
@ -183,7 +183,8 @@ public class kelondroRecords {
this.entryFile = ra;
this.overhead = ohbytec + 4 * ohhandlec;
this.recordsize = this.overhead;
for (int i = 0; i < columns.length; i++) this.recordsize += columns[i];
for (int i = 0; i < columns.length; i++)
this.recordsize += columns[i];
this.headchunksize = overhead + columns[0];
this.tailchunksize = this.recordsize - this.headchunksize;
@ -199,8 +200,10 @@ public class kelondroRecords {
OHBYTEC = ohbytec;
OHHANDLEC = ohhandlec;
COLWIDTHS = columns;
HANDLES = new Handle[FHandles]; for (int i = 0; i < FHandles; i++) HANDLES[i] = new Handle(NUL);
TXTPROPS = new byte[txtProps][]; for (int i = 0; i < txtProps; i++) TXTPROPS[i] = new byte[0];
HANDLES = new Handle[FHandles];
for (int i = 0; i < FHandles; i++) HANDLES[i] = new Handle(NUL);
TXTPROPS = new byte[txtProps][];
for (int i = 0; i < txtProps; i++) TXTPROPS[i] = new byte[0];
TXTPROPW = txtPropWidth;
// write data to file
@ -233,10 +236,9 @@ public class kelondroRecords {
}
for (int i = 0; i < this.TXTPROPS.length; i++) {
entryFile.seek(POS_TXTPROPS + TXTPROPW * i);
for (int j = 0; j < TXTPROPW; j++) entryFile.writeByte(0);
for (int j = 0; j < TXTPROPW; j++)
entryFile.writeByte(0);
}
// thats it!
}
public void setLogger(Logger newLogger) {
@ -256,18 +258,20 @@ public class kelondroRecords {
USEDC = 0;
FREEC = 0;
FREEH = new Handle(NUL);
entryFile.seek(POS_USEDC); entryFile.writeInt(this.USEDC);
entryFile.seek(POS_FREEC); entryFile.writeInt(this.FREEC);
entryFile.seek(POS_FREEH); entryFile.writeInt(this.FREEH.index);
entryFile.seek(POS_USEDC);
entryFile.writeInt(this.USEDC);
entryFile.seek(POS_FREEC);
entryFile.writeInt(this.FREEC);
entryFile.seek(POS_FREEH);
entryFile.writeInt(this.FREEH.index);
}
public kelondroRecords(File file, long buffersize) throws IOException{
// opens an existing tree
if (!file.exists()) throw new IOException("kelondroRecords: file " + file.getAbsoluteFile().toString() + " does not exist");
assert (file.exists()): "file " + file.getAbsoluteFile().toString() + " does not exist";
this.filename = file.getCanonicalPath();
kelondroRA raf = new kelondroFileRA(this.filename);
//kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename));
//kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100);
//kelondroRA raf = new kelondroCachedRA(new kelondroFileRA(this.filename), 5000000, 1000);
//kelondroRA raf = new kelondroNIOFileRA(this.filename, (file.length() < 4000000), 10000);
init(raf);
@ -281,17 +285,17 @@ public class kelondroRecords {
}
private void init(kelondroRA ra) throws IOException {
// assign values that are only present at run-time
this.entryFile = ra;
// read dynamic variables that are back-ups of stored values in file; read/defined on instantiation
// read dynamic variables that are back-ups of stored values in file;
// read/defined on instantiation
entryFile.seek(POS_USEDC); this.USEDC = entryFile.readInt();
entryFile.seek(POS_FREEC); this.FREEC = entryFile.readInt();
entryFile.seek(POS_FREEH); this.FREEH = new Handle(entryFile.readInt());
entryFile.seek(POS_OHBYTEC); OHBYTEC = entryFile.readShort();
entryFile.seek(POS_OHHANDLEC); OHHANDLEC = entryFile.readShort();
entryFile.seek(POS_OHBYTEC); this.OHBYTEC = entryFile.readShort();
entryFile.seek(POS_OHHANDLEC); this.OHHANDLEC = entryFile.readShort();
entryFile.seek(POS_COLUMNS); this.COLWIDTHS = new int[entryFile.readShort()];
entryFile.seek(POS_INTPROPC); this.HANDLES = new Handle[entryFile.readInt()];
@ -380,7 +384,6 @@ public class kelondroRecords {
return new int[]{XcacheSize - (XcacheHeaders[CP_HIGH].size() + XcacheHeaders[CP_MEDIUM].size() + XcacheHeaders[CP_LOW].size()), XcacheHeaders[CP_HIGH].size(), XcacheHeaders[CP_MEDIUM].size(), XcacheHeaders[CP_LOW].size()};
}
protected Node newNode() throws IOException {
return new Node();
}
@ -416,8 +419,6 @@ public class kelondroRecords {
dispose(handle);
}
public class Node {
// an Node holds all information of one row of data. This includes the key to the entry
// which is stored as entry element at position 0
@ -483,12 +484,15 @@ public class kelondroRecords {
*/
private Node(Handle handle, Node parentNode, int referenceInParent) throws IOException {
// this creates an entry with an pre-reserved entry position
// values can be written using the setValues() method
// but we expect that values are already there in the file ready to be read which we do not here
if (handle == null) throw new IllegalArgumentException("INTERNAL ERROR: node handle is null.");
// the parentNode can be given if an auto-fix in the following case is wanted
// this creates an entry with an pre-reserved entry position values can be written
// using the setValues() method but we expect that values are already there in the file
// ready to be read which we do not here
assert (handle != null): "node handle is null";
assert (handle.index >= 0): "node handle too low: " + handle.index;
assert (handle.index < USEDC + FREEC) : "node handle too high: " + handle.index + ", USEDC=" + USEDC + ", FREEC=" + FREEC;
// the parentNode can be given if an auto-fix in the following case
// is wanted
if (handle.index >= USEDC + FREEC) {
if (parentNode == null) {
throw new kelondroException(filename, "INTERNAL ERROR, Node/init: node handle index exceeds size. No auto-fix node was submitted. This is a serious failure.");
@ -573,10 +577,11 @@ public class kelondroRecords {
while (valuewidth-- > 0) targetarray[targetoffset + valuewidth] = 0;
} else {
System.arraycopy(value, 0, targetarray, targetoffset, Math.min(value.length, valuewidth)); // error?
if (value.length < valuewidth)
if (value.length < valuewidth) {
while (valuewidth-- > value.length) targetarray[targetoffset + valuewidth] = 0;
}
}
}
protected Handle handle() {
// if this entry has an index, return it
@ -591,14 +596,14 @@ public class kelondroRecords {
this.headChanged = true;
}
protected void setOHHandle(int i, Handle handle) {
if (i >= OHHANDLEC) throw new IllegalArgumentException("setOHHandle: wrong array size " + i);
if (this.handle.index == NUL) throw new kelondroException(filename, "setOHHandle: no handle assigned");
if (handle == null) {
protected void setOHHandle(int i, Handle otherhandle) {
assert (i < OHHANDLEC): "setOHHandle: wrong array size " + i;
assert (this.handle.index != NUL): "setOHHandle: no handle assigned ind file" + filename;
if (otherhandle == null) {
NUL2bytes(this.headChunk, OHBYTEC + 4 * i);
} else {
if (handle.index > USEDC + FREEC) throw new kelondroException(filename, "INTERNAL ERROR, setOHHandles: handle " + i + " exceeds file size (" + handle.index + " > " + (USEDC + FREEC) + ")");
int2bytes(handle.index, this.headChunk, OHBYTEC + 4 * i);
if (otherhandle.index > USEDC + FREEC) throw new kelondroException(filename, "INTERNAL ERROR, setOHHandles: handle " + i + " exceeds file size (" + handle.index + " > " + (USEDC + FREEC) + ")");
int2bytes(otherhandle.index, this.headChunk, OHBYTEC + 4 * i);
}
this.headChanged = true;
}
@ -611,7 +616,7 @@ public class kelondroRecords {
protected Handle getOHHandle(int i) {
if (this.handle.index == NUL) throw new kelondroException(filename, "Cannot load OH values");
if (i >= OHHANDLEC) throw new kelondroException(filename, "handle index out of bounds: " + i);
assert (i < OHHANDLEC): "handle index out of bounds: " + i + " in file " + filename;
int h = bytes2int(this.headChunk, OHBYTEC + 4 * i);
return (h == NUL) ? null : new Handle(h);
}
@ -640,6 +645,7 @@ public class kelondroRecords {
}
public byte[][] getValues() throws IOException {
if (this.tailChunk == null) {
// load all values from the database file
this.tailChunk = new byte[tailchunksize];
@ -667,7 +673,8 @@ public class kelondroRecords {
}
public synchronized void commit(int cachePriority) throws IOException {
// this must be called after all write operations to the node are finished
// this must be called after all write operations to the node are
// finished
// place the data to the file
@ -680,30 +687,36 @@ public class kelondroRecords {
if (this.headChanged) {
synchronized (entryFile) {
entryFile.seek(seekpos(this.handle));
//System.out.print("#write "); printChunk(this.handle, this.headChunk); System.out.println();
// System.out.print("#write "); printChunk(this.handle,
// this.headChunk); System.out.println();
entryFile.write(this.headChunk);
}
update2Cache(cachePriority);
}
// save tail
if ((this.tailChunk != null) && (this.tailChanged)) synchronized (entryFile) {
if ((this.tailChunk != null) && (this.tailChanged))
synchronized (entryFile) {
entryFile.seek(seekpos(this.handle) + headchunksize);
entryFile.write(this.tailChunk);
}
}
public synchronized void collapse() {
// this must be called after all write and read operations to the node are finished
// this must be called after all write and read operations to the
// node are finished
this.headChunk = null;
this.tailChunk = null;
this.handle = null;
}
private byte[] trimCopy(byte[] a, int offset, int length) {
if (length > a.length - offset) length = a.length - offset;
while ((length > 0) && (a[offset + length - 1] == 0)) length--;
if (length == 0) return null;
if (length > a.length - offset)
length = a.length - offset;
while ((length > 0) && (a[offset + length - 1] == 0))
length--;
if (length == 0)
return null;
byte[] b = new byte[length];
System.arraycopy(a, offset, b, 0, length);
return b;
@ -804,9 +817,9 @@ public class kelondroRecords {
// we simply clear the cache
String error = "cachScore error: " + e.getMessage() + "; cachesize=" + XcacheSize + ", cache.size()=[" + XcacheHeaders[0].size() + "," + XcacheHeaders[1].size() + "," + XcacheHeaders[2].size() + "], cacheScore.size()=" + cacheScore.size();
cacheScore = new kelondroMScoreCluster();
XcacheHeaders[0] = new HashMap();
XcacheHeaders[1] = new HashMap();
XcacheHeaders[2] = new HashMap();
XcacheHeaders[CP_LOW] = new HashMap();
XcacheHeaders[CP_MEDIUM] = new HashMap();
XcacheHeaders[CP_HIGH] = new HashMap();
throw new kelondroException(filename, error);
}
@ -902,6 +915,8 @@ public class kelondroRecords {
}
private long seekpos(Handle handle) {
assert (handle.index >= 0): "handle index too low: " + handle.index;
assert (handle.index < FREEC + USEDC): "handle index too high:" + handle.index;
return POS_NODES + ((long) recordsize * handle.index);
}
@ -913,10 +928,12 @@ public class kelondroRecords {
protected void setHandle(int pos, Handle handle) throws IOException {
if (pos >= HANDLES.length) throw new IllegalArgumentException("setHandle: handle array exceeded");
if (handle == null) handle = new Handle(NUL);
synchronized (entryFile) {
HANDLES[pos] = handle;
entryFile.seek(POS_HANDLES + 4 * pos);
entryFile.writeInt(handle.index);
}
}
protected Handle getHandle(int pos) {
if (pos >= HANDLES.length) throw new IllegalArgumentException("getHandle: handle array exceeded");
@ -928,10 +945,12 @@ public class kelondroRecords {
if (pos >= TXTPROPS.length) throw new IllegalArgumentException("setText: text array exceeded");
if (text.length > TXTPROPW) throw new IllegalArgumentException("setText: text lemgth exceeded");
if (text == null) text = new byte[0];
synchronized (entryFile) {
TXTPROPS[pos] = text;
entryFile.seek(POS_TXTPROPS + TXTPROPW * pos);
entryFile.write(text);
}
}
public byte[] getText(int pos) {
if (pos >= TXTPROPS.length) throw new IllegalArgumentException("getText: text array exceeded");
@ -954,11 +973,13 @@ public class kelondroRecords {
private void dispose(Handle h) throws IOException {
// delete element with handle h
// this element is then connected to the deleted-chain and can be re-used
// change counter
// this element is then connected to the deleted-chain and can be
// re-used change counter
synchronized (entryFile) {
USEDC--; entryFile.seek(POS_USEDC); entryFile.writeInt(USEDC);
FREEC++; entryFile.seek(POS_FREEC); entryFile.writeInt(FREEC);
USEDC--;
entryFile.seek(POS_USEDC); entryFile.writeInt(USEDC);
FREEC++;
entryFile.seek(POS_FREEC); entryFile.writeInt(FREEC);
// change pointer
if (this.FREEH.index == NUL) {
// the first entry
@ -1049,15 +1070,22 @@ public class kelondroRecords {
System.out.println("--");
System.out.println("CONTROL DATA");
System.out.print(" HANDLES : " + HANDLES.length + " int-values");
if (HANDLES.length == 0) System.out.println(); else {
if (HANDLES.length == 0)
System.out.println();
else {
System.out.print(" {" + HANDLES[0].toString());
for (int i = 1; i < HANDLES.length; i++) System.out.print(", " + HANDLES[i].toString());
for (int i = 1; i < HANDLES.length; i++)
System.out.print(", " + HANDLES[i].toString());
System.out.println("}");
}
System.out.print(" TXTPROPS : " + TXTPROPS.length + " strings, max length " + TXTPROPW + " bytes");
if (TXTPROPS.length == 0) System.out.println(); else {
System.out.print(" {'" + (new String(TXTPROPS[0])).trim()); System.out.print("'");
for (int i = 1; i < TXTPROPS.length; i++) System.out.print(", '" + (new String(TXTPROPS[i])).trim() + "'");
if (TXTPROPS.length == 0)
System.out.println();
else {
System.out.print(" {'" + (new String(TXTPROPS[0])).trim());
System.out.print("'");
for (int i = 1; i < TXTPROPS.length; i++)
System.out.print(", '" + (new String(TXTPROPS[i])).trim() + "'");
System.out.println("}");
}
System.out.println(" USEDC : " + this.USEDC);
@ -1077,7 +1105,8 @@ public class kelondroRecords {
if (!(records)) return;
// print also all records
for (int i = 0; i < USEDC + FREEC; i++) System.out.println("NODE: " + new Node(new Handle(i), null, 0).toString());
for (int i = 0; i < USEDC + FREEC; i++)
System.out.println("NODE: " + new Node(new Handle(i), null, 0).toString());
}
public String toString() {
@ -1086,26 +1115,40 @@ public class kelondroRecords {
protected class Handle implements Comparable {
private int index;
private Handle() throws IOException {
// reserves a new record and returns index of record
// the return value is not a seek position
// the seek position can be retrieved using the seekpos() function
synchronized (this) {
if (FREEC == 0) {
// generate new entry
synchronized (entryFile) {
index = USEDC + FREEC;
USEDC++; entryFile.seek(POS_USEDC); entryFile.writeInt(USEDC);
USEDC++;
entryFile.seek(POS_USEDC); entryFile.writeInt(USEDC);
}
} else {
// re-use record from free-list
USEDC++; entryFile.seek(POS_USEDC); entryFile.writeInt(USEDC);
FREEC--; entryFile.seek(POS_FREEC); entryFile.writeInt(FREEC);
synchronized (entryFile) {
USEDC++;
entryFile.seek(POS_USEDC); entryFile.writeInt(USEDC);
FREEC--;
entryFile.seek(POS_FREEC); entryFile.writeInt(FREEC);
}
// take link
if (FREEH.index == NUL) {
System.out.println("INTERNAL ERROR (DATA INCONSISTENCY): re-use of records failed, lost " + (FREEC + 1) + " records. Affected file: " + filename);
// try to heal..
USEDC = USEDC + FREEC + 1; entryFile.seek(POS_USEDC); entryFile.writeInt(USEDC);
FREEC = 0; entryFile.seek(POS_FREEC); entryFile.writeInt(FREEC);
synchronized (entryFile) {
USEDC = USEDC + FREEC + 1;
entryFile.seek(POS_USEDC); entryFile.writeInt(USEDC);
FREEC = 0;
entryFile.seek(POS_FREEC); entryFile.writeInt(FREEC);
index = USEDC - 1;
}
} else {
synchronized (entryFile) {
index = FREEH.index;
// read link to next element to FREEH chain
entryFile.seek(seekpos(FREEH)); FREEH.index = entryFile.readInt();
@ -1114,32 +1157,57 @@ public class kelondroRecords {
}
}
}
protected Handle(int index) {
this.index = index;
}
}
protected Handle(int i) {
assert (i == NUL) || (i >= 0) : "node handle index too low: " + i;
assert (i == NUL) || (i < USEDC + FREEC) : "node handle index too high: " + i + ", USEDC=" + USEDC + ", FREEC=" + FREEC;
this.index = i;
}
public boolean isNUL() {
return index == NUL;
}
public String toString() {
if (index == NUL) return "NULL";
String s = Integer.toHexString(index);
while (s.length() < 4) s = "0" + s;
return s;
}
public boolean equals(Handle h) {
assert (index != NUL);
assert (h.index != NUL);
return (this.index == h.index);
}
public boolean equals(Object h) {
assert (index != NUL);
assert (((Handle) h).index != NUL);
return (this.index == ((Handle) h).index);
}
public int compare(Object h0, Object h1) {
assert (((Handle) h0).index != NUL);
assert (((Handle) h1).index != NUL);
if (((Handle) h0).index < ((Handle) h1).index) return -1;
if (((Handle) h0).index > ((Handle) h1).index) return 1;
return 0;
}
public int compareTo(Object h) { // this is needed for a treeMap compare
public int compareTo(Object h) {
// this is needed for a treeMap
assert (index != NUL);
assert (((Handle) h).index != NUL);
if (index < ((Handle) h).index) return -1;
if (index > ((Handle) h).index) return 1;
return 0;
}
public int hashCode() {
assert (index != NUL);
return this.index;
}
}

6
source/de/anomic/kelondro/kelondroTree.java
Unescape View File

@ -133,7 +133,7 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr
}
// Returns the value to which this map maps the specified key.
public synchronized byte[][] get(byte[] key) throws IOException {
public byte[][] get(byte[] key) throws IOException {
//System.out.println("kelondroTree.get " + new String(key) + " in " + filename);
Search search = new Search();
search.process(key);
@ -145,7 +145,7 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr
}
}
public synchronized long[] getLong(byte[] key) throws IOException {
public long[] getLong(byte[] key) throws IOException {
byte[][] row = get(key);
long[] longs = new long[columns() - 1];
if (row == null) {
@ -1311,7 +1311,7 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr
// Returns the comparator used to order this map,
// or null if this map uses its keys' natural order.
public synchronized Comparator comparator() {
public Comparator comparator() {
return this;
}

Write Preview
Loading…
Cancel
Save