From 595ee10468e165ab3e56baddc02f57b9de713aef Mon Sep 17 00:00:00 2001 From: orbiter Date: Thu, 19 Apr 2007 13:37:02 +0000 Subject: [PATCH] fixed datatabase inconsistency bugs inserted many debug lines added a huge number of asserts extended database test methods git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3579 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/dbtest.java | 65 ++++++-- .../anomic/kelondro/kelondroBase64Order.java | 30 ++-- .../anomic/kelondro/kelondroBytesIntMap.java | 4 +- source/de/anomic/kelondro/kelondroCache.java | 2 +- .../kelondro/kelondroFlexSplitTable.java | 2 +- .../de/anomic/kelondro/kelondroFlexTable.java | 155 ++++++++++-------- .../kelondro/kelondroFlexWidthArray.java | 15 +- source/de/anomic/kelondro/kelondroIndex.java | 2 +- .../anomic/kelondro/kelondroIntBytesMap.java | 55 +++++-- .../de/anomic/kelondro/kelondroMapTable.java | 2 +- .../anomic/kelondro/kelondroNaturalOrder.java | 13 +- .../de/anomic/kelondro/kelondroRecords.java | 74 ++++++--- source/de/anomic/kelondro/kelondroRow.java | 32 ++-- .../kelondro/kelondroRowCollection.java | 44 ++--- source/de/anomic/kelondro/kelondroRowSet.java | 11 +- .../de/anomic/plasma/plasmaCrawlBalancer.java | 22 +-- source/de/anomic/plasma/plasmaCrawlLURL.java | 6 +- source/de/anomic/plasma/plasmaCrawlZURL.java | 6 +- .../de/anomic/server/logging/serverLog.java | 1 + 19 files changed, 323 insertions(+), 218 deletions(-) diff --git a/source/dbtest.java b/source/dbtest.java index 44040750b..3324b1a1e 100644 --- a/source/dbtest.java +++ b/source/dbtest.java @@ -11,6 +11,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.Date; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Random; @@ -23,15 +24,18 @@ import de.anomic.kelondro.kelondroCloneableIterator; import de.anomic.kelondro.kelondroFlexSplitTable; import de.anomic.kelondro.kelondroFlexTable; import de.anomic.kelondro.kelondroIndex; +import de.anomic.kelondro.kelondroIntBytesMap; import de.anomic.kelondro.kelondroNaturalOrder; import de.anomic.kelondro.kelondroOrder; import de.anomic.kelondro.kelondroProfile; import de.anomic.kelondro.kelondroRow; +import de.anomic.kelondro.kelondroRowSet; import de.anomic.kelondro.kelondroSplittedTree; import de.anomic.kelondro.kelondroTree; import de.anomic.kelondro.kelondroRow.Entry; import de.anomic.server.serverInstantThread; import de.anomic.server.serverMemory; +import de.anomic.server.logging.serverLog; import de.anomic.ymage.ymageChart; public class dbtest { @@ -50,8 +54,9 @@ public class dbtest { public static byte[] randomHash(final long r0, final long r1) { // a long can have 64 bit, but a 12-byte hash can have 6 * 12 = 72 bits // so we construct a generic Hash using two long values - return (kelondroBase64Order.enhancedCoder.encodeLong(Math.abs(r0), 11).substring(5) + - kelondroBase64Order.enhancedCoder.encodeLong(Math.abs(r1), 11).substring(5)).getBytes(); + String s = (kelondroBase64Order.enhancedCoder.encodeLong(Math.abs(r0), 6) + + kelondroBase64Order.enhancedCoder.encodeLong(Math.abs(r1), 6)); + return s.getBytes(); } public static byte[] randomHash(Random r) { @@ -124,6 +129,7 @@ public class dbtest { public void run() { final STEntry entry = new STEntry(this.getSource()); + System.out.println("write: " + serverLog.arrayList(entry.getKey(), 0, entry.getKey().length)); try { getTable().put(getTable().row().newEntry(new byte[][] { entry.getKey(), entry.getValue() , entry.getValue() })); } catch (IOException e) { @@ -141,6 +147,7 @@ public class dbtest { public void run() { final STEntry entry = new STEntry(this.getSource()); + System.out.println("remove: " + serverLog.arrayList(entry.getKey(), 0, entry.getKey().length)); try { getTable().remove(entry.getKey()); } catch (IOException e) { @@ -194,6 +201,9 @@ public class dbtest { // create the database access kelondroRow testRow = new kelondroRow("byte[] key-" + keylength + ", byte[] dummy-" + keylength + ", value-" + valuelength, kelondroBase64Order.enhancedCoder, 0); + if (dbe.equals("kelondroRowSet")) { + table = new kelondroRowSet(testRow, 0); + } if (dbe.equals("kelondroTree")) { File tablefile = new File(tablename + ".kelondro.db"); table = new kelondroCache(new kelondroTree(tablefile, true, preload, testRow), true, false); @@ -376,21 +386,31 @@ public class dbtest { long randomstart = Long.parseLong(args[5]); final Random random = new Random(randomstart); long r; - int p; + Long R; + int p, rc=0; ArrayList ra = new ArrayList(); + HashSet jcontrol = new HashSet(); + kelondroIntBytesMap kcontrol = new kelondroIntBytesMap(1, 0); for (int i = 0; i < writeCount; i++) { - r = random.nextLong() % 1000; + r = Math.abs(random.nextLong() % 1000); + jcontrol.add(new Long(r)); + kcontrol.putb((int) r, "x".getBytes()); serverInstantThread.oneTimeJob(new WriteJob(table, r), 0, 50); if (random.nextLong() % 5 == 0) ra.add(new Long(r)); for (int j = 0; j < readCount; j++) { serverInstantThread.oneTimeJob(new ReadJob(table, random.nextLong() % writeCount), random.nextLong() % 1000, 20); } if ((ra.size() > 0) && (random.nextLong() % 7 == 0)) { + rc++; p = Math.abs(random.nextInt()) % ra.size(); - System.out.println("remove: " + ((Long) ra.get(p)).longValue()); + R = (Long) ra.get(p); + jcontrol.remove(R); + kcontrol.removeb((int) R.longValue()); + System.out.println("remove: " + R.longValue()); serverInstantThread.oneTimeJob(new RemoveJob(table, ((Long) ra.remove(p)).longValue()), 0, 50); } } + System.out.println("removed: " + rc + ", size of jcontrol set: " + jcontrol.size() + ", size of kcontrol set: " + kcontrol.size()); while (serverInstantThread.instantThreadCounter > 0) { try {Thread.sleep(1000);} catch (InterruptedException e) {} // wait for all tasks to finish System.out.println("count: " + serverInstantThread.instantThreadCounter + ", jobs: " + serverInstantThread.jobs.toString()); @@ -404,13 +424,34 @@ public class dbtest { long writeCount = Long.parseLong(args[3]); long readCount = Long.parseLong(args[4]); long randomstart = Long.parseLong(args[5]); - final Random random = new Random(randomstart); + Random random = new Random(randomstart); + long r; + Long R; + int p, rc=0; + ArrayList ra = new ArrayList(); + HashSet jcontrol = new HashSet(); + kelondroIntBytesMap kcontrol = new kelondroIntBytesMap(1, 0); for (int i = 0; i < writeCount; i++) { - new WriteJob(table, i).run(); + //if (i == 30) random = new Random(randomstart); + r = Math.abs(random.nextLong() % 1000); + jcontrol.add(new Long(r)); + kcontrol.putb((int) r, "x".getBytes()); + new WriteJob(table, r).run(); + if (random.nextLong() % 5 == 0) ra.add(new Long(r)); for (int j = 0; j < readCount; j++) { new ReadJob(table, random.nextLong() % writeCount).run(); } + if ((ra.size() > 0) && (random.nextLong() % 7 == 0)) { + rc++; + p = Math.abs(random.nextInt()) % ra.size(); + R = (Long) ra.get(p); + jcontrol.remove(R); + kcontrol.removeb((int) R.longValue()); + new RemoveJob(table, ((Long) ra.remove(p)).longValue()).run(); + } } + try {Thread.sleep(1000);} catch (InterruptedException e) {} + System.out.println("removed: " + rc + ", size of jcontrol set: " + jcontrol.size() + ", size of kcontrol set: " + kcontrol.size()); } long aftercommand = System.currentTimeMillis(); @@ -418,10 +459,7 @@ public class dbtest { System.out.println("Database size = " + table.size() + " unique entries."); // finally close the database/table - if (table instanceof kelondroTree) ((kelondroTree) table).close(); - if (table instanceof kelondroFlexTable) ((kelondroFlexTable) table).close(); - if (table instanceof kelondroSplittedTree) ((kelondroSplittedTree) table).close(); - if (table instanceof dbTable) ((dbTable)table).close(); + table.close(); long afterclose = System.currentTimeMillis(); @@ -501,7 +539,7 @@ final class dbTable implements kelondroIndex { this.theDBConnection = null; } - public int size() throws IOException { + public int size() { int size = -1; try { String sqlQuery = new String @@ -521,7 +559,8 @@ final class dbTable implements kelondroIndex { return size; } catch (Exception e) { - throw new IOException(e.getMessage()); + e.printStackTrace(); + return -1; } } diff --git a/source/de/anomic/kelondro/kelondroBase64Order.java b/source/de/anomic/kelondro/kelondroBase64Order.java index 67b9eed47..18d1d4301 100644 --- a/source/de/anomic/kelondro/kelondroBase64Order.java +++ b/source/de/anomic/kelondro/kelondroBase64Order.java @@ -299,8 +299,8 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond public final int compare0(byte[] a, int aoffset, int alength, byte[] b, int boffset, int blength) { if (zero == null) return compares(a, aoffset, alength, b, boffset, blength); // we have an artificial start point. check all combinations - int az = compares(a, aoffset, alength, zero, 0, zero.length); // -1 if a < z; 0 if a == z; 1 if a > z - int bz = compares(b, boffset, blength, zero, 0, zero.length); // -1 if b < z; 0 if b == z; 1 if b > z + int az = compares(a, aoffset, alength, zero, 0, Math.min(alength, zero.length)); // -1 if a < z; 0 if a == z; 1 if a > z + int bz = compares(b, boffset, blength, zero, 0, Math.min(blength, zero.length)); // -1 if b < z; 0 if b == z; 1 if b > z if ((az == 0) && (bz == 0)) return 0; if (az == 0) return -1; if (bz == 0) return 1; @@ -315,32 +315,28 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond int i = 0; final int al = Math.min(alength, a.length - aoffset); final int bl = Math.min(blength, b.length - boffset); - final int len = (al > bl) ? bl : al; + if (al > bl) return 1; + if (al < bl) return -1; byte ac, bc; byte acc, bcc; - while (i < len) { - assert (i + aoffset < a.length) : "i = " + i + ", aoffset = " + aoffset + ", a.length = " + a.length + ", a = " + serverLog.arrayList(a, aoffset, len); - assert (i + boffset < b.length) : "i = " + i + ", boffset = " + boffset + ", b.length = " + b.length + ", b = " + serverLog.arrayList(b, boffset, len); + while (i < al) { + assert (i + aoffset < a.length) : "i = " + i + ", aoffset = " + aoffset + ", a.length = " + a.length + ", a = " + serverLog.arrayList(a, aoffset, al); + assert (i + boffset < b.length) : "i = " + i + ", boffset = " + boffset + ", b.length = " + b.length + ", b = " + serverLog.arrayList(b, boffset, al); ac = a[aoffset + i]; - assert (ac >= 0) && (ac < 128) : "ac = " + ac + ", a = " + serverLog.arrayList(a, aoffset, len); + assert (ac >= 0) && (ac < 128) : "ac = " + ac + ", a = " + serverLog.arrayList(a, aoffset, al); bc = b[boffset + i]; - assert (bc >= 0) && (bc < 128) : "bc = " + bc + ", b = " + serverLog.arrayList(b, boffset, len); + if ((ac == 0) && (bc == 0)) return 0; // zero-terminated length + assert (bc >= 0) && (bc < 128) : "bc = " + bc + ", b = " + serverLog.arrayList(b, boffset, al); acc = ahpla[ac]; - assert (acc >= 0) : "acc = " + acc + ", a = " + serverLog.arrayList(a, aoffset, len) + "/" + new String(a, aoffset, len) + ", aoffset = 0x" + Integer.toHexString(aoffset) + ", i = " + i + "\n" + serverLog.table(a, 16, aoffset); + assert (acc >= 0) : "acc = " + acc + ", a = " + serverLog.arrayList(a, aoffset, al) + "/" + new String(a, aoffset, al) + ", aoffset = 0x" + Integer.toHexString(aoffset) + ", i = " + i + "\n" + serverLog.table(a, 16, aoffset); bcc = ahpla[bc]; - assert (bcc >= 0) : "bcc = " + bcc + ", b = " + serverLog.arrayList(b, boffset, len) + "/" + new String(b, boffset, len) + ", boffset = 0x" + Integer.toHexString(boffset) + ", i = " + i + "\n" + serverLog.table(b, 16, boffset); + assert (bcc >= 0) : "bcc = " + bcc + ", b = " + serverLog.arrayList(b, boffset, al) + "/" + new String(b, boffset, al) + ", boffset = 0x" + Integer.toHexString(boffset) + ", i = " + i + "\n" + serverLog.table(b, 16, boffset); if (acc > bcc) return 1; if (acc < bcc) return -1; // else the bytes are equal and it may go on yet undecided i++; } - // check if we have a zero-terminated equality - if ((i == al) && (i < bl) && (b[i + boffset] == 0)) return 0; - if ((i == bl) && (i < al) && (a[i + aoffset] == 0)) return 0; - // no, decide by length - if (al > bl) return 1; - if (al < bl) return -1; - // no, they are equal + // they are equal return 0; } diff --git a/source/de/anomic/kelondro/kelondroBytesIntMap.java b/source/de/anomic/kelondro/kelondroBytesIntMap.java index 766b341cd..aa5a92e55 100644 --- a/source/de/anomic/kelondro/kelondroBytesIntMap.java +++ b/source/de/anomic/kelondro/kelondroBytesIntMap.java @@ -49,6 +49,7 @@ public class kelondroBytesIntMap { } public synchronized int puti(byte[] key, int i) throws IOException { + assert i >= 0 : "i = " + i; assert (key != null); //assert (!(serverLog.allZero(key))); kelondroRow.Entry newentry = ki.row().newEntry(); @@ -60,6 +61,7 @@ public class kelondroBytesIntMap { } public synchronized void addi(byte[] key, int i) throws IOException { + assert i >= 0 : "i = " + i; assert (key != null); //assert (!(serverLog.allZero(key))); kelondroRow.Entry newentry = ki.row().newEntry(); @@ -87,7 +89,7 @@ public class kelondroBytesIntMap { return (int) indexentry.getColLong(1); } - public synchronized int size() throws IOException { + public synchronized int size() { return ki.size(); } diff --git a/source/de/anomic/kelondro/kelondroCache.java b/source/de/anomic/kelondro/kelondroCache.java index 64dcd4b61..5c1f32bcb 100644 --- a/source/de/anomic/kelondro/kelondroCache.java +++ b/source/de/anomic/kelondro/kelondroCache.java @@ -638,7 +638,7 @@ public class kelondroCache implements kelondroIndex { return index.rows(up, firstKey); } - public int size() throws IOException { + public int size() { return index.size() + ((writeBufferUnique == null) ? 0 : writeBufferUnique.size()); } diff --git a/source/de/anomic/kelondro/kelondroFlexSplitTable.java b/source/de/anomic/kelondro/kelondroFlexSplitTable.java index fe281f014..97560521b 100644 --- a/source/de/anomic/kelondro/kelondroFlexSplitTable.java +++ b/source/de/anomic/kelondro/kelondroFlexSplitTable.java @@ -137,7 +137,7 @@ public class kelondroFlexSplitTable implements kelondroIndex { return new String(suffix); } - public int size() throws IOException { + public int size() { Iterator i = tables.values().iterator(); int s = 0; while (i.hasNext()) { diff --git a/source/de/anomic/kelondro/kelondroFlexTable.java b/source/de/anomic/kelondro/kelondroFlexTable.java index 43427e6bb..d4a9ff95b 100644 --- a/source/de/anomic/kelondro/kelondroFlexTable.java +++ b/source/de/anomic/kelondro/kelondroFlexTable.java @@ -44,7 +44,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr private static TreeMap tableTracker = new TreeMap(); // class objects - protected kelondroBytesIntMap ROindex, RWindex; + protected kelondroBytesIntMap index; private boolean RAMIndex; public kelondroFlexTable(File path, String tablename, long preloadTime, kelondroRow rowdef, boolean resetOnFail) { @@ -91,8 +91,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr + " index entries initialized and sorted from " + super.col[0].size() + " keys."); RAMIndex = true; - ROindex = new kelondroBytesIntMap(ki); - RWindex = new kelondroBytesIntMap(new kelondroRowSet(new kelondroRow(new kelondroColumn[]{super.row().column(0), new kelondroColumn("int c-4 {b256}")}, super.rowdef.objectOrder, super.rowdef.primaryKey), 100)); + index = new kelondroBytesIntMap(ki); tableTracker.put(this.filename(), this); } else { // too less ram for a ram index @@ -111,8 +110,9 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr System.out.println(ki.size() + " entries indexed from " + super.col[0].size() + " keys."); RAMIndex = false; } - ROindex = null; - RWindex = new kelondroBytesIntMap(ki); + index = new kelondroBytesIntMap(ki); + assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); + } // assign index to wrapper description = "stt=" + Long.toString(System.currentTimeMillis() - start) + ";"; @@ -120,9 +120,8 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr } catch (IOException e) { if (resetOnFail) { RAMIndex = true; - ROindex = null; try { - RWindex = new kelondroBytesIntMap(new kelondroRowSet(new kelondroRow(new kelondroColumn[]{super.row().column(0), new kelondroColumn("int c-4 {b256}")}, super.rowdef.objectOrder, super.rowdef.primaryKey), 100)); + index = new kelondroBytesIntMap(new kelondroRowSet(new kelondroRow(new kelondroColumn[]{super.row().column(0), new kelondroColumn("int c-4 {b256}")}, super.rowdef.objectOrder, super.rowdef.primaryKey), 100)); } catch (IOException e1) { throw new kelondroException(e1.getMessage()); } @@ -135,8 +134,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr public void reset() throws IOException { super.reset(); RAMIndex = true; - ROindex = null; - RWindex = new kelondroBytesIntMap(new kelondroRowSet(new kelondroRow(new kelondroColumn[]{super.row().column(0), new kelondroColumn("int c-4 {b256}")}, super.rowdef.objectOrder, super.rowdef.primaryKey), 100)); + index = new kelondroBytesIntMap(new kelondroRowSet(new kelondroRow(new kelondroColumn[]{super.row().column(0), new kelondroColumn("int c-4 {b256}")}, super.rowdef.objectOrder, super.rowdef.primaryKey), 100)); } public static int staticSize(File path, String tablename) { @@ -155,7 +153,8 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr // it is not recommended to implement or use a has predicate unless // it can be ensured that it causes no IO if ((kelondroRecords.debugmode) && (RAMIndex != true)) serverLog.logWarning("kelondroFlexTable", "RAM index warning in file " + super.tablename); - return (RWindex.geti(key) >= 0) || ((ROindex != null) && (ROindex.geti(key) >= 0)); + assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); + return index.geti(key) >= 0; } private kelondroIndex initializeRamIndex() { @@ -175,6 +174,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr indexentry = ri.row().newEntry(); indexentry.setCol(0, key); indexentry.setCol(1, i); + //System.out.println("ENTRY: " + serverLog.arrayList(indexentry.bytes(), 0, indexentry.objectsize())); ri.addUnique(indexentry); if ((i % 10000) == 0) { System.out.print('.'); @@ -184,6 +184,9 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr System.out.print(" -ordering- "); System.out.flush(); ri.sort(); + int sbu = ri.size(); + ri.uniq(); + if (ri.size() != sbu) serverLog.logSevere("kelondroFlexTable.initializeRamIndex: " + tablename, "; size before uniq = " + sbu + ", after uniq = " + ri.size()); return ri; } @@ -217,20 +220,17 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr } public synchronized kelondroRow.Entry get(byte[] key) throws IOException { - int pos = RWindex.geti(key); - if (ROindex != null) { - if (pos < 0) { - pos = ROindex.geti(key); - } else { - assert ROindex.geti(key) < 0; - } - } - if (pos < 0) return null; - // i may be greater than this.size(), because this table may have deleted entries - // the deleted entries are subtracted from the 'real' tablesize, so the size may be - // smaller than an index to a row entry - return super.get(pos); - } + int pos = index.geti(key); + assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); + if (pos < 0) return null; + // i may be greater than this.size(), because this table may have deleted entries + // the deleted entries are subtracted from the 'real' tablesize, + // so the size may be smaller than an index to a row entry + kelondroRow.Entry result = super.get(pos); + assert result != null; + assert rowdef.objectOrder.compare(result.getColBytes(rowdef.primaryKey), key) == 0 : "key and row does not match; key = " + serverLog.arrayList(key, 0, key.length) + " row.key = " + serverLog.arrayList(result.getColBytes(rowdef.primaryKey), 0, rowdef.width(rowdef.primaryKey)); + return result; + } public synchronized void putMultiple(List rows, Date entryDate) throws IOException { // put a list of entries in a ordered way. @@ -241,11 +241,11 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr byte[] key; TreeMap old_rows_ordered = new TreeMap(); ArrayList new_rows_sequential = new ArrayList(); + assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); while (i.hasNext()) { row = (kelondroRow.Entry) i.next(); key = row.getColBytes(0); - pos = RWindex.geti(key); - if ((pos < 0) && (ROindex != null)) pos = ROindex.geti(key); + pos = index.geti(key); if (pos < 0) { new_rows_sequential.add(row); } else { @@ -257,10 +257,12 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr // write new entries to index addUniqueMultiple(new_rows_sequential, entryDate); + assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); } public synchronized kelondroRow.Entry put(kelondroRow.Entry row, Date entryDate) throws IOException { - return put(row); + assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); + return put(row); } public synchronized kelondroRow.Entry put(kelondroRow.Entry row) throws IOException { @@ -268,24 +270,43 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr assert (!(serverLog.allZero(row.getColBytes(0)))); assert row.objectsize() <= this.rowdef.objectsize; byte[] key = row.getColBytes(0); - int pos = RWindex.geti(key); - if ((pos < 0) && (ROindex != null)) pos = ROindex.geti(key); + int pos = index.geti(key); if (pos < 0) { - RWindex.puti(key, super.add(row)); + pos = super.add(row); + index.puti(key, pos); + assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); return null; } + //System.out.println("row.key=" + serverLog.arrayList(row.bytes(), 0, row.objectsize())); kelondroRow.Entry oldentry = super.get(pos); + assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); + if (oldentry == null) { + serverLog.logSevere("kelondroFlexTable", "put(): index failure; the index pointed to a cell which is empty. content.size() = " + this.size() + ", index.size() = " + ((index == null) ? 0 : index.size())); + // patch bug ***** FIND CAUSE! (see also: remove) + int oldindex = index.removei(key); + assert oldindex >= 0; + assert index.geti(key) == -1; + // here is this.size() > index.size() because of remove operation above + index.puti(key, super.add(row)); + assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); + return null; + } + assert oldentry != null : "overwrite of empty position " + pos + ", index management must have failed before"; + assert rowdef.objectOrder.compare(oldentry.getColBytes(rowdef.primaryKey), key) == 0 : "key and row does not match; key = " + serverLog.arrayList(key, 0, key.length) + " row.key = " + serverLog.arrayList(oldentry.getColBytes(rowdef.primaryKey), 0, rowdef.width(rowdef.primaryKey)); super.set(pos, row); + assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); return oldentry; } public synchronized void addUnique(kelondroRow.Entry row, Date entryDate) throws IOException { - addUnique(row); + assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); + addUnique(row); } public synchronized void addUnique(kelondroRow.Entry row) throws IOException { assert row.objectsize() == this.rowdef.objectsize; - RWindex.addi(row.getColBytes(0), super.add(row)); + assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); + index.addi(row.getColBytes(0), super.add(row)); } public synchronized void addUniqueMultiple(List rows, Date entryDate) throws IOException { @@ -298,49 +319,48 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr Map.Entry entry; while (i.hasNext()) { entry = (Map.Entry) i.next(); - RWindex.puti((byte[]) entry.getValue(), ((Integer) entry.getKey()).intValue()); + index.puti((byte[]) entry.getValue(), ((Integer) entry.getKey()).intValue()); } + assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); + } public synchronized kelondroRow.Entry remove(byte[] key) throws IOException { - int i = RWindex.removei(key); - if (ROindex != null) { - if (i < 0) { - i = ROindex.removei(key); // yes, we are allowed to remove entries from RO partition of the index - } else { - assert ROindex.removei(key) < 0; - } + int i = index.removei(key); + assert (index.geti(key) < 0); // must be deleted + if (i < 0) { + assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); + return null; } - assert (RWindex.removei(key) < 0); - assert (ROindex == null) || (ROindex.removei(key) < 0); - if (i < 0) return null; kelondroRow.Entry r = super.get(i); - assert r != null; // error + if (r == null) { + serverLog.logSevere("kelondroFlexTable", "remove(): index failure; the index pointed to a cell which is empty. content.size() = " + this.size() + ", index.size() = " + ((index == null) ? 0 : index.size())); + // patch bug ***** FIND CAUSE! (see also: put) + assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); + return null; + } + assert r != null : "r == null"; // should be avoided with path above + assert rowdef.objectOrder.compare(r.getColBytes(rowdef.primaryKey), key) == 0 : "key and row does not match; key = " + serverLog.arrayList(key, 0, key.length) + " row.key = " + serverLog.arrayList(r.getColBytes(rowdef.primaryKey), 0, rowdef.width(rowdef.primaryKey)); super.remove(i); assert super.get(i) == null : "i = " + i + ", get(i) = " + serverLog.arrayList(super.get(i).bytes(), 0, 12); - return r; + assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); + return r; } public synchronized kelondroRow.Entry removeOne() throws IOException { - int i = RWindex.removeonei(); - if ((i < 0) && (ROindex != null)) i = ROindex.removeonei(); + int i = index.removeonei(); if (i < 0) return null; kelondroRow.Entry r; r = super.get(i); super.remove(i); - return r; + assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); + return r; } public synchronized kelondroCloneableIterator rows(boolean up, byte[] firstKey) throws IOException { - if (ROindex == null) return new rowIterator(RWindex, up, firstKey); - if (RWindex == null) return new rowIterator(ROindex, up, firstKey); - return new kelondroMergeIterator( - new rowIterator(ROindex, up, firstKey), - new rowIterator(RWindex, up, firstKey), - row().objectOrder, - kelondroMergeIterator.simpleMerge, - up - ); + if (index == null) return new rowIterator(index, up, firstKey); + assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); + return new rowIterator(index, up, firstKey); } public class rowIterator implements kelondroCloneableIterator { @@ -392,11 +412,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr } public kelondroProfile profile() { - if (ROindex == null) { - return RWindex.profile(); - } else { - return kelondroProfile.consolidate(ROindex.profile(), RWindex.profile()); - } + return index.profile(); } public static final Iterator filenames() { @@ -424,11 +440,9 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr // returns statistical data about this object HashMap map = new HashMap(); try { - map.put("tableIndexChunkSize", (!RAMIndex) ? "0" : Integer.toString(RWindex.row().objectsize)); - map.put("tableROIndexCount", ((!RAMIndex) || (ROindex == null)) ? "0" : Integer.toString(ROindex.size())); - map.put("tableROIndexMem", ((!RAMIndex) || (ROindex == null)) ? "0" : Integer.toString((int) (ROindex.row().objectsize * ROindex.size()))); - map.put("tableRWIndexCount", (!RAMIndex) ? "0" : Integer.toString(RWindex.size())); - map.put("tableRWIndexMem", (!RAMIndex) ? "0" : Integer.toString((int) (RWindex.row().objectsize * RWindex.size() * kelondroRowCollection.growfactor))); + map.put("tableIndexChunkSize", (!RAMIndex) ? "0" : Integer.toString(index.row().objectsize)); + map.put("tableIndexCount", (!RAMIndex) ? "0" : Integer.toString(index.size())); + map.put("tableIndexMem", (!RAMIndex) ? "0" : Integer.toString((int) (index.row().objectsize * index.size() * kelondroRowCollection.growfactor))); } catch (IOException e) { } return map; @@ -438,8 +452,11 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr if (tableTracker.remove(this.filename) == null) { serverLog.logWarning("kelondroFlexTable", "close(): file '" + this.filename + "' was not tracked with record tracker."); } - if (ROindex != null) {ROindex.close(); ROindex = null;} - if (RWindex != null) {RWindex.close(); RWindex = null;} + if ((index != null) && (this.size() != ((index == null) ? 0 : index.size()))) { + serverLog.logSevere("kelondroFlexTable", "close(): inconsistent content/index size. content.size() = " + this.size() + ", index.size() = " + ((index == null) ? 0 : index.size())); + } + + if (index != null) {index.close(); index = null;} super.close(); } diff --git a/source/de/anomic/kelondro/kelondroFlexWidthArray.java b/source/de/anomic/kelondro/kelondroFlexWidthArray.java index 565dd331d..2a89c6b64 100644 --- a/source/de/anomic/kelondro/kelondroFlexWidthArray.java +++ b/source/de/anomic/kelondro/kelondroFlexWidthArray.java @@ -231,7 +231,7 @@ public class kelondroFlexWidthArray implements kelondroArray { rowentry = (kelondroRow.Entry) entry.getValue(); assert rowentry.objectsize() == this.rowdef.objectsize; - e = col[c].row().newEntry(rowentry.bytes(), rowdef.colstart[c]); + e = col[c].row().newEntry(rowentry.bytes(), rowdef.colstart[c], false); col[c].set(index, e); } c = c + col[c].row().columns(); @@ -244,7 +244,7 @@ public class kelondroFlexWidthArray implements kelondroArray { kelondroRow.Entry e; byte[] reb = rowentry.bytes(); while (c < rowdef.columns()) { - e = col[c].row().newEntry(reb, rowdef.colstart[c]); + e = col[c].row().newEntry(reb, rowdef.colstart[c], false); col[c].set(index, e); c = c + col[c].row().columns(); } @@ -252,16 +252,13 @@ public class kelondroFlexWidthArray implements kelondroArray { public synchronized int add(kelondroRow.Entry rowentry) throws IOException { assert rowentry.objectsize() == this.rowdef.objectsize; - kelondroRow.Entry e; int index = -1; byte[] reb = rowentry.bytes(); - e = col[0].row().newEntry(reb, 0); - index = col[0].add(e); + index = col[0].add(col[0].row().newEntry(reb, 0, false)); int c = col[0].row().columns(); while (c < rowdef.columns()) { - e = col[c].row().newEntry(reb, rowdef.colstart[c]); - col[c].set(index, e); + col[c].set(index, col[c].row().newEntry(reb, rowdef.colstart[c], false)); c = c + col[c].row().columns(); } return index; @@ -286,12 +283,12 @@ public class kelondroFlexWidthArray implements kelondroArray { kelondroRow.Entry e; int index = -1; byte[] reb = rowentry.bytes(); - e = col[0].row().newEntry(reb, 0); + e = col[0].row().newEntry(reb, 0, false); index = col[0].add(e); int c = col[0].row().columns(); while (c < rowdef.columns()) { - e = col[c].row().newEntry(reb, rowdef.colstart[c]); + e = col[c].row().newEntry(reb, rowdef.colstart[c], false); // remember write to column, but do not write directly colm[c].put(new Integer(index), e); // col[c].set(index,e); c = c + col[c].row().columns(); diff --git a/source/de/anomic/kelondro/kelondroIndex.java b/source/de/anomic/kelondro/kelondroIndex.java index c56769a3a..d44cfc090 100644 --- a/source/de/anomic/kelondro/kelondroIndex.java +++ b/source/de/anomic/kelondro/kelondroIndex.java @@ -57,7 +57,7 @@ import java.util.List; public interface kelondroIndex { public String filename(); // returns a unique identified for this index; can be a real or artificial file name - public int size() throws IOException; + public int size(); public kelondroProfile profile(); public kelondroRow row(); public boolean has(byte[] key) throws IOException; // use this only if there is no get in case that has returns true diff --git a/source/de/anomic/kelondro/kelondroIntBytesMap.java b/source/de/anomic/kelondro/kelondroIntBytesMap.java index 6fda2f02d..714a1fa0e 100644 --- a/source/de/anomic/kelondro/kelondroIntBytesMap.java +++ b/source/de/anomic/kelondro/kelondroIntBytesMap.java @@ -24,9 +24,10 @@ package de.anomic.kelondro; +import java.util.ArrayList; +import java.util.HashSet; import java.util.Iterator; - -//import java.util.Random; +import java.util.Random; public class kelondroIntBytesMap { @@ -58,7 +59,6 @@ public class kelondroIntBytesMap { newentry.setCol(1, value); index.addUnique(newentry); } - public byte[] putb(int ii, byte[] value) { kelondroRow.Entry newentry; @@ -100,19 +100,40 @@ public class kelondroIntBytesMap { } public static void main(String[] args) { - long start = System.currentTimeMillis(); - kelondroIntBytesMap c = new kelondroIntBytesMap(30, 0); - //Random random = new Random(0); - int x; - for (int i = 0; i < 100000; i++) { - //x = random.nextInt(100000); - x = i; - c.putb(x, new byte[30]); - //if (c.getb(x) == null) System.out.println("consistency error at " + i + " with key " + x); - if (i % 10000 == 0) System.out.println(i + " entries. "); - } - System.out.println("RESULT SIZE: " + c.size()); - System.out.println("Time: " + ((System.currentTimeMillis() - start) / 1000) + " seconds"); - } + boolean assertEnabled = false; + assert assertEnabled = true; + System.out.println((assertEnabled) ? "asserts are enabled" : "enable asserts with 'java -ea'; not enabled yet"); + long start = System.currentTimeMillis(); + long randomstart = 0; + Random random = new Random(randomstart); + long r; + Long R; + int p, rc = 0; + ArrayList ra = new ArrayList(); + HashSet jcontrol = new HashSet(); + kelondroIntBytesMap kcontrol = new kelondroIntBytesMap(1, 0); + for (int i = 0; i < 1000000; i++) { + r = Math.abs(random.nextLong() % 10000); + //System.out.println("add " + r); + jcontrol.add(new Long(r)); + kcontrol.putb((int) r, "x".getBytes()); + if (random.nextLong() % 5 == 0) ra.add(new Long(r)); + if ((ra.size() > 0) && (random.nextLong() % 7 == 0)) { + rc++; + p = Math.abs(random.nextInt()) % ra.size(); + R = (Long) ra.get(p); + //System.out.println("remove " + R.longValue()); + jcontrol.remove(R); + kcontrol.removeb((int) R.longValue()); + assert kcontrol.removeb((int) R.longValue()) == null; + } + assert jcontrol.size() == kcontrol.size(); + } + System.out.println("removed: " + rc + ", size of jcontrol set: " + + jcontrol.size() + ", size of kcontrol set: " + + kcontrol.size()); + System.out.println("Time: " + + ((System.currentTimeMillis() - start) / 1000) + " seconds"); + } } diff --git a/source/de/anomic/kelondro/kelondroMapTable.java b/source/de/anomic/kelondro/kelondroMapTable.java index eb74c028a..2629d50c6 100644 --- a/source/de/anomic/kelondro/kelondroMapTable.java +++ b/source/de/anomic/kelondro/kelondroMapTable.java @@ -178,7 +178,7 @@ public class kelondroMapTable { if (table != null) return table.size(); kelondroIndex Tree = (kelondroIndex) tTables.get(tablename); - if (Tree != null) try { return Tree.size(); } catch (IOException e) {return 0;} + if (Tree != null) return Tree.size(); throw new RuntimeException("kelondroTables.accumulator: table '" + tablename + "' does not exist."); } diff --git a/source/de/anomic/kelondro/kelondroNaturalOrder.java b/source/de/anomic/kelondro/kelondroNaturalOrder.java index cd4f188f7..e3bfb4a8a 100644 --- a/source/de/anomic/kelondro/kelondroNaturalOrder.java +++ b/source/de/anomic/kelondro/kelondroNaturalOrder.java @@ -168,9 +168,10 @@ public class kelondroNaturalOrder extends kelondroAbstractOrder implements kelon int i = 0; final int al = Math.min(alength, a.length - aoffset); final int bl = Math.min(blength, b.length - boffset); - final int len = Math.min(al, bl); + if (al > bl) return 1; + if (al < bl) return -1; int aa, bb; - while (i < len) { + while (i < al) { aa = 0xff & (int) a[i + aoffset]; bb = 0xff & (int) b[i + boffset]; if (aa > bb) return 1; @@ -178,13 +179,7 @@ public class kelondroNaturalOrder extends kelondroAbstractOrder implements kelon // else the bytes are equal and it may go on yet undecided i++; } - // check if we have a zero-terminated equality - if ((i == al) && (i < bl) && (b[i + boffset] == 0)) return 0; - if ((i == bl) && (i < al) && (a[i + aoffset] == 0)) return 0; - // no, decide by length - if (al > bl) return 1; - if (al < bl) return -1; - // no, they are equal + // they are equal return 0; } diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java index 7f3890112..e2ed97e21 100644 --- a/source/de/anomic/kelondro/kelondroRecords.java +++ b/source/de/anomic/kelondro/kelondroRecords.java @@ -874,6 +874,10 @@ public class kelondroRecords { System.arraycopy(rowinstance, ROW.width(0), this.tailChunk, 0, tailchunksize); } + if (cacheHeaders != null) synchronized (cacheHeaders) { + updateNodeCache(); + } + // mark chunks as changed // if the head/tail chunks come from a file system read, setChanged should be false // if the chunks come from a overwrite attempt, it should be true @@ -960,7 +964,6 @@ public class kelondroRecords { } } else synchronized(cacheHeaders) { byte[] cacheEntry = null; - int cp = CP_HIGH; cacheEntry = cacheHeaders.getb(this.handle.index); if (cacheEntry == null) { // cache miss, we read overhead and key from file @@ -981,17 +984,8 @@ public class kelondroRecords { entryFile.readFully(seekpos(this.handle), this.headChunk, 0, headchunksize); } - // calculate cache priority - cp = CP_HIGH; - if (OHHANDLEC == 3) { - Handle l = getOHHandle(1); - Handle r = getOHHandle(2); - if ((l == null) && (r == null)) cp = CP_LOW; - else if ((l == null) || (r == null)) cp = CP_MEDIUM; - } - // if space left in cache, copy these value to the cache - updateNodeCache(cp); + updateNodeCache(); } else { readHit++; this.headChunk = cacheEntry; @@ -1116,7 +1110,7 @@ public class kelondroRecords { //System.out.println("WRITEH(" + filename + ", " + seekpos(this.handle) + ", " + this.headChunk.length + ")"); assert (headChunk == null) || (headChunk.length == headchunksize); entryFile.write(seekpos(this.handle), (this.headChunk == null) ? new byte[headchunksize] : this.headChunk); - updateNodeCache(cachePriority); + updateNodeCache(); this.headChanged = false; } @@ -1177,14 +1171,11 @@ public class kelondroRecords { return cacheGrowStatus() > 0; } - private void updateNodeCache(int priority) { + private void updateNodeCache() { if (this.handle == null) return; // wrong access if (this.headChunk == null) return; // nothing there to cache - if (priority == CP_NONE) return; // it is not wanted that this shall be cached if (cacheHeaders == null) return; // we do not use the cache - if (!(cacheSpace())) return; - - synchronized (cacheHeaders) { + if (cacheSpace()) synchronized (cacheHeaders) { // generate cache entry //byte[] cacheEntry = new byte[headchunksize]; //System.arraycopy(headChunk, 0, cacheEntry, 0, headchunksize); @@ -1196,6 +1187,11 @@ public class kelondroRecords { //System.out.println("kelondroRecords cache4" + filename + ": cache record size = " + (memBefore - Runtime.getRuntime().freeMemory()) + " bytes" + ((newentry) ? " new" : "")); //printCache(); + } else { + // there shall be no entry in the cache. If one exists, we remove it + boolean rem = false; + rem = (cacheHeaders.removeb(this.handle.index) != null); + if (rem) cacheDelete++; } } @@ -1636,8 +1632,12 @@ public class kelondroRecords { printCache(); System.out.println("--"); System.out.println("NODES"); - for (int i = 0; i < USAGE.allCount(); i++) - System.out.println("NODE: " + new Node(new Handle(i), (Node) null, 0, true).toString()); + Iterator i = new contentNodeIterator(-1); + Node n; + while (i.hasNext()) { + n = (Node) i.next(); + System.out.println("NODE: " + n.toString()); + } } public String toString() { @@ -1712,4 +1712,40 @@ public class kelondroRecords { return kelondroProfile.consolidate(profiles()); } + public static void main(String[] args) { + File testfile = new File("kelondroRecordsTest.records"); + kelondroRow testRow = new kelondroRow("byte[] key-16, value-16", kelondroBase64Order.enhancedCoder, 0); + + // generate + if (testfile.exists()) testfile.delete(); + try { + kelondroRecords r = new kelondroRecords(testfile, true, (long) -1, + (short) 0, (short) 0, testRow, 0, 0, 0); + + r.newNode("ABCDEFGHIJKLMNOPQRSTUVWXYZ012345".getBytes()); + r.newNode("abcdefghijklmnopqrstuvwxyz012345".getBytes()); + r.newNode("A______________________________B".getBytes()); + r.newNode("C______________________________D".getBytes()); + r.newNode("E______________________________F".getBytes()); + r.newNode("X______________________________Y".getBytes()); + r.deleteNode(r.new Handle(1)); + r.deleteNode(r.new Handle(3)); + r.newNode("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".getBytes()); + r.newNode("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".getBytes()); + r.newNode("cccccccccccccccccccccccccccccccc".getBytes()); + r.close(); + } catch (IOException e) { + e.printStackTrace(); + } + + // re-open + try { + kelondroRecords r = new kelondroRecords(testfile, true, (long) -1, + (short) 0, (short) 0, testRow, 0, 0, 0); + r.print(true); + } catch (IOException e) { + e.printStackTrace(); + } + } + } diff --git a/source/de/anomic/kelondro/kelondroRow.java b/source/de/anomic/kelondro/kelondroRow.java index 319921dbb..d2966d1f7 100644 --- a/source/de/anomic/kelondro/kelondroRow.java +++ b/source/de/anomic/kelondro/kelondroRow.java @@ -151,21 +151,24 @@ public class kelondroRow { //assert (rowinstance[0] != 0); assert (this.objectOrder.wellformed(rowinstance, 0, row[0].cellwidth())) : "rowinstance[0] = " + serverLog.arrayList(rowinstance, 0, row[0].cellwidth()); if (!(this.objectOrder.wellformed(rowinstance, 0, row[0].cellwidth()))) return null; - return new Entry(rowinstance); + return new Entry(rowinstance, false); } public Entry newEntry(Entry oldrow, int fromColumn) { if (oldrow == null) return null; assert (oldrow.getColBytes(0)[0] != 0); assert (this.objectOrder.wellformed(oldrow.getColBytes(0), 0, row[0].cellwidth())); - return new Entry(oldrow, fromColumn); + return new Entry(oldrow, fromColumn, false); } - public Entry newEntry(byte[] rowinstance, int start) { + public Entry newEntry(byte[] rowinstance, int start, boolean clone) { if (rowinstance == null) return null; //assert (rowinstance[0] != 0); assert (this.objectOrder.wellformed(rowinstance, start, row[0].cellwidth())); - return new Entry(rowinstance, start); + // this method offers the option to clone the content + // this is necessary if it is known that the underlying byte array may change and therefore + // the reference to the byte array does not contain the original content + return new Entry(rowinstance, start, clone); } public Entry newEntry(byte[][] cells) { @@ -198,22 +201,23 @@ public class kelondroRow { offset = 0; } - public Entry(byte[] newrow) { - this(newrow, 0); + public Entry(byte[] newrow, boolean forceclone) { + this(newrow, 0, forceclone); } - public Entry(Entry oldrow, int fromColumn) { - this(oldrow.rowinstance, oldrow.offset + oldrow.colstart(fromColumn)); + public Entry(Entry oldrow, int fromColumn, boolean forceclone) { + this(oldrow.rowinstance, oldrow.offset + oldrow.colstart(fromColumn), forceclone); } - public Entry(byte[] newrow, int start) { - if (newrow.length - start >= objectsize) { + public Entry(byte[] newrow, int start, boolean forceclone) { + if ((!forceclone) && (newrow.length - start >= objectsize)) { this.rowinstance = newrow; + this.offset = start; } else { this.rowinstance = new byte[objectsize]; - System.arraycopy(newrow, start, this.rowinstance, 0, newrow.length - start); + System.arraycopy(newrow, start, this.rowinstance, 0, objectsize); + this.offset = 0; } - this.offset = start; //for (int i = ll; i < objectsize; i++) this.rowinstance[i] = 0; } @@ -500,6 +504,8 @@ public class kelondroRow { } public byte[] getColBytes(int column) { + assert offset + colstart[column] + row[column].cellwidth() <= rowinstance.length : + "column = " + column + ", offset = " + offset + ", colstart[column] = " + colstart[column] + ", row[column].cellwidth() = " + row[column].cellwidth() + ", rowinstance.length = " + rowinstance.length; byte[] c = new byte[row[column].cellwidth()]; System.arraycopy(rowinstance, offset + colstart[column], c, 0, row[column].cellwidth()); return c; @@ -552,7 +558,7 @@ public class kelondroRow { public final class EntryIndex extends Entry { private int index; public EntryIndex(byte[] row, int i) { - super(row); + super(row, false); this.index = i; } public int index() { diff --git a/source/de/anomic/kelondro/kelondroRowCollection.java b/source/de/anomic/kelondro/kelondroRowCollection.java index 2de360de1..65677b4d8 100644 --- a/source/de/anomic/kelondro/kelondroRowCollection.java +++ b/source/de/anomic/kelondro/kelondroRowCollection.java @@ -39,7 +39,7 @@ public class kelondroRowCollection { public static final double growfactor = 1.4; - private byte[] chunkcache; + protected byte[] chunkcache; protected int chunkcount; protected long lastTimeRead, lastTimeWrote; protected kelondroRow rowdef; @@ -222,7 +222,7 @@ public class kelondroRowCollection { if (index >= chunkcount) return null; if (index * rowdef.objectsize() >= chunkcache.length) return null; this.lastTimeRead = System.currentTimeMillis(); - return rowdef.newEntry(chunkcache, index * rowdef.objectsize()); + return rowdef.newEntry(chunkcache, index * rowdef.objectsize(), true); } public synchronized final void set(int index, kelondroRow.Entry a) { @@ -235,7 +235,8 @@ public class kelondroRowCollection { } public synchronized void addUnique(kelondroRow.Entry row) { - addUnique(row.bytes(), 0, row.bytes().length); + byte[] r = row.bytes(); + addUnique(r, 0, r.length); } public synchronized void addUnique(kelondroRow.Entry row, Date entryDate) { @@ -288,30 +289,27 @@ public class kelondroRowCollection { System.arraycopy(c.chunkcache, 0, chunkcache, rowdef.objectsize() * chunkcount, rowdef.objectsize() * c.size()); chunkcount += c.size(); } - - private final void removeShift(int pos, int dist, int upBound) { - assert ((pos + dist) * rowdef.objectsize() >= 0) : "pos = " + pos + ", dist = " + dist + ", rowdef.objectsize() = " + rowdef.objectsize; - assert (pos * rowdef.objectsize() >= 0) : "pos = " + pos + ", rowdef.objectsize() = " + rowdef.objectsize; - assert ((pos + dist) * rowdef.objectsize() + (upBound - pos - dist) * rowdef.objectsize() <= chunkcache.length) : "pos = " + pos + ", dist = " + dist + ", rowdef.objectsize() = " + rowdef.objectsize + ", upBound = " + upBound + ", chunkcache.length = " + chunkcache.length; - assert (pos * rowdef.objectsize() + (upBound - pos - dist) * rowdef.objectsize() <= chunkcache.length) : "pos = " + pos + ", dist = " + dist + ", rowdef.objectsize() = " + rowdef.objectsize + ", upBound = " + upBound + ", chunkcache.length = " + chunkcache.length; - System.arraycopy(chunkcache, (pos + dist) * rowdef.objectsize(), - chunkcache, pos * rowdef.objectsize(), - (upBound - pos - dist) * rowdef.objectsize()); - } - - private final void copytop(int i) { - // copies the topmost row element to given position - if (i == chunkcount - 1) return; - System.arraycopy(chunkcache, this.rowdef.objectsize() * (chunkcount - 1), chunkcache, this.rowdef.objectsize() * i, this.rowdef.objectsize()); - } protected synchronized final void removeRow(int p) { - assert ((p >= 0) && (p < chunkcount) && (chunkcount > 0)) : "p = " + p + ", chunkcount = " + chunkcount; + assert p >= 0 : "p = " + p; + assert p < chunkcount : "p = " + p + ", chunkcount = " + chunkcount; + assert chunkcount > 0 : "chunkcount = " + chunkcount; + assert sortBound <= chunkcount : "sortBound = " + sortBound + ", chunkcount = " + chunkcount; if (p < sortBound) { - removeShift(p, 1, chunkcount); + // remove by shift + System.arraycopy( + chunkcache, (p + 1) * this.rowdef.objectsize(), + chunkcache, p * this.rowdef.objectsize(), + (chunkcount - p - 1) * this.rowdef.objectsize()); sortBound--; } else { - copytop(p); + // remove by copying the top-element to the remove position + if (p != chunkcount - 1) { + System.arraycopy( + chunkcache, (chunkcount - 1) * this.rowdef.objectsize(), + chunkcache, p * this.rowdef.objectsize(), + this.rowdef.objectsize()); + } } chunkcount--; this.lastTimeWrote = System.currentTimeMillis(); @@ -482,6 +480,8 @@ public class kelondroRowCollection { if (chunkcount <= 1) return; int i = 0; while (i < chunkcount - 1) { + //System.out.println("ENTRY0: " + serverLog.arrayList(chunkcache, rowdef.objectsize*i, rowdef.objectsize)); + //System.out.println("ENTRY1: " + serverLog.arrayList(chunkcache, rowdef.objectsize*(i+1), rowdef.objectsize)); if (compare(i, i + 1) == 0) { removeRow(i); } else { diff --git a/source/de/anomic/kelondro/kelondroRowSet.java b/source/de/anomic/kelondro/kelondroRowSet.java index d4e1848c3..74910be7d 100644 --- a/source/de/anomic/kelondro/kelondroRowSet.java +++ b/source/de/anomic/kelondro/kelondroRowSet.java @@ -30,6 +30,8 @@ import java.util.Iterator; import java.util.List; import java.util.Random; +import de.anomic.server.logging.serverLog; + public class kelondroRowSet extends kelondroRowCollection implements kelondroIndex { private static final int collectionReSortLimit = 90; @@ -107,9 +109,12 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd private synchronized kelondroRow.Entry remove(byte[] a, int start, int length) { int index = find(a, start, length); if (index < 0) return null; + //System.out.println("remove: chunk found at index position (before remove) " + index + ", inset=" + serverLog.arrayList(super.chunkcache, super.rowdef.objectsize() * index, length + 10) + ", searchkey=" + serverLog.arrayList(a, start, length)); kelondroRow.Entry entry = super.get(index); super.removeRow(index); - assert find(a, start, length) < 0; // check if the remove worked + //System.out.println("remove: chunk found at index position (after remove) " + index + ", inset=" + serverLog.arrayList(super.chunkcache, super.rowdef.objectsize() * index, length) + ", searchkey=" + serverLog.arrayList(a, start, length)); + int findagainindex = find(a, start, length); + assert findagainindex < 0 : "remove: chunk found again at index position (after remove) " + findagainindex + ", inset=" + serverLog.arrayList(super.chunkcache, super.rowdef.objectsize() * findagainindex, length) + ", searchkey=" + serverLog.arrayList(a, start, length); // check if the remove worked return entry; } @@ -132,7 +137,9 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd if (rowdef.objectOrder == null) return iterativeSearch(a, astart, alength, 0, this.chunkcount); // check if a re-sorting make sense - if ((this.chunkcount - this.sortBound) > collectionReSortLimit) sort(); + if ((this.chunkcount - this.sortBound) > collectionReSortLimit) { + sort(); + } // first try to find in sorted area int p = binarySearch(a, astart, alength); diff --git a/source/de/anomic/plasma/plasmaCrawlBalancer.java b/source/de/anomic/plasma/plasmaCrawlBalancer.java index 3fc79d1e4..ebd3446d9 100644 --- a/source/de/anomic/plasma/plasmaCrawlBalancer.java +++ b/source/de/anomic/plasma/plasmaCrawlBalancer.java @@ -192,19 +192,15 @@ public class plasmaCrawlBalancer { public synchronized int size() { int componentsize = urlFileStack.size() + urlRAMStack.size() + sizeDomainStacks(); - try { - if (componentsize != urlFileIndex.size()) { - // here is urlIndexFile.size() always smaller. why? - if (kelondroRecords.debugmode) { - serverLog.logWarning("PLASMA BALANCER", "size operation wrong in " + stackname + " - componentsize = " + componentsize + ", urlFileIndex.size() = " + urlFileIndex.size()); - } - if ((componentsize == 0) && (urlFileIndex.size() > 0)) { - resetFileIndex(); - } - } - } catch (IOException e) { - e.printStackTrace(); - } + if (componentsize != urlFileIndex.size()) { + // here is urlIndexFile.size() always smaller. why? + if (kelondroRecords.debugmode) { + serverLog.logWarning("PLASMA BALANCER", "size operation wrong in " + stackname + " - componentsize = " + componentsize + ", urlFileIndex.size() = " + urlFileIndex.size()); + } + if ((componentsize == 0) && (urlFileIndex.size() > 0)) { + resetFileIndex(); + } + } return componentsize; } diff --git a/source/de/anomic/plasma/plasmaCrawlLURL.java b/source/de/anomic/plasma/plasmaCrawlLURL.java index d3b949f08..9738ef4cc 100644 --- a/source/de/anomic/plasma/plasmaCrawlLURL.java +++ b/source/de/anomic/plasma/plasmaCrawlLURL.java @@ -112,11 +112,7 @@ public final class plasmaCrawlLURL { } public int size() { - try { - return urlIndexFile.size(); - } catch (IOException e) { - return 0; - } + return urlIndexFile.size(); } public void close() { diff --git a/source/de/anomic/plasma/plasmaCrawlZURL.java b/source/de/anomic/plasma/plasmaCrawlZURL.java index ee54fe6c1..48e449fde 100644 --- a/source/de/anomic/plasma/plasmaCrawlZURL.java +++ b/source/de/anomic/plasma/plasmaCrawlZURL.java @@ -62,11 +62,7 @@ public class plasmaCrawlZURL { } public int size() { - try { - return urlIndexFile.size() ; - } catch (IOException e) { - return 0; - } + return urlIndexFile.size() ; } public void close() { diff --git a/source/de/anomic/server/logging/serverLog.java b/source/de/anomic/server/logging/serverLog.java index 4e126b328..406ff22ee 100644 --- a/source/de/anomic/server/logging/serverLog.java +++ b/source/de/anomic/server/logging/serverLog.java @@ -221,6 +221,7 @@ public final class serverLog { public static final String arrayList(byte[] b, int start, int length) { if (b == null) return "NULL"; if (b.length == 0) return "[]"; + length = Math.min(length, b.length - start); StringBuffer sb = new StringBuffer(b.length * 4); sb.append('[').append(Integer.toString((int) b[start])).append(','); for (int i = 1; i < length; i++) sb.append(' ').append(Integer.toString((int) b[start + i])).append(',');