From 5bb565944f5b7bad68633507417779f708921a32 Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 30 May 2006 14:36:20 +0000 Subject: [PATCH] integration of new kelondroRow into some parts of kelondro, especially into the array storage git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2155 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/de/anomic/index/indexRAMCacheRI.java | 33 ++--- source/de/anomic/kelondro/kelondroArray.java | 45 +++--- .../kelondro/kelondroCollectionIndex.java | 12 +- source/de/anomic/kelondro/kelondroColumn.java | 12 +- .../de/anomic/kelondro/kelondroHashtable.java | 35 ++--- .../anomic/kelondro/kelondroNaturalOrder.java | 27 +++- .../de/anomic/kelondro/kelondroRecords.java | 93 ++++++------ source/de/anomic/kelondro/kelondroRow.java | 134 ++++++++++++++++-- source/de/anomic/kelondro/kelondroStack.java | 16 +-- source/de/anomic/kelondro/kelondroTables.java | 13 -- source/de/anomic/kelondro/kelondroTree.java | 51 ++----- .../plasma/plasmaWordIndexAssortment.java | 8 +- 12 files changed, 286 insertions(+), 193 deletions(-) diff --git a/source/de/anomic/index/indexRAMCacheRI.java b/source/de/anomic/index/indexRAMCacheRI.java index e48dfcfc7..b0b048ddc 100644 --- a/source/de/anomic/index/indexRAMCacheRI.java +++ b/source/de/anomic/index/indexRAMCacheRI.java @@ -35,7 +35,8 @@ import java.util.TreeMap; import de.anomic.kelondro.kelondroArray; import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroMScoreCluster; -import de.anomic.kelondro.kelondroRecords; +import de.anomic.kelondro.kelondroNaturalOrder; +import de.anomic.kelondro.kelondroRow; import de.anomic.plasma.plasmaWordIndexAssortment; import de.anomic.server.logging.serverLog; import de.anomic.yacy.yacySeedDB; @@ -102,7 +103,7 @@ public final class indexRAMCacheRI extends indexAbstractRI implements indexRI { indexTreeMapContainer container; long updateTime; indexURLEntry wordEntry; - byte[][] row = new byte[5][]; + kelondroRow.Entry row = dumpArray.row().newEntry(); // write kCache, this will be melted with the wCache upon load synchronized (kCache) { @@ -115,11 +116,11 @@ public final class indexRAMCacheRI extends indexAbstractRI implements indexRI { Iterator ci = container.entries(); while (ci.hasNext()) { wordEntry = (indexURLEntry) ci.next(); - row[0] = container.wordHash().getBytes(); - row[1] = kelondroRecords.long2bytes(container.size(), 4); - row[2] = kelondroRecords.long2bytes(container.updated(), 8); - row[3] = wordEntry.getUrlHash().getBytes(); - row[4] = wordEntry.toEncodedStringForm().getBytes(); + row.setCol(0, container.wordHash().getBytes()); + row.setCol(1, kelondroNaturalOrder.encodeLong(container.size(), 4)); + row.setCol(2, kelondroNaturalOrder.encodeLong(container.updated(), 8)); + row.setCol(3, wordEntry.getUrlHash().getBytes()); + row.setCol(4, wordEntry.toEncodedStringForm().getBytes()); dumpArray.set((int) urlcount++, row); } } @@ -144,11 +145,11 @@ public final class indexRAMCacheRI extends indexAbstractRI implements indexRI { Iterator ci = container.entries(); while (ci.hasNext()) { wordEntry = (indexURLEntry) ci.next(); - row[0] = wordHash.getBytes(); - row[1] = kelondroRecords.long2bytes(container.size(), 4); - row[2] = kelondroRecords.long2bytes(updateTime, 8); - row[3] = wordEntry.getUrlHash().getBytes(); - row[4] = wordEntry.toEncodedStringForm().getBytes(); + row.setCol(0, wordHash.getBytes()); + row.setCol(1, kelondroNaturalOrder.encodeLong(container.size(), 4)); + row.setCol(2, kelondroNaturalOrder.encodeLong(updateTime, 8)); + row.setCol(3, wordEntry.getUrlHash().getBytes()); + row.setCol(4, wordEntry.toEncodedStringForm().getBytes()); dumpArray.set((int) urlcount++, row); } } @@ -183,15 +184,15 @@ public final class indexRAMCacheRI extends indexAbstractRI implements indexRI { String wordHash; //long creationTime; indexURLEntry wordEntry; - byte[][] row; + kelondroRow.Entry row; //Runtime rt = Runtime.getRuntime(); while (i-- > 0) { // get out one entry row = dumpArray.get(i); - if ((row[0] == null) || (row[1] == null) || (row[2] == null) || (row[3] == null) || (row[4] == null)) continue; - wordHash = new String(row[0], "UTF-8"); + if (row == null) continue; + wordHash = row.getColString(0, "UTF-8"); //creationTime = kelondroRecords.bytes2long(row[2]); - wordEntry = new indexURLEntry(new String(row[3], "UTF-8"), new String(row[4], "UTF-8")); + wordEntry = new indexURLEntry(row.getColString(3, "UTF-8"), row.getColString(4, "UTF-8")); // store to cache addEntry(wordHash, wordEntry, startTime, false); urlCount++; diff --git a/source/de/anomic/kelondro/kelondroArray.java b/source/de/anomic/kelondro/kelondroArray.java index 37ac43676..1cca47340 100644 --- a/source/de/anomic/kelondro/kelondroArray.java +++ b/source/de/anomic/kelondro/kelondroArray.java @@ -68,13 +68,11 @@ public class kelondroArray extends kelondroRecords { } public kelondroArray(File file) throws IOException{ - // this opens a file with an existing array - super(file, 0); + // this opens a file with an existing array + super(file, 0); } - public synchronized byte[][] set(int index, byte[][] row) throws IOException { - if (row.length != columns()) - throw new IllegalArgumentException("set: wrong row length " + row.length + "; must be " + columns()); + public synchronized kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException { // make room for element Node n; @@ -82,20 +80,20 @@ public class kelondroArray extends kelondroRecords { n = newNode(); n.commit(CP_NONE); } - + // get the node at position index n = getNode(new Handle(index)); - + // write the row - byte[][] before = n.setValues(row); + byte[] before = n.setValueRow(rowentry.bytes()); n.commit(CP_NONE); - - return before; - } - public synchronized byte[][] get(int index) throws IOException { + return row().newEntry(before); + } + + public synchronized kelondroRow.Entry get(int index) throws IOException { if (index >= size()) throw new kelondroException(filename, "out of bounds, index=" + index + ", size=" + size()); - return getNode(new Handle(index)).getValues(); + return row().newEntry(getNode(new Handle(index)).getValueRow()); } public synchronized int seti(int index, int value) throws IOException { @@ -107,15 +105,12 @@ public class kelondroArray extends kelondroRecords { public synchronized int geti(int index) { return getHandle(index).hashCode(); } - - public synchronized int add(byte[][] row) throws IOException { - if (row.length != columns()) - throw new IllegalArgumentException("add: wrong row length " + row.length + "; must be " + columns()); + public synchronized int add(kelondroRow.Entry rowinstance) throws IOException { Node n = newNode(); n.commit(CP_LOW); int index = n.handle().hashCode(); - set(index, row); + set(index, rowinstance); return index; } @@ -125,11 +120,11 @@ public class kelondroArray extends kelondroRecords { public void print() throws IOException { System.out.println("PRINTOUT of table, length=" + size()); - byte[][] row; + kelondroRow.Entry row; for (int i = 0; i < size(); i++) { System.out.print("row " + i + ": "); row = get(i); - for (int j = 0; j < columns(); j++) System.out.print(((row[j] == null) ? "NULL" : new String(row[j], "UTF-8")) + ", "); + for (int j = 0; j < columns(); j++) System.out.print(((row.empty(j)) ? "NULL" : row.getColString(j, "UTF-8")) + ", "); System.out.println(); } System.out.println("EndOfTable"); @@ -162,22 +157,22 @@ public class kelondroArray extends kelondroRecords { if ((args.length == 3) && (args[0].equals("-g"))) { // get kelondroArray fm = new kelondroArray(new File(args[1])); - byte[][] row = fm.get(Integer.parseInt(args[2])); - for (int j = 0; j < fm.columns(); j++) System.out.print(new String(row[j]) + " "); + kelondroRow.Entry row = fm.get(Integer.parseInt(args[2])); + for (int j = 0; j < fm.columns(); j++) System.out.print(row.getColString(j, null) + " "); System.out.println(); fm.close(); } else if ((args.length == 4) && (args[0].equals("-s"))) { // set kelondroArray fm = new kelondroArray(new File(args[1])); - byte[][] row = new byte[][]{args[3].getBytes()}; + kelondroRow.Entry row = fm.row().newEntry(new byte[][]{args[3].getBytes()}); fm.set(Integer.parseInt(args[2]), row); fm.close(); } else if ((args.length == 3) && (args[0].equals("-a"))) { // add kelondroArray fm = new kelondroArray(new File(args[1])); - byte[][] row = new byte[][] { args[2].getBytes() }; + kelondroRow.Entry row = fm.row().newEntry(new byte[][] {args[2].getBytes()}); int index = fm.add(row); System.out.println("Added to row " + index); fm.close(); @@ -193,7 +188,7 @@ public class kelondroArray extends kelondroRecords { if (testfile.exists()) testfile.delete(); kelondroArray fm = new kelondroArray(testfile, new int[]{30, 50}, 9, true); for (int i = 0; i < 100; i++) { - fm.set(i, new byte[][]{("name" + i).getBytes(), ("value" + i).getBytes()}); + fm.set(i, fm.row().newEntry(new byte[][]{("name" + i).getBytes(), ("value" + i).getBytes()})); } fm.close(); } else diff --git a/source/de/anomic/kelondro/kelondroCollectionIndex.java b/source/de/anomic/kelondro/kelondroCollectionIndex.java index 85e711d95..11b5fe89c 100644 --- a/source/de/anomic/kelondro/kelondroCollectionIndex.java +++ b/source/de/anomic/kelondro/kelondroCollectionIndex.java @@ -132,7 +132,7 @@ public class kelondroCollectionIndex { int part = arrayIndex(collection.size()); // write a new entry in this array - int newRowNumber = array[part].add(newarrayrow); + int newRowNumber = array[part].add(array[part].row().newEntry(newarrayrow)); // store the new row number in the index index.put(new byte[][]{key, kelondroNaturalOrder.encodeLong(this.chunksize, 4), @@ -152,7 +152,7 @@ public class kelondroCollectionIndex { // see if we need new space or if we can overwrite the old space if (oldPartitionNumber == newPartitionNumber) { // we don't need a new slot, just write in the old one - array[oldPartitionNumber].set(rownumber, newarrayrow); + array[oldPartitionNumber].set(rownumber, array[oldPartitionNumber].row().newEntry(newarrayrow)); // update the index entry index.put(new byte[][]{key, kelondroNaturalOrder.encodeLong(this.chunksize, 4), @@ -164,7 +164,7 @@ public class kelondroCollectionIndex { // we need a new slot, that means we must first delete the old entry array[oldPartitionNumber].remove(rownumber); // write a new entry in the other array - int newRowNumber = array[newPartitionNumber].add(newarrayrow); + int newRowNumber = array[newPartitionNumber].add(array[newPartitionNumber].row().newEntry(newarrayrow)); // store the new row number in the index index.put(new byte[][]{key, kelondroNaturalOrder.encodeLong(this.chunksize, 4), @@ -186,12 +186,12 @@ public class kelondroCollectionIndex { int rownumber = (int) kelondroNaturalOrder.decodeLong(indexrow[3]); int partitionnumber = arrayIndex(chunkcount); // open array entry - byte[][] arrayrow = array[partitionnumber].get(rownumber); + kelondroRow.Entry arrayrow = array[partitionnumber].get(rownumber); if (arrayrow == null) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, partitionnumber).toString(), "array does not contain expected row"); // read the row and define a collection - int chunkcountInArray = (int) kelondroNaturalOrder.decodeLong(arrayrow[1]); + int chunkcountInArray = (int) arrayrow.getColLong(1); if (chunkcountInArray != chunkcount) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, partitionnumber).toString(), "array has different chunkcount than index: index = " + chunkcount + ", array = " + chunkcountInArray); - return new kelondroCollection(chunksize, chunkcount, new String(arrayrow[2]), arrayrow[3]); + return new kelondroCollection(chunksize, chunkcount, arrayrow.getColString(2, null), arrayrow.getColBytes(3)); } public void remove(byte[] key) throws IOException { diff --git a/source/de/anomic/kelondro/kelondroColumn.java b/source/de/anomic/kelondro/kelondroColumn.java index 586919494..72d82f6ef 100644 --- a/source/de/anomic/kelondro/kelondroColumn.java +++ b/source/de/anomic/kelondro/kelondroColumn.java @@ -31,17 +31,17 @@ public class kelondroColumn { public static final int celltype_undefined = 0; public static final int celltype_boolean = 1; - public static final int celltype_bytes = 2; + public static final int celltype_binary = 2; public static final int celltype_string = 3; public static final int celltype_cardinal = 4; public static final int celltype_real = 5; - private int celltype, dbwidth; + private int celltype, cellwidth; private String nickname, description; - public kelondroColumn(int celltype, int dbwidth, String nickname, String description) { + public kelondroColumn(int celltype, int cellwidth, String nickname, String description) { this.celltype = celltype; - this.dbwidth = dbwidth; + this.cellwidth = cellwidth; this.nickname = nickname; this.description = description; } @@ -50,8 +50,8 @@ public class kelondroColumn { return this.celltype; } - public int dbwidth() { - return this.dbwidth; + public int cellwidth() { + return this.cellwidth; } public String nickname() { diff --git a/source/de/anomic/kelondro/kelondroHashtable.java b/source/de/anomic/kelondro/kelondroHashtable.java index 62f60fb75..b9963f19e 100644 --- a/source/de/anomic/kelondro/kelondroHashtable.java +++ b/source/de/anomic/kelondro/kelondroHashtable.java @@ -138,7 +138,7 @@ public class kelondroHashtable { private int offset; private int maxk; private int maxrehash; - private byte[][] dummyRow; + private kelondroRow.Entry dummyRow; private static final byte[] dummyKey = kelondroBase64Order.enhancedCoder.encodeLong(0, 5).getBytes(); @@ -157,8 +157,8 @@ public class kelondroHashtable { this.maxk = kelondroMSetTools.log2a(maxsize); // equal to |log2(maxsize)| + 1 if (this.maxk >= kelondroMSetTools.log2a(maxsize + power2(offset + 1) + 1) - 1) this.maxk--; this.maxrehash = maxrehash; - dummyRow = new byte[hashArray.columns()][]; - dummyRow[0] = dummyKey; + dummyRow = this.hashArray.row().newEntry(); + dummyRow.setCol(0, dummyKey); for (int i = 0; i < hashArray.columns(); i++) try { hashArray.seti(0, this.offset); @@ -201,40 +201,43 @@ public class kelondroHashtable { return result; } - public synchronized byte[][] put(int key, byte[][] row) throws IOException { + public synchronized kelondroRow.Entry put(int key, kelondroRow.Entry rowentry) throws IOException { Hash hash = new Hash(key); + // find row Object[] search = search(hash); - byte[][] oldrow; + kelondroRow.Entry oldhkrow; int rowNumber = ((Integer) search[0]).intValue(); if (search[1] == null) { - oldrow = null; + oldhkrow = null; } else { - oldrow = (byte[][]) search[1]; + oldhkrow = (kelondroRow.Entry) search[1]; } + // make space while (rowNumber >= hashArray.size()) hashArray.set(hashArray.size(), dummyRow); + // write row - byte[][] newrow = new byte[hashArray.columns()][]; - newrow[0] = kelondroBase64Order.enhancedCoder.encodeLong(hash.key(), 5).getBytes(); - System.arraycopy(row, 0, newrow, 1, row.length); - hashArray.set(rowNumber, row); - return oldrow; + kelondroRow.Entry newhkrow = hashArray.row().newEntry(); + newhkrow.setCol(0, hash.key()); + newhkrow.setCol(1, rowentry.bytes()); + hashArray.set(rowNumber, newhkrow); + return hashArray.row().newEntry(oldhkrow.getColBytes(1)); } private Object[] search(Hash hash) throws IOException { - byte[][] row; + kelondroRow.Entry hkrow; int rowKey; int rowNumber; do { rowNumber = hash.node(); if (rowNumber >= hashArray.size()) return new Object[]{new Integer(rowNumber), null}; - row = hashArray.get(rowNumber); - rowKey = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(row[0], "UTF-8")); + hkrow = hashArray.get(rowNumber); + rowKey = (int) hkrow.getColLong(0); if (rowKey == 0) return new Object[]{new Integer(rowNumber), null}; hash.rehash(); } while (rowKey != hash.key()); - return new Object[]{new Integer(rowNumber), row}; + return new Object[]{new Integer(rowNumber), hkrow}; } diff --git a/source/de/anomic/kelondro/kelondroNaturalOrder.java b/source/de/anomic/kelondro/kelondroNaturalOrder.java index b8cdce5c2..a6437ad72 100644 --- a/source/de/anomic/kelondro/kelondroNaturalOrder.java +++ b/source/de/anomic/kelondro/kelondroNaturalOrder.java @@ -92,7 +92,7 @@ public class kelondroNaturalOrder extends kelondroAbstractOrder implements kelon return Long.MAX_VALUE - keyCardinal + zeroCardinal + 1; } - public static byte[] encodeLong(long c, int length) { + public final static byte[] encodeLong(long c, int length) { byte[] b = new byte[length]; while (length > 0) { b[--length] = (byte) (c & 0xFF); @@ -101,14 +101,31 @@ public class kelondroNaturalOrder extends kelondroAbstractOrder implements kelon return b; } - public static long decodeLong(byte[] s) { + public final static void encodeLong(long c, byte[] b, int offset, int length) { + assert offset + length <= b.length; + while (length > 0) { + b[--length + offset] = (byte) (c & 0xFF); + c >>= 8; + } + } + + public final static long decodeLong(byte[] s) { + if (s == null) return 0; long c = 0; int p = 0; - while ((p < 8) && (p < s.length)) c = (c << 8) | ((long) s[p++] & 0xFF); + while (p < s.length) c = (c << 8) | ((long) s[p++] & 0xFF); return c; } - - + + public final static long decodeLong(byte[] s, int offset, int length) { + if (s == null) return 0; + long c = 0; + int m = Math.min(s.length, offset + length); + while (offset < m) c = (c << 8) | ((long) s[offset++] & 0xFF); + return c; + } + + // Compares its two arguments for order. // Returns -1, 0, or 1 as the first argument // is less than, equal to, or greater than the second. diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java index f8d50d07a..161eae343 100644 --- a/source/de/anomic/kelondro/kelondroRecords.java +++ b/source/de/anomic/kelondro/kelondroRecords.java @@ -127,19 +127,13 @@ public class kelondroRecords { private int headchunksize;// overheadsize + key element column size private int tailchunksize;// sum(all: COLWIDTHS) minus the size of the key element colum private int recordsize; // (overhead + sum(all: COLWIDTHS)) = the overall size of a record - protected int objectsize; // sum(all: COLWIDTHS)) = the size of all data fields - + // dynamic run-time seek pointers private long POS_HANDLES = 0; // starts after end of POS_COLWIDHS which is POS_COLWIDTHS + COLWIDTHS.length * 4 private long POS_TXTPROPS = 0; // starts after end of POS_HANDLES which is POS_HANDLES + HANDLES.length * 4 private long POS_NODES = 0; // starts after end of POS_TXTPROPS which is POS_TXTPROPS + TXTPROPS.length * TXTPROPW // dynamic variables that are back-ups of stored values in file; read/defined on instantiation - /* - private int USEDC; // counter of used elements - private int FREEC; // counter of free elements in list of free Nodes - private Handle FREEH; // pointer to first element in list of free Nodes, empty = NUL - */ private usageControl USAGE; // counter for used and re-use records and pointer to free-list private short OHBYTEC; // number of extra bytes in each node private short OHHANDLEC; // number of handles in each node @@ -247,11 +241,12 @@ public class kelondroRecords { this.entryFile = new kelondroRAIOChunks(ra, ra.name()); } + // create row + ROW = new kelondroRow(columns); + // store dynamic run-time data this.overhead = ohbytec + 4 * ohhandlec; - this.objectsize = 0; - for (int i = 0; i < columns.length; i++) this.objectsize += columns[i]; - this.recordsize = this.overhead + this.objectsize; + this.recordsize = this.overhead + ROW.size(); this.headchunksize = overhead + columns[0]; this.tailchunksize = this.recordsize - this.headchunksize; @@ -264,7 +259,6 @@ public class kelondroRecords { USAGE = new usageControl(0, 0, new Handle(NUL)); OHBYTEC = ohbytec; OHHANDLEC = ohhandlec; - ROW = new kelondroRow(columns); HANDLES = new Handle[FHandles]; for (int i = 0; i < FHandles; i++) HANDLES[i] = new Handle(NUL); TXTPROPS = new byte[txtProps][]; @@ -343,7 +337,7 @@ public class kelondroRecords { else this.theLogger.fine("KELONDRO DEBUG for file " + this.filename + ": " + message); } - + public void clear() throws IOException { // Removes all mappings from this map // throw new UnsupportedOperationException("clear not supported"); @@ -416,9 +410,7 @@ public class kelondroRecords { // assign remaining values that are only present at run-time this.overhead = OHBYTEC + 4 * OHHANDLEC; this.recordsize = this.overhead; - this.objectsize = 0; - for (int i = 0; i < this.ROW.columns(); i++) this.objectsize += this.ROW.width(i); - this.recordsize = this.overhead + this.objectsize; + this.recordsize = this.overhead + ROW.size(); this.headchunksize = this.overhead + this.ROW.width(0); this.tailchunksize = this.recordsize - this.headchunksize; } @@ -738,9 +730,9 @@ public class kelondroRecords { return (h == NUL) ? null : new Handle(h); } - public byte[][] setValues(byte[][] row) throws IOException { + public byte[][] setValueCells(byte[][] row) throws IOException { // if the index is defined, then write values directly to the file, else only to the object - byte[][] result = getValues(); // previous value (this loads the values if not already happened) + byte[][] result = getValueCells(); // previous value (this loads the values if not already happened) // set values if (this.handle.index != NUL) { @@ -756,12 +748,27 @@ public class kelondroRecords { return result; // return previous value } + public byte[] setValueRow(byte[] row) throws IOException { + // if the index is defined, then write values directly to the file, else only to the object + assert row.length == ROW.size(); + byte[] result = getValueRow(); // previous value (this loads the values if not already happened) + + // set values + if (this.handle.index != NUL) { + setValue(row, ROW.width(0), headChunk, overhead); + setValue(row, ROW.width(1), tailChunk, 0); + } + this.headChanged = true; + this.tailChanged = true; + return result; // return previous value + } + public byte[] getKey() { // read key return trimCopy(headChunk, overhead, ROW.width(0)); } - public byte[][] getValues() throws IOException { + public byte[][] getValueCells() throws IOException { if (this.tailChunk == null) { // load all values from the database file @@ -786,6 +793,27 @@ public class kelondroRecords { return values; } + public byte[] getValueRow() throws IOException { + + if (this.tailChunk == null) { + // load all values from the database file + this.tailChunk = new byte[tailchunksize]; + // read values + entryFile.readFully(seekpos(this.handle) + headchunksize, this.tailChunk, 0, this.tailChunk.length); + } + + // create return value + byte[] row = new byte[ROW.size()]; + + // read key + System.arraycopy(headChunk, overhead, row, 0, ROW.width(0)); + + // read remaining values + System.arraycopy(tailChunk, 0, row, ROW.width(0), tailchunksize); + + return row; + } + public synchronized void commit(int cachePriority) throws IOException { // this must be called after all write operations to the node are // finished @@ -820,12 +848,9 @@ public class kelondroRecords { } private byte[] trimCopy(byte[] a, int offset, int length) { - if (length > a.length - offset) - length = a.length - offset; - while ((length > 0) && (a[offset + length - 1] == 0)) - length--; - if (length == 0) - return null; + if (length > a.length - offset) length = a.length - offset; + while ((length > 0) && (a[offset + length - 1] == 0)) length--; + if (length == 0) return null; byte[] b = new byte[length]; System.arraycopy(a, offset, b, 0, length); return b; @@ -842,7 +867,7 @@ public class kelondroRecords { h = getOHHandle(i); if (h == null) s = s + ":hNULL"; else s = s + ":h" + h.toString(); } - byte[][] content = getValues(); + byte[][] content = getValueCells(); for (int i = 0; i < content.length; i++) s = s + ":" + ((content[i] == null) ? "NULL" : (new String(content[i], "UTF-8")).trim()); } catch (IOException e) { s = s + ":***LOAD ERROR***:" + e.getMessage(); @@ -1140,7 +1165,7 @@ public class kelondroRecords { Node n = new Node(pos); pos.index++; while ((markedDeleted.contains(pos)) && (pos.index < USAGE.allCount())) pos.index++; - return n.getValues(); + return n.getValueCells(); } catch (IOException e) { throw new kelondroException(filename, e.getMessage()); } @@ -1185,22 +1210,6 @@ public class kelondroRecords { return true; } - public static byte[] long2bytes(long x, int length) { - byte[] b = new byte[length]; - for (int i = length - 1; i >= 0; i--) { - b[i] = (byte) (x & 0XFF); - x >>= 8; - } - return b; - } - - public static long bytes2long(byte[] b) { - if (b == null) return 0; - long x = 0; - for (int i = 0; i < b.length; i++) x = (x << 8) | (0xff & b[i]); - return x; - } - public final static void NUL2bytes(byte[] b, int offset) { b[offset ] = (byte) (0XFF & (NUL >> 24)); b[offset + 1] = (byte) (0XFF & (NUL >> 16)); diff --git a/source/de/anomic/kelondro/kelondroRow.java b/source/de/anomic/kelondro/kelondroRow.java index 70ef11afe..e9b11749a 100644 --- a/source/de/anomic/kelondro/kelondroRow.java +++ b/source/de/anomic/kelondro/kelondroRow.java @@ -27,6 +27,7 @@ package de.anomic.kelondro; +import java.io.UnsupportedEncodingException; import java.util.HashMap; public class kelondroRow { @@ -38,18 +39,32 @@ public class kelondroRow { private kelondroColumn[] row; + private int[] colstart; private HashMap encodedFormConfiguration; private int encodedFormLength; + private int objectsize; public kelondroRow(kelondroColumn[] row) { this.row = row; + this.colstart = new int[row.length]; + this.objectsize = 0; + for (int i = 0; i < row.length; i++) { + this.colstart[i] = this.objectsize; + this.objectsize += row[i].cellwidth(); + } this.encodedFormConfiguration = null; this.encodedFormLength = -1; } public kelondroRow(int[] row) { this.row = new kelondroColumn[row.length]; - for (int i = 0; i < row.length; i++) this.row[i] = new kelondroColumn(kelondroColumn.celltype_undefined, row[i], "col_" + i, ""); + this.colstart = new int[row.length]; + this.objectsize = 0; + for (int i = 0; i < row.length; i++) { + this.row[i] = new kelondroColumn(kelondroColumn.celltype_undefined, row[i], "col_" + i, ""); + this.colstart[i] = this.objectsize; + this.objectsize += row[i]; + } this.encodedFormConfiguration = null; this.encodedFormLength = -1; } @@ -58,13 +73,17 @@ public class kelondroRow { return this.row.length; } + public int size() { + return this.objectsize; + } + public int width(int row) { - return this.row[row].dbwidth(); + return this.row[row].cellwidth(); } public int[] widths() { int[] w = new int[this.row.length]; - for (int i = 0; i < this.row.length; i++) w[i] = row[i].dbwidth(); + for (int i = 0; i < this.row.length; i++) w[i] = row[i].cellwidth(); return w; } @@ -89,12 +108,94 @@ public class kelondroRow { } } + public Entry newEntry() { + return new Entry(); + } + + public Entry newEntry(byte[] rowinstance) { + return new Entry(rowinstance); + } + + public Entry newEntry(byte[][] cells) { + return new Entry(cells); + } + public class Entry { - private byte[][] cols; + private byte[] rowinstance; + + public Entry() { + rowinstance = new byte[objectsize]; + for (int i = 0; i < objectsize; i++) this.rowinstance[i] = 0; + } + + public Entry(byte[] rowinstance) { + if (rowinstance.length == objectsize) { + this.rowinstance = rowinstance; + } else { + this.rowinstance = new byte[objectsize]; + System.arraycopy(rowinstance, 0, this.rowinstance, 0, rowinstance.length); + for (int i = rowinstance.length; i < objectsize; i++) this.rowinstance[i] = 0; + } + } public Entry(byte[][] cols) { - this.cols = cols; + rowinstance = new byte[objectsize]; + for (int i = 0; i < objectsize; i++) this.rowinstance[i] = 0; + for (int i = 0; i < cols.length; i++) { + System.arraycopy(cols[i], 0, rowinstance, colstart[i], row[i].cellwidth()); + } + } + + public byte[] bytes() { + return rowinstance; + } + + public boolean empty(int column) { + return rowinstance[colstart[column]] == 0; + } + + public void setCol(int column, byte[] cell) { + int valuewidth = row[column].cellwidth(); + int targetoffset = colstart[column]; + if (cell == null) { + while (valuewidth-- > 0) rowinstance[targetoffset + valuewidth] = 0; + } else { + System.arraycopy(cell, 0, rowinstance, targetoffset, Math.min(cell.length, valuewidth)); // error? + if (cell.length < valuewidth) { + while (valuewidth-- > cell.length) rowinstance[targetoffset + valuewidth] = 0; + } + } + } + + public void setCol(int column, long cell) { + kelondroNaturalOrder.encodeLong(cell, rowinstance, colstart[column], row[column].cellwidth()); + } + + public String getColString(int column, String encoding) { + int length = row[column].cellwidth(); + int offset = colstart[column]; + if (length > rowinstance.length - offset) length = rowinstance.length - offset; + while ((length > 0) && (rowinstance[offset + length - 1] == 0)) length--; + if (length == 0) return null; + try { + if ((encoding == null) || (encoding.length() == 0)) + return new String (rowinstance, offset, length); + else + return new String(rowinstance, offset, length, encoding); + } catch (UnsupportedEncodingException e) { + return ""; + } + } + + public long getColLong(int column) { + return kelondroNaturalOrder.decodeLong(rowinstance, colstart[column], row[column].cellwidth()); + } + + public byte[] getColBytes(int column) { + byte[] c = new byte[row[column].cellwidth()]; + System.arraycopy(rowinstance, colstart[column], c, 0, row[column].cellwidth()); + return c; } public byte[] toEncodedBytesForm() { @@ -111,17 +212,17 @@ public class kelondroRow { throw new kelondroException("ROW", "toEncodedForm of celltype undefined not possible"); case kelondroColumn.celltype_boolean: throw new kelondroException("ROW", "toEncodedForm of celltype boolean not yet implemented"); - case kelondroColumn.celltype_bytes: - System.arraycopy(cols[i], 0, b, p, length); + case kelondroColumn.celltype_binary: + System.arraycopy(rowinstance, colstart[i], b, p, length); p += length; continue; case kelondroColumn.celltype_string: - System.arraycopy(cols[i], 0, b, p, length); + System.arraycopy(rowinstance, colstart[i], b, p, length); p += length; continue; case kelondroColumn.celltype_cardinal: if (encoder == encoder_b64e) { - long c = kelondroRecords.bytes2long(cols[i]); + long c = bytes2long(rowinstance, colstart[i]); System.arraycopy(kelondroBase64Order.enhancedCoder.encodeLongSmart(c, length).getBytes(), 0, b, p, length); p += length; continue; @@ -134,4 +235,19 @@ public class kelondroRow { return b; } } + + public final static void long2bytes(long x, byte[] b, int offset, int length) { + for (int i = length - 1; i >= 0; i--) { + b[offset + i] = (byte) (x & 0XFF); + x >>= 8; + } + } + + public final static long bytes2long(byte[] b, int offset) { + if (b == null) return 0; + long x = 0; + for (int i = 0; i < b.length; i++) x = (x << 8) | (0xff & b[offset + i]); + return x; + } + } diff --git a/source/de/anomic/kelondro/kelondroStack.java b/source/de/anomic/kelondro/kelondroStack.java index 65422fa09..b2814363d 100644 --- a/source/de/anomic/kelondro/kelondroStack.java +++ b/source/de/anomic/kelondro/kelondroStack.java @@ -139,7 +139,7 @@ public final class kelondroStack extends kelondroRecords { if (getHandle(root) != null) throw new RuntimeException("push: internal organisation of root and toor"); // create node Node n = newNode(); - n.setValues(row); + n.setValueCells(row); n.setOHHandle(left, null); n.setOHHandle(right, null); n.commit(CP_NONE); @@ -150,7 +150,7 @@ public final class kelondroStack extends kelondroRecords { } else { // expand the list at the end Node n = newNode(); - n.setValues(row); + n.setValueCells(row); n.setOHHandle(left, getHandle(toor)); n.setOHHandle(right, null); Node n1 = getNode(getHandle(toor), null, 0); @@ -172,7 +172,7 @@ public final class kelondroStack extends kelondroRecords { // return row relative to top of the stack and remove addressed element Node n = topNode(dist); if (n == null) return null; - byte[][] ret = n.getValues(); + byte[][] ret = n.getValueCells(); // remove node unlinkNode(n); @@ -191,7 +191,7 @@ public final class kelondroStack extends kelondroRecords { // with dist == 0 this is the same function as with top() Node n = topNode(dist); if (n == null) return null; - return n.getValues(); + return n.getValueCells(); } public synchronized byte[][] pot() throws IOException { @@ -203,7 +203,7 @@ public final class kelondroStack extends kelondroRecords { // return row relative to the bottom of the stack and remove addressed element Node n = botNode(dist); if (n == null) return null; - byte[][] ret = n.getValues(); + byte[][] ret = n.getValueCells(); // remove node unlinkNode(n); @@ -222,7 +222,7 @@ public final class kelondroStack extends kelondroRecords { // with dist == 0 this is the same function as with bot() Node n = botNode(dist); if (n == null) return null; - return n.getValues(); + return n.getValueCells(); } public synchronized ArrayList botList(int dist) throws IOException { @@ -344,9 +344,9 @@ public final class kelondroStack extends kelondroRecords { //n = getNode(h, null, 0); System.out.println("> NODE " + hp(n.handle()) + "; left " + hp(n.getOHHandle(left)) + ", right " + hp(n.getOHHandle(right))); - System.out.print(" KEY:'" + (new String(n.getValues()[0])).trim() + "'"); + System.out.print(" KEY:'" + (new String(n.getValueCells()[0])).trim() + "'"); for (int j = 1; j < columns(); j++) - System.out.print(", V[" + j + "]:'" + (new String(n.getValues()[j])).trim() + "'"); + System.out.print(", V[" + j + "]:'" + (new String(n.getValueCells()[j])).trim() + "'"); System.out.println(); } System.out.println(); diff --git a/source/de/anomic/kelondro/kelondroTables.java b/source/de/anomic/kelondro/kelondroTables.java index 3d9b05bcb..091214d9f 100644 --- a/source/de/anomic/kelondro/kelondroTables.java +++ b/source/de/anomic/kelondro/kelondroTables.java @@ -119,13 +119,6 @@ public class kelondroTables { tTables.put(tablename, tree); } - public synchronized void update(String tablename, String key, long[] row) throws IOException { - kelondroTree tree = (kelondroTree) tTables.get(tablename); - if (tree == null) throw new RuntimeException("kelondroTables.update: tree table '" + tablename + "' does not exist."); - tree.putLong(key.getBytes(), row); - tTables.put(tablename, tree); - } - public synchronized Map selectMap(String tablename, String key) throws IOException { kelondroMap table = (kelondroMap) mTables.get(tablename); if (table == null) throw new RuntimeException("kelondroTables.selectMap: map table '" + tablename + "' does not exist."); @@ -139,12 +132,6 @@ public class kelondroTables { return tree.get(key.getBytes()); } - public synchronized long[] selectLong(String tablename, String key) throws IOException { - kelondroTree tree = (kelondroTree) tTables.get(tablename); - if (tree == null) throw new RuntimeException("kelondroTables.selectLong: tree table '" + tablename + "' does not exist."); - return tree.getLong(key.getBytes()); - } - public synchronized kelondroMap.mapIterator /* of Map-Elements */ maps(String tablename, boolean up, boolean rotating) throws IOException { kelondroMap table = (kelondroMap) mTables.get(tablename); if (table == null) throw new RuntimeException("kelondroTables.maps: map table '" + tablename + "' does not exist."); diff --git a/source/de/anomic/kelondro/kelondroTree.java b/source/de/anomic/kelondro/kelondroTree.java index 45b92e9f7..562b4f5a9 100644 --- a/source/de/anomic/kelondro/kelondroTree.java +++ b/source/de/anomic/kelondro/kelondroTree.java @@ -171,7 +171,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { } public final int cacheObjectChunkSize() { - return super.objectsize + 8 * super.columns(); + return row().size() + 8 * super.columns(); } public String[] cacheObjectStatus() { @@ -232,7 +232,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { synchronized (writeSearchObj) { writeSearchObj.process(key); if (writeSearchObj.found()) { - result = writeSearchObj.getMatcher().getValues(); + result = writeSearchObj.getMatcher().getValueCells(); if (objectCache != null) objectCache.put(key, result); } else { result = null; @@ -241,22 +241,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { } return result; } - - public long[] getLong(byte[] key) throws IOException { - byte[][] row = get(key); - long[] longs = new long[columns() - 1]; - if (row == null) { - for (int i = 0; i < columns() - 1; i++) { - longs[i] = 0; - } - } else { - for (int i = 0; i < columns() - 1; i++) { - longs[i] = bytes2long(row[i + 1]); - } - } - return longs; - } - + public class Search { // a search object combines the results of a search in the tree, which are @@ -386,26 +371,6 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { return (lc.equals(childn.handle())); } - public long[] putLong(byte[] key, long[] newlongs) throws IOException { - byte[][] newrow = new byte[newlongs.length + 1][]; - newrow[0] = key; - for (int i = 0; i < newlongs.length; i++) { - newrow[i + 1] = long2bytes(newlongs[i], columnSize(i + 1)); - } - byte[][] oldrow = put(newrow); - long[] oldlongs = new long[columns() - 1]; - if (oldrow == null) { - for (int i = 0; i < columns() - 1; i++) { - oldlongs[i] = 0; - } - } else { - for (int i = 0; i < columns() - 1; i++) { - oldlongs[i] = bytes2long(oldrow[i + 1]); - } - } - return oldlongs; - } - // Associates the specified value with the specified key in this map public byte[][] put(byte[][] newrow) throws IOException { byte[][] result = null; @@ -418,7 +383,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { if (writeSearchObj.found()) { // a node with this key exist. simply overwrite the content and return old content Node e = writeSearchObj.getMatcher(); - result = e.setValues(newrow); + result = e.setValueCells(newrow); commitNode(e); } else if (writeSearchObj.isRoot()) { // a node with this key does not exist and there is no node at all @@ -427,7 +392,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { throw new kelondroException(filename, "tried to create root node twice"); // we dont have any Nodes in the file, so start here to create one Node e = newNode(); - e.setValues(newrow); + e.setValueCells(newrow); // write the propetries e.setOHByte(magic, (byte) 1); e.setOHByte(balance, (byte) 0); @@ -449,7 +414,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { // create new node and assign values Node parentNode = writeSearchObj.getParent(); Node theNode = newNode(); - theNode.setValues(newrow); + theNode.setValueCells(newrow); theNode.setOHByte(0, (byte) 1); // fresh magic theNode.setOHByte(1, (byte) 0); // fresh balance theNode.setOHHandle(parent, parentNode.handle()); @@ -682,7 +647,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { writeSearchObj.process(key); if (writeSearchObj.found()) { Node result = writeSearchObj.getMatcher(); - byte[][] values = result.getValues(); + byte[][] values = result.getValueCells(); remove(result, writeSearchObj.getParent()); return values; } else { @@ -1041,7 +1006,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { Iterator i = (firstKey == null) ? new nodeIterator(up, rotating) : new nodeIterator(up, rotating, firstKey, including); while ((rows.size() < count) && (i.hasNext())) { n = (Node) i.next(); - if (n != null) rows.put(new String(n.getKey()), n.getValues()); + if (n != null) rows.put(new String(n.getKey()), n.getValueCells()); } } return rows; diff --git a/source/de/anomic/plasma/plasmaWordIndexAssortment.java b/source/de/anomic/plasma/plasmaWordIndexAssortment.java index 13571b1ad..7be4d2b20 100644 --- a/source/de/anomic/plasma/plasmaWordIndexAssortment.java +++ b/source/de/anomic/plasma/plasmaWordIndexAssortment.java @@ -61,7 +61,7 @@ import de.anomic.index.indexEntryAttribute; import de.anomic.index.indexTreeMapContainer; import de.anomic.index.indexURLEntry; import de.anomic.kelondro.kelondroException; -import de.anomic.kelondro.kelondroRecords; +import de.anomic.kelondro.kelondroNaturalOrder; import de.anomic.kelondro.kelondroTree; import de.anomic.server.logging.serverLog; @@ -135,8 +135,8 @@ public final class plasmaWordIndexAssortment { if (newContainer.size() != assortmentLength) throw new RuntimeException("plasmaWordIndexAssortment.store: wrong container size"); byte[][] row = new byte[this.bufferStructureLength][]; row[0] = newContainer.wordHash().getBytes(); - row[1] = kelondroRecords.long2bytes(1, 4); - row[2] = kelondroRecords.long2bytes(newContainer.updated(), 8); + row[1] = kelondroNaturalOrder.encodeLong(1, 4); + row[2] = kelondroNaturalOrder.encodeLong(newContainer.updated(), 8); Iterator entries = newContainer.entries(); indexURLEntry entry; for (int i = 0; i < assortmentLength; i++) { @@ -216,7 +216,7 @@ public final class plasmaWordIndexAssortment { public indexContainer row2container(String wordHash, byte[][] row) { if (row == null) return null; - final long updateTime = kelondroRecords.bytes2long(row[2]); + final long updateTime = kelondroNaturalOrder.decodeLong(row[2]); indexTreeMapContainer container = new indexTreeMapContainer(wordHash); for (int i = 0; i < assortmentLength; i++) { container.add(