From 4ff742e42de448da5449b36776ca91bf74635da7 Mon Sep 17 00:00:00 2001 From: orbiter Date: Sat, 5 Aug 2006 19:18:33 +0000 Subject: [PATCH] implemented indexCollectionRI this is the new database structure that is supposed to replace the plasmaAssortmentCluster AND the plasmaWordIndexFileCluster The new structure is not yet active and needs to be integrated into plasmaWordIndex. This has some migration constraints that are not yet completely solved. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2347 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/dbtest.java | 2 +- source/de/anomic/index/indexCollectionRI.java | 71 ++++--- .../de/anomic/index/indexRowSetContainer.java | 5 + .../anomic/kelondro/kelondroBytesIntMap.java | 8 + .../kelondro/kelondroCollectionIndex.java | 177 +++++++++++++----- source/de/anomic/kelondro/kelondroColumn.java | 2 +- .../de/anomic/kelondro/kelondroFlexTable.java | 40 +++- source/de/anomic/kelondro/kelondroIndex.java | 3 +- .../kelondro/kelondroRowCollection.java | 12 +- source/de/anomic/kelondro/kelondroRowSet.java | 6 + .../anomic/kelondro/kelondroSplittedTree.java | 10 +- source/de/anomic/plasma/plasmaWordIndex.java | 2 +- 12 files changed, 254 insertions(+), 84 deletions(-) diff --git a/source/dbtest.java b/source/dbtest.java index fff7a7a5b..1197d1154 100644 --- a/source/dbtest.java +++ b/source/dbtest.java @@ -290,7 +290,7 @@ public class dbtest { if (command.equals("list")) { Iterator i = null; - if (table instanceof kelondroSplittedTree) i = ((kelondroSplittedTree) table).rows(true, false); + if (table instanceof kelondroSplittedTree) i = ((kelondroSplittedTree) table).rows(true, false, null); if (table instanceof kelondroTree) i = ((kelondroTree) table).rows(true, false, null); if (table instanceof dbTable) i = ((dbTable) table).rows(true, false, null); byte[][] row; diff --git a/source/de/anomic/index/indexCollectionRI.java b/source/de/anomic/index/indexCollectionRI.java index bc67a5322..4616997c4 100644 --- a/source/de/anomic/index/indexCollectionRI.java +++ b/source/de/anomic/index/indexCollectionRI.java @@ -28,12 +28,16 @@ package de.anomic.index; import java.io.File; import java.io.IOException; +import java.util.HashSet; import java.util.Iterator; import java.util.Set; import de.anomic.kelondro.kelondroCollectionIndex; import de.anomic.kelondro.kelondroNaturalOrder; +import de.anomic.kelondro.kelondroOutOfLimitsException; import de.anomic.kelondro.kelondroRow; +import de.anomic.kelondro.kelondroRowCollection; +import de.anomic.kelondro.kelondroRowSet; public class indexCollectionRI extends indexAbstractRI implements indexRI { @@ -63,68 +67,87 @@ public class indexCollectionRI extends indexAbstractRI implements indexRI { public class wordContainersIterator implements Iterator { - //private Iterator wci; + private Iterator wci; public wordContainersIterator(String startWordHash, boolean rot) { - + wci = collectionIndex.keycollections(startWordHash.getBytes(), rot); } public boolean hasNext() { - // TODO Auto-generated method stub - return false; + return wci.hasNext(); } public Object next() { - // TODO Auto-generated method stub - return null; + Object[] oo = (Object[]) wci.next(); + byte[] key = (byte[]) oo[0]; + kelondroRowSet collection = (kelondroRowSet) oo[1]; + if (collection == null) return null; + return new indexRowSetContainer(new String(key), collection); } public void remove() { - // TODO Auto-generated method stub - + wci.remove(); } } public indexContainer getContainer(String wordHash, boolean deleteIfEmpty, long maxtime) { try { - indexRowSetContainer idx = (indexRowSetContainer) collectionIndex.get(wordHash.getBytes()); - idx.setWordHash(wordHash); - return idx; + kelondroRowSet collection = collectionIndex.get(wordHash.getBytes(), deleteIfEmpty); + if (collection == null) return null; + return new indexRowSetContainer(wordHash, collection); } catch (IOException e) { - e.printStackTrace(); return null; } } public indexContainer deleteContainer(String wordHash) { - indexContainer idx = getContainer(wordHash, true, -1); try { - collectionIndex.remove(wordHash.getBytes()); + kelondroRowSet collection = collectionIndex.delete(wordHash.getBytes()); + if (collection == null) return null; + return new indexRowSetContainer(wordHash, collection); } catch (IOException e) { - e.printStackTrace(); + return null; } - return idx; } public boolean removeEntry(String wordHash, String urlHash, boolean deleteComplete) { - // TODO Auto-generated method stub - return false; + HashSet hs = new HashSet(); + hs.add(urlHash.getBytes()); + return removeEntries(wordHash, hs, deleteComplete) == 1; } public int removeEntries(String wordHash, Set urlHashes, boolean deleteComplete) { - // TODO Auto-generated method stub - return 0; + try { + return collectionIndex.remove(wordHash.getBytes(), urlHashes, deleteComplete); + } catch (kelondroOutOfLimitsException e) { + e.printStackTrace(); + return 0; + } catch (IOException e) { + e.printStackTrace(); + return 0; + } } public indexContainer addEntries(indexContainer newEntries, long creationTime, boolean dhtCase) { - // TODO Auto-generated method stub - return null; + String wordHash = newEntries.getWordHash(); + try { + collectionIndex.merge(wordHash.getBytes(), (kelondroRowCollection) newEntries); + return getContainer(wordHash, true, -1); // FIXME: this is not optimal + } catch (kelondroOutOfLimitsException e) { + e.printStackTrace(); + return null; + } catch (IOException e) { + return null; + } } public void close(int waitingSeconds) { - // TODO Auto-generated method stub - + try { + collectionIndex.close(); + } catch (IOException e) { + e.printStackTrace(); + } } } diff --git a/source/de/anomic/index/indexRowSetContainer.java b/source/de/anomic/index/indexRowSetContainer.java index 39b38608d..71a8c96be 100644 --- a/source/de/anomic/index/indexRowSetContainer.java +++ b/source/de/anomic/index/indexRowSetContainer.java @@ -46,6 +46,11 @@ public class indexRowSetContainer extends kelondroRowSet implements indexContain this(wordHash, new kelondroNaturalOrder(true), 0); } + public indexRowSetContainer(String wordHash, kelondroRowSet collection) { + super(collection); + this.wordHash = wordHash; + } + public indexRowSetContainer(String wordHash, kelondroOrder ordering, int column) { super(indexURLEntry.urlEntryRow); this.wordHash = wordHash; diff --git a/source/de/anomic/kelondro/kelondroBytesIntMap.java b/source/de/anomic/kelondro/kelondroBytesIntMap.java index c68cb941c..72c3e42b2 100644 --- a/source/de/anomic/kelondro/kelondroBytesIntMap.java +++ b/source/de/anomic/kelondro/kelondroBytesIntMap.java @@ -25,6 +25,7 @@ package de.anomic.kelondro; import java.io.IOException; +import java.util.Iterator; public class kelondroBytesIntMap { @@ -62,4 +63,11 @@ public class kelondroBytesIntMap { return ki.size(); } + public Iterator rows(boolean up, boolean rotating, byte[] firstKey) throws IOException { + // returns the row-iterator of the underlying kelondroIndex + // col[0] = key + // col[1] = integer as {b265} + return ki.rows(up, rotating, firstKey); + } + } diff --git a/source/de/anomic/kelondro/kelondroCollectionIndex.java b/source/de/anomic/kelondro/kelondroCollectionIndex.java index 735e6dbfe..27c7c65e3 100644 --- a/source/de/anomic/kelondro/kelondroCollectionIndex.java +++ b/source/de/anomic/kelondro/kelondroCollectionIndex.java @@ -30,6 +30,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.Iterator; import java.util.Map; +import java.util.Set; public class kelondroCollectionIndex { @@ -45,7 +46,8 @@ public class kelondroCollectionIndex { private static final int idx_col_chunksize = 1; // chunksize (number of bytes in a single chunk, needed for migration option) private static final int idx_col_chunkcount = 2; // chunkcount (number of chunks in this collection) needed to identify array file that has the chunks private static final int idx_col_indexpos = 3; // indexpos (position in index file) - private static final int idx_col_update = 4; // a time stamp, update time in days since 1.1.2000 + private static final int idx_col_lastread = 4; // a time stamp, update time in days since 1.1.2000 + private static final int idx_col_lastwrote = 5; // a time stamp, update time in days since 1.1.2000 private static kelondroRow indexRow(int keylen) { return new kelondroRow( @@ -53,11 +55,12 @@ public class kelondroCollectionIndex { "int chunksize-4 {b256}," + "int chunkcount-4 {b256}," + "int indexpos-4 {b256}," + - "short update-2 {b256}" + "short lastread-2 {b256}" + + "short lastwrote-2 {b256}" ); } - private static File arrayFile(File path, String filenameStub, int loadfactor, int chunksize, int partitionNumber) { + private static File arrayFile(File path, String filenameStub, int loadfactor, int chunksize, int partitionNumber, int serialNumber) { String lf = Integer.toHexString(loadfactor).toUpperCase(); while (lf.length() < 2) lf = "0" + lf; @@ -65,7 +68,9 @@ public class kelondroCollectionIndex { while (cs.length() < 4) cs = "0" + cs; String pn = Integer.toHexString(partitionNumber).toUpperCase(); while (pn.length() < 2) pn = "0" + pn; - return new File(path, filenameStub + "." + lf + "." + cs + "." + pn + ".kca"); // kelondro collection array + String sn = Integer.toHexString(serialNumber).toUpperCase(); + while (sn.length() < 2) sn = "0" + sn; + return new File(path, filenameStub + "." + lf + "." + cs + "." + pn + "." + sn + ".kca"); // kelondro collection array } public kelondroCollectionIndex(File path, String filenameStub, int keyLength, kelondroOrder indexOrder, @@ -84,8 +89,8 @@ public class kelondroCollectionIndex { this.arrays = new HashMap(); // all entries will be dynamically created with getArray() } - private kelondroFixedWidthArray openArrayFile(int partitionNumber, boolean create) throws IOException { - File f = arrayFile(path, filenameStub, loadfactor, rowdef.objectsize(), partitionNumber); + private kelondroFixedWidthArray openArrayFile(int partitionNumber, int serialNumber, boolean create) throws IOException { + File f = arrayFile(path, filenameStub, loadfactor, rowdef.objectsize(), partitionNumber, serialNumber); if (f.exists()) { return new kelondroFixedWidthArray(f); @@ -101,12 +106,12 @@ public class kelondroCollectionIndex { } } - private kelondroFixedWidthArray getArray(int partitionNumber, int chunksize) { + private kelondroFixedWidthArray getArray(int partitionNumber, int serialNumber, int chunksize) { String accessKey = partitionNumber + "-" + chunksize; kelondroFixedWidthArray array = (kelondroFixedWidthArray) arrays.get(accessKey); if (array != null) return array; try { - array = openArrayFile(partitionNumber, true); + array = openArrayFile(partitionNumber, serialNumber, true); } catch (IOException e) { return null; } @@ -138,28 +143,35 @@ public class kelondroCollectionIndex { public void put(byte[] key, kelondroRowCollection collection) throws IOException, kelondroOutOfLimitsException { // this replaces an old collection by a new one // this method is not approriate to extend an existing collection with another collection - insert(key, collection, false); + putmergeremove(key, collection, false, null, false); } - public void join(byte[] key, kelondroRowCollection collection) throws IOException, kelondroOutOfLimitsException { - insert(key, collection, true); + public void merge(byte[] key, kelondroRowCollection collection) throws IOException, kelondroOutOfLimitsException { + putmergeremove(key, collection, true, null, false); } - private void insert(byte[] key, kelondroRowCollection collection, boolean join) throws IOException, kelondroOutOfLimitsException { + public int remove(byte[] key, Set removekeys, boolean deletecomplete) throws IOException, kelondroOutOfLimitsException { + return putmergeremove(key, null, false, removekeys, deletecomplete); + } + + private int putmergeremove(byte[] key, kelondroRowCollection collection, boolean merge, Set removekeys, boolean deletecomplete) throws IOException, kelondroOutOfLimitsException { //if (collection.size() > maxChunks) throw new kelondroOutOfLimitsException(maxChunks, collection.size()); - if (collection.size() == 0) { + if ((!merge) && (collection.size() == 0)) { // this is not a replacement, it is a deletion - remove(key); - return; + delete(key); + return 0; } // first find an old entry, if one exists kelondroRow.Entry oldindexrow = index.get(key); if (oldindexrow == null) { - // the collection is new - overwrite(key, collection); + if ((collection != null) && (collection.size() > 0)) { + // the collection is new + overwrite(key, collection); + } + return 0; } else { // overwrite the old collection // read old information @@ -167,14 +179,15 @@ public class kelondroCollectionIndex { int oldchunkcount = (int) oldindexrow.getColLongB256(idx_col_chunkcount); int oldrownumber = (int) oldindexrow.getColLongB256(idx_col_indexpos); int oldPartitionNumber = arrayIndex(oldchunkcount); + int oldSerialNumber = 0; - if (join) { + if (merge) { // load the old collection and join it with the old // open array entry - kelondroFixedWidthArray oldarray = getArray(oldPartitionNumber, oldchunksize); + kelondroFixedWidthArray oldarray = getArray(oldPartitionNumber, oldSerialNumber, oldchunksize); //System.out.println("joining for key " + new String(key) + ", oldrow=" + oldrownumber + ", oldchunkcount=" + oldchunkcount + ", array file=" + oldarray.filename); kelondroRow.Entry oldarrayrow = oldarray.get(oldrownumber); - if (oldarrayrow == null) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, oldchunksize, oldPartitionNumber).toString(), "array does not contain expected row"); + if (oldarrayrow == null) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, oldchunksize, oldPartitionNumber, oldSerialNumber).toString(), "array does not contain expected row"); // read the row and define a collection kelondroRowSet oldcollection = new kelondroRowSet(this.rowdef, oldarrayrow.getColBytes(1)); // FIXME: this does not yet work with different rowdef in case of several rowdef.objectsize() @@ -184,14 +197,45 @@ public class kelondroCollectionIndex { collection = oldcollection; } + int removed = 0; + if (removekeys != null) { + // load the old collection and remove keys + // open array entry + kelondroFixedWidthArray oldarray = getArray(oldPartitionNumber, oldSerialNumber, oldchunksize); + kelondroRow.Entry oldarrayrow = oldarray.get(oldrownumber); + if (oldarrayrow == null) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, oldchunksize, oldPartitionNumber, oldSerialNumber).toString(), "array does not contain expected row"); + + // read the row and define a collection + kelondroRowSet oldcollection = new kelondroRowSet(this.rowdef, oldarrayrow.getColBytes(1)); // FIXME: this does not yet work with different rowdef in case of several rowdef.objectsize() + + // remove the keys from the set + Iterator i = removekeys.iterator(); + Object k; + while (i.hasNext()) { + k = i.next(); + if (k instanceof byte[]) {if (oldcollection.remove((byte[]) k) != null) removed++;} + if (k instanceof String) {if (oldcollection.remove(((String) k).getBytes()) != null) removed++;} + } + collection = oldcollection; + } + + if (collection.size() == 0) { + if (deletecomplete) { + kelondroFixedWidthArray array = getArray(oldPartitionNumber, oldSerialNumber, oldchunksize); + array.remove(oldrownumber); + } + return removed; + } + int newPartitionNumber = arrayIndex(collection.size()); + int newSerialNumber = 0; // see if we need new space or if we can overwrite the old space if (oldPartitionNumber == newPartitionNumber) { // we don't need a new slot, just write into the old one // find array file - kelondroFixedWidthArray array = getArray(newPartitionNumber, this.rowdef.objectsize()); + kelondroFixedWidthArray array = getArray(newPartitionNumber, newSerialNumber, this.rowdef.objectsize()); // define row kelondroRow.Entry arrayEntry = array.row().newEntry(); @@ -203,12 +247,12 @@ public class kelondroCollectionIndex { // update the index entry oldindexrow.setColLongB256(idx_col_chunkcount, collection.size()); - oldindexrow.setColLongB256(idx_col_update, kelondroRowCollection.daysSince2000(System.currentTimeMillis())); + oldindexrow.setColLongB256(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis())); index.put(oldindexrow); } else { // we need a new slot, that means we must first delete the old entry // find array file - kelondroFixedWidthArray array = getArray(oldPartitionNumber, oldchunksize); + kelondroFixedWidthArray array = getArray(oldPartitionNumber, oldSerialNumber, oldchunksize); // delete old entry array.remove(oldrownumber); @@ -216,6 +260,7 @@ public class kelondroCollectionIndex { // write a new entry in the other array overwrite(key, collection); } + return removed; } } @@ -224,7 +269,7 @@ public class kelondroCollectionIndex { // simply store a collection without check if the collection existed before // find array file - kelondroFixedWidthArray array = getArray(arrayIndex(collection.size()), this.rowdef.objectsize()); + kelondroFixedWidthArray array = getArray(arrayIndex(collection.size()), 0, this.rowdef.objectsize()); // define row kelondroRow.Entry arrayEntry = array.row().newEntry(); @@ -240,55 +285,88 @@ public class kelondroCollectionIndex { indexEntry.setColLongB256(idx_col_chunksize, this.rowdef.objectsize()); indexEntry.setColLongB256(idx_col_chunkcount, collection.size()); indexEntry.setColLongB256(idx_col_indexpos, (long) newRowNumber); - indexEntry.setColLongB256(idx_col_update, kelondroRowCollection.daysSince2000(System.currentTimeMillis())); + indexEntry.setColLongB256(idx_col_lastread, kelondroRowCollection.daysSince2000(System.currentTimeMillis())); + indexEntry.setColLongB256(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis())); index.put(indexEntry); } - public kelondroRowSet get(byte[] key) throws IOException { + public kelondroRowSet get(byte[] key, boolean deleteIfEmpty) throws IOException { // find an entry, if one exists kelondroRow.Entry indexrow = index.get(key); if (indexrow == null) return null; - + return getdelete(indexrow, false, deleteIfEmpty); + } + + public kelondroRowSet delete(byte[] key) throws IOException { + // find an entry, if one exists + kelondroRow.Entry indexrow = index.get(key); + if (indexrow == null) return null; + return getdelete(indexrow, true, false); + } + + private kelondroRowSet getdelete(kelondroRow.Entry indexrow, boolean remove, boolean deleteIfEmpty) throws IOException { // read values int chunksize = (int) indexrow.getColLongB256(idx_col_chunksize); int chunkcount = (int) indexrow.getColLongB256(idx_col_chunkcount); int rownumber = (int) indexrow.getColLongB256(idx_col_indexpos); int partitionnumber = arrayIndex(chunkcount); + int serialnumber = 0; // open array entry - kelondroFixedWidthArray array = getArray(partitionnumber, chunksize); + kelondroFixedWidthArray array = getArray(partitionnumber, serialnumber, chunksize); kelondroRow.Entry arrayrow = array.get(rownumber); - if (arrayrow == null) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, partitionnumber).toString(), "array does not contain expected row"); + if (arrayrow == null) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, partitionnumber, serialnumber).toString(), "array does not contain expected row"); // read the row and define a collection kelondroRowSet collection = new kelondroRowSet(this.rowdef, arrayrow.getColBytes(1)); // FIXME: this does not yet work with different rowdef in case of several rowdef.objectsize() int chunkcountInArray = collection.size(); - if (chunkcountInArray != chunkcount) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, partitionnumber).toString(), "array has different chunkcount than index: index = " + chunkcount + ", array = " + chunkcountInArray); + if (chunkcountInArray != chunkcount) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, partitionnumber, serialnumber).toString(), "array has different chunkcount than index: index = " + chunkcount + ", array = " + chunkcountInArray); + + if ((remove) || ((chunkcountInArray == 0) && (deleteIfEmpty))) array.remove(rownumber); + return collection; } + + public Iterator keycollections(byte[] startKey, boolean rot) { + // returns an iteration of {byte[], kelondroRowSet} Objects + try { + return new keycollectionIterator(startKey, rot); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } - public int remove(byte[] key) throws IOException { - // returns the number of chunks that have been deleted with the removed collection + public class keycollectionIterator implements Iterator { - // find an entry, if one exists - kelondroRow.Entry indexrow = index.get(key); - if (indexrow == null) return 0; - - // read values - int chunksize = (int) indexrow.getColLongB256(idx_col_chunksize); - int chunkcount = (int) indexrow.getColLongB256(idx_col_chunkcount); - int rownumber = (int) indexrow.getColLongB256(idx_col_indexpos); - int partitionnumber = arrayIndex(chunkcount); + Iterator indexRowIterator; - // open array entry - kelondroFixedWidthArray array = getArray(partitionnumber, chunksize); + public keycollectionIterator(byte[] startKey, boolean rot) throws IOException { + // iterator of {byte[], kelondroRowSet} Objects + indexRowIterator = index.rows(true, rot, startKey); + } - // remove array entry - array.remove(rownumber); + public boolean hasNext() { + return indexRowIterator.hasNext(); + } + + public Object next() { + kelondroRow.Entry indexrow = (kelondroRow.Entry) indexRowIterator.next(); + if (indexrow == null) return null; + try { + return new Object[]{indexrow.getColBytes(0), getdelete(indexrow, false, false)}; + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + + public void remove() { + indexRowIterator.remove(); + } - return chunkcount; } - + public void close() throws IOException { this.index.close(); Iterator i = arrays.values().iterator(); @@ -300,7 +378,7 @@ public class kelondroCollectionIndex { public static void main(String[] args) { // define payload structure - kelondroRow rowdef = new kelondroRow("byte[] eins-10, byte[] zwei-80"); + kelondroRow rowdef = new kelondroRow("byte[] a-10, byte[] b-80"); File path = new File(args[0]); String filenameStub = args[1]; @@ -323,6 +401,7 @@ public class kelondroCollectionIndex { for (int j = 0; j < i; j++) { collection.add(rowdef.newEntry(new byte[][]{("abc" + j).getBytes(), "xxx".getBytes()})); } + System.out.println("put key-" + i + ": " + collection.toString()); collectionIndex.put(("key-" + i).getBytes(), collection); } @@ -332,7 +411,7 @@ public class kelondroCollectionIndex { for (int j = 0; j < i; j++) { collection.add(rowdef.newEntry(new byte[][]{("def" + j).getBytes(), "xxx".getBytes()})); } - collectionIndex.join(("key-" + i).getBytes(), collection); + collectionIndex.merge(("key-" + i).getBytes(), collection); } collectionIndex.close(); diff --git a/source/de/anomic/kelondro/kelondroColumn.java b/source/de/anomic/kelondro/kelondroColumn.java index 0f2ccb66e..d235c26c3 100644 --- a/source/de/anomic/kelondro/kelondroColumn.java +++ b/source/de/anomic/kelondro/kelondroColumn.java @@ -138,7 +138,7 @@ public class kelondroColumn { } // check length constraints - if (this.cellwidth <= 0) throw new kelondroException("kelondroColumn - no cell width given for " + this.nickname); + if (this.cellwidth < 0) throw new kelondroException("kelondroColumn - no cell width given for " + this.nickname); if (((typename.equals("boolean")) && (this.cellwidth > 1)) || ((typename.equals("byte")) && (this.cellwidth > 1)) || ((typename.equals("short")) && (this.cellwidth > 2)) || diff --git a/source/de/anomic/kelondro/kelondroFlexTable.java b/source/de/anomic/kelondro/kelondroFlexTable.java index 641af20ab..ccd6e62cd 100644 --- a/source/de/anomic/kelondro/kelondroFlexTable.java +++ b/source/de/anomic/kelondro/kelondroFlexTable.java @@ -44,11 +44,16 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr System.out.println("*** Last Startup time: " + stt + " milliseconds"); long start = System.currentTimeMillis(); + if ((preloadTime < 0) && (indexfile.exists())) { + // delete existing index file + System.out.println("*** Delete File index " + indexfile); + indexfile.delete(); + } if (indexfile.exists()) { // use existing index file System.out.println("*** Using File index " + indexfile); ki = new kelondroTree(indexfile, buffersize, preloadTime, 10); - } else if (stt > preloadTime) { + } else if ((preloadTime >= 0) && (stt > preloadTime)) { // generate new index file System.out.print("*** Generating File index for " + size() + " entries from " + indexfile); ki = initializeTreeIndex(indexfile, buffersize, preloadTime, objectOrder); @@ -153,4 +158,37 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr } } + public Iterator rows(boolean up, boolean rotating, byte[] firstKey) throws IOException { + return new rowIterator(up, rotating, firstKey); + } + + public class rowIterator implements Iterator { + + Iterator indexIterator; + + public rowIterator(boolean up, boolean rotating, byte[] firstKey) throws IOException { + indexIterator = index.rows(up, rotating, firstKey); + } + + public boolean hasNext() { + return indexIterator.hasNext(); + } + + public Object next() { + kelondroRow.Entry idxEntry = (kelondroRow.Entry) indexIterator.next(); + int idx = (int) idxEntry.getColLongB256(1); + try { + return get(idx); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + + public void remove() { + indexIterator.remove(); + } + + } + } diff --git a/source/de/anomic/kelondro/kelondroIndex.java b/source/de/anomic/kelondro/kelondroIndex.java index 8ee30d9a5..67b69cecb 100644 --- a/source/de/anomic/kelondro/kelondroIndex.java +++ b/source/de/anomic/kelondro/kelondroIndex.java @@ -51,6 +51,7 @@ package de.anomic.kelondro; import java.io.IOException; +import java.util.Iterator; public interface kelondroIndex { @@ -59,6 +60,6 @@ public interface kelondroIndex { public kelondroRow.Entry get(byte[] key) throws IOException; public kelondroRow.Entry put(kelondroRow.Entry row) throws IOException; public kelondroRow.Entry remove(byte[] key) throws IOException; - //public Iterator rows(boolean up, boolean rotating, byte[] firstKey) throws IOException; + public Iterator rows(boolean up, boolean rotating, byte[] firstKey) throws IOException; public void close() throws IOException; } diff --git a/source/de/anomic/kelondro/kelondroRowCollection.java b/source/de/anomic/kelondro/kelondroRowCollection.java index deeb9db2e..23a3436f0 100644 --- a/source/de/anomic/kelondro/kelondroRowCollection.java +++ b/source/de/anomic/kelondro/kelondroRowCollection.java @@ -48,6 +48,17 @@ public class kelondroRowCollection { this(rowdef, 0); } + public kelondroRowCollection(kelondroRowCollection rc) { + this.rowdef = rc.rowdef; + this.chunkcache = rc.chunkcache; + this.chunkcount = rc.chunkcount; + this.sortColumn = rc.sortColumn; + this.sortOrder = rc.sortOrder; + this.sortBound = rc.sortBound; + this.lastTimeRead = rc.lastTimeRead; + this.lastTimeWrote = rc.lastTimeWrote; + } + public kelondroRowCollection(kelondroRow rowdef, int objectCount) { this.rowdef = rowdef; this.chunkcache = new byte[objectCount * rowdef.objectsize()]; @@ -111,7 +122,6 @@ public class kelondroRowCollection { public byte[] exportCollection() { // returns null if the collection is empty - if (size() == 0) return null; trim(); kelondroRow row = exportRow(chunkcache.length); kelondroRow.Entry entry = row.newEntry(); diff --git a/source/de/anomic/kelondro/kelondroRowSet.java b/source/de/anomic/kelondro/kelondroRowSet.java index 6b5fda434..1790af66f 100644 --- a/source/de/anomic/kelondro/kelondroRowSet.java +++ b/source/de/anomic/kelondro/kelondroRowSet.java @@ -37,6 +37,12 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd private kelondroProfile profile; private TreeSet removeMarker; + public kelondroRowSet(kelondroRowSet rs) { + super(rs); + this.profile = rs.profile; + this.removeMarker = rs.removeMarker; + } + public kelondroRowSet(kelondroRow rowdef) { super(rowdef); this.removeMarker = new TreeSet(); diff --git a/source/de/anomic/kelondro/kelondroSplittedTree.java b/source/de/anomic/kelondro/kelondroSplittedTree.java index 90b48c0b7..c9c5e9f2f 100644 --- a/source/de/anomic/kelondro/kelondroSplittedTree.java +++ b/source/de/anomic/kelondro/kelondroSplittedTree.java @@ -153,9 +153,8 @@ public class kelondroSplittedTree implements kelondroIndex { public kelondroRow.Entry remove(byte[] key) throws IOException { return ktfs[partition(key)].remove(key); } - - public Iterator rows(boolean up, boolean rotating) throws IOException { - return new ktfsIterator(up, rotating); + public Iterator rows(boolean up, boolean rotating, byte[] firstKey) throws IOException { + return new ktfsIterator(up, rotating, firstKey); } public class ktfsIterator implements Iterator { @@ -164,11 +163,12 @@ public class kelondroSplittedTree implements kelondroIndex { Iterator ktfsI; boolean up, rot; - public ktfsIterator(boolean up, boolean rotating) throws IOException { + public ktfsIterator(boolean up, boolean rotating, byte[] firstKey) throws IOException { this.up = up; this.rot = rotating; c = (up) ? 0 : (ff - 1); - ktfsI = ktfs[c].rows(up, false, null); + if (firstKey != null) throw new UnsupportedOperationException("ktfsIterator does not work with a start key"); + ktfsI = ktfs[c].rows(up, false, firstKey); // FIXME: this works only correct with firstKey == null } public boolean hasNext() { diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index fe63e6b77..3a61ee628 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -405,7 +405,7 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI { } public static final int RL_RAMCACHE = 0; - public static final int RL_FILECACHE = 1; + public static final int RL_COLLECTIONS = 1; // the 'new' index structure public static final int RL_ASSORTMENTS = 2; public static final int RL_WORDFILES = 3;