implemented indexCollectionRI

this is the new database structure that is supposed to replace the
plasmaAssortmentCluster AND the plasmaWordIndexFileCluster
The new structure is not yet active and needs to be integrated into
plasmaWordIndex. This has some migration constraints that are not yet
completely solved.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2347 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/dbtest.java

@@ -290,7 +290,7 @@ public class dbtest {
             
             if (command.equals("list")) {
                 Iterator i = null;
-                if (table instanceof kelondroSplittedTree) i = ((kelondroSplittedTree) table).rows(true, false);
+                if (table instanceof kelondroSplittedTree) i = ((kelondroSplittedTree) table).rows(true, false, null);
                 if (table instanceof kelondroTree) i = ((kelondroTree) table).rows(true, false, null);
                 if (table instanceof dbTable) i = ((dbTable) table).rows(true, false, null);
                 byte[][] row;

source/de/anomic/index/indexCollectionRI.java

@@ -28,12 +28,16 @@ package de.anomic.index;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Set;
 
 import de.anomic.kelondro.kelondroCollectionIndex;
 import de.anomic.kelondro.kelondroNaturalOrder;
+import de.anomic.kelondro.kelondroOutOfLimitsException;
 import de.anomic.kelondro.kelondroRow;
+import de.anomic.kelondro.kelondroRowCollection;
+import de.anomic.kelondro.kelondroRowSet;
 
 public class indexCollectionRI extends indexAbstractRI implements indexRI {
 
@@ -63,68 +67,87 @@ public class indexCollectionRI extends indexAbstractRI implements indexRI {
 
     public class wordContainersIterator implements Iterator {
 
-        //private Iterator wci;
+        private Iterator wci;
         
         public wordContainersIterator(String startWordHash, boolean rot) {
-
+            wci = collectionIndex.keycollections(startWordHash.getBytes(), rot);
         }
         
         public boolean hasNext() {
-            // TODO Auto-generated method stub
-            return false;
+            return wci.hasNext();
         }
 
         public Object next() {
-            // TODO Auto-generated method stub
-            return null;
+            Object[] oo = (Object[]) wci.next();
+            byte[] key = (byte[]) oo[0];
+            kelondroRowSet collection = (kelondroRowSet) oo[1];
+            if (collection == null) return null;
+            return new indexRowSetContainer(new String(key), collection);
         }
         
         public void remove() {
-            // TODO Auto-generated method stub
-            
+            wci.remove();
         }
 
     }
      
     public indexContainer getContainer(String wordHash, boolean deleteIfEmpty, long maxtime) {
         try {
-            indexRowSetContainer idx = (indexRowSetContainer) collectionIndex.get(wordHash.getBytes());
-            idx.setWordHash(wordHash);
-            return idx;
+            kelondroRowSet collection = collectionIndex.get(wordHash.getBytes(), deleteIfEmpty);
+            if (collection == null) return null;
+            return new indexRowSetContainer(wordHash, collection);
         } catch (IOException e) {
-            e.printStackTrace();
             return null;
         }
     }
 
     public indexContainer deleteContainer(String wordHash) {
-        indexContainer idx = getContainer(wordHash, true, -1);
         try {
-            collectionIndex.remove(wordHash.getBytes());
+            kelondroRowSet collection = collectionIndex.delete(wordHash.getBytes());
+            if (collection == null) return null;
+            return new indexRowSetContainer(wordHash, collection);
         } catch (IOException e) {
-            e.printStackTrace();
+            return null;
         }
-        return idx;
     }
 
     public boolean removeEntry(String wordHash, String urlHash, boolean deleteComplete) {
-        // TODO Auto-generated method stub
-        return false;
+        HashSet hs = new HashSet();
+        hs.add(urlHash.getBytes());
+        return removeEntries(wordHash, hs, deleteComplete) == 1;
     }
     
     public int removeEntries(String wordHash, Set urlHashes, boolean deleteComplete) {
-        // TODO Auto-generated method stub
-        return 0;
+        try {
+            return collectionIndex.remove(wordHash.getBytes(), urlHashes, deleteComplete);
+        } catch (kelondroOutOfLimitsException e) {
+            e.printStackTrace();
+            return 0;
+        } catch (IOException e) {
+            e.printStackTrace();
+            return 0;
+        }
     }
 
     public indexContainer addEntries(indexContainer newEntries, long creationTime, boolean dhtCase) {
-        // TODO Auto-generated method stub
-        return null;
+        String wordHash = newEntries.getWordHash();
+        try {
+            collectionIndex.merge(wordHash.getBytes(), (kelondroRowCollection) newEntries);
+            return getContainer(wordHash, true, -1); // FIXME: this is not optimal
+        } catch (kelondroOutOfLimitsException e) {
+            e.printStackTrace();
+            return null;
+        } catch (IOException e) {
+            return null;
+        }
     }
 
     public void close(int waitingSeconds) {
-        // TODO Auto-generated method stub
-        
+        try {
+            collectionIndex.close();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
     }
     
 }

source/de/anomic/index/indexRowSetContainer.java

@@ -46,6 +46,11 @@ public class indexRowSetContainer extends kelondroRowSet implements indexContain
         this(wordHash, new kelondroNaturalOrder(true), 0);
     }
     
+    public indexRowSetContainer(String wordHash, kelondroRowSet collection) {
+        super(collection);
+        this.wordHash = wordHash;
+    }
+    
     public indexRowSetContainer(String wordHash, kelondroOrder ordering, int column) {
         super(indexURLEntry.urlEntryRow);
         this.wordHash = wordHash;

source/de/anomic/kelondro/kelondroBytesIntMap.java

@@ -25,6 +25,7 @@
 package de.anomic.kelondro;
 
 import java.io.IOException;
+import java.util.Iterator;
 
 public class kelondroBytesIntMap {
     
@@ -62,4 +63,11 @@ public class kelondroBytesIntMap {
         return ki.size();
     }
     
+    public Iterator rows(boolean up, boolean rotating, byte[] firstKey) throws IOException {
+        // returns the row-iterator of the underlying kelondroIndex
+        // col[0] = key
+        // col[1] = integer as {b265}
+        return ki.rows(up, rotating, firstKey);
+    }
+    
 }

source/de/anomic/kelondro/kelondroCollectionIndex.java

@@ -30,6 +30,7 @@ import java.io.IOException;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.Set;
 
 public class kelondroCollectionIndex {
 
@@ -45,7 +46,8 @@ public class kelondroCollectionIndex {
     private static final int idx_col_chunksize  = 1;  // chunksize (number of bytes in a single chunk, needed for migration option)
     private static final int idx_col_chunkcount = 2;  // chunkcount (number of chunks in this collection) needed to identify array file that has the chunks
     private static final int idx_col_indexpos   = 3;  // indexpos (position in index file)
-    private static final int idx_col_update     = 4;  // a time stamp, update time in days since 1.1.2000
+    private static final int idx_col_lastread   = 4;  // a time stamp, update time in days since 1.1.2000
+    private static final int idx_col_lastwrote  = 5;  // a time stamp, update time in days since 1.1.2000
 
     private static kelondroRow indexRow(int keylen) {
         return new kelondroRow(
@@ -53,11 +55,12 @@ public class kelondroCollectionIndex {
             "int chunksize-4 {b256}," +
             "int chunkcount-4 {b256}," +
             "int indexpos-4 {b256}," +
-            "short update-2 {b256}"
+            "short lastread-2 {b256}" +
+            "short lastwrote-2 {b256}"
             );
     }
     
-    private static File arrayFile(File path, String filenameStub, int loadfactor, int chunksize, int partitionNumber) {
+    private static File arrayFile(File path, String filenameStub, int loadfactor, int chunksize, int partitionNumber, int serialNumber) {
 
         String lf = Integer.toHexString(loadfactor).toUpperCase();
         while (lf.length() < 2) lf = "0" + lf;
@@ -65,7 +68,9 @@ public class kelondroCollectionIndex {
         while (cs.length() < 4) cs = "0" + cs;
         String pn = Integer.toHexString(partitionNumber).toUpperCase();
         while (pn.length() < 2) pn = "0" + pn;
-        return new File(path, filenameStub + "." + lf + "." + cs + "." + pn + ".kca"); // kelondro collection array
+        String sn = Integer.toHexString(serialNumber).toUpperCase();
+        while (sn.length() < 2) sn = "0" + sn;
+        return new File(path, filenameStub + "." + lf + "." + cs + "." + pn + "." + sn + ".kca"); // kelondro collection array
     }
 
     public kelondroCollectionIndex(File path, String filenameStub, int keyLength, kelondroOrder indexOrder,
@@ -84,8 +89,8 @@ public class kelondroCollectionIndex {
         this.arrays = new HashMap(); // all entries will be dynamically created with getArray()
     }
     
-    private kelondroFixedWidthArray openArrayFile(int partitionNumber, boolean create) throws IOException {
-        File f = arrayFile(path, filenameStub, loadfactor, rowdef.objectsize(), partitionNumber);
+    private kelondroFixedWidthArray openArrayFile(int partitionNumber, int serialNumber, boolean create) throws IOException {
+        File f = arrayFile(path, filenameStub, loadfactor, rowdef.objectsize(), partitionNumber, serialNumber);
         
         if (f.exists()) {
             return new kelondroFixedWidthArray(f);
@@ -101,12 +106,12 @@ public class kelondroCollectionIndex {
         }
     }
     
-    private kelondroFixedWidthArray getArray(int partitionNumber, int chunksize) {
+    private kelondroFixedWidthArray getArray(int partitionNumber, int serialNumber, int chunksize) {
         String accessKey = partitionNumber + "-" + chunksize;
         kelondroFixedWidthArray array = (kelondroFixedWidthArray) arrays.get(accessKey);
         if (array != null) return array;
         try {
-            array = openArrayFile(partitionNumber, true);
+            array = openArrayFile(partitionNumber, serialNumber, true);
         } catch (IOException e) {
             return null;
         }
@@ -138,28 +143,35 @@ public class kelondroCollectionIndex {
     public void put(byte[] key, kelondroRowCollection collection) throws IOException, kelondroOutOfLimitsException {
         // this replaces an old collection by a new one
         // this method is not approriate to extend an existing collection with another collection
-        insert(key, collection, false);
+        putmergeremove(key, collection, false, null, false);
     }
     
-    public void join(byte[] key, kelondroRowCollection collection) throws IOException, kelondroOutOfLimitsException {
-        insert(key, collection, true);
+    public void merge(byte[] key, kelondroRowCollection collection) throws IOException, kelondroOutOfLimitsException {
+        putmergeremove(key, collection, true, null, false);
     }       
 
-    private void insert(byte[] key, kelondroRowCollection collection, boolean join) throws IOException, kelondroOutOfLimitsException {
+    public int remove(byte[] key, Set removekeys, boolean deletecomplete) throws IOException, kelondroOutOfLimitsException {
+        return putmergeremove(key, null, false, removekeys, deletecomplete);
+    }
+
+    private int putmergeremove(byte[] key, kelondroRowCollection collection, boolean merge, Set removekeys, boolean deletecomplete) throws IOException, kelondroOutOfLimitsException {
         //if (collection.size() > maxChunks) throw new kelondroOutOfLimitsException(maxChunks, collection.size());
 
-        if (collection.size() == 0) {
+        if ((!merge) && (collection.size() == 0)) {
             // this is not a replacement, it is a deletion
-            remove(key);
-            return;
+            delete(key);
+            return 0;
         }
         
         // first find an old entry, if one exists
         kelondroRow.Entry oldindexrow = index.get(key);
         
         if (oldindexrow == null) {
-            // the collection is new
-            overwrite(key, collection);
+            if ((collection != null) && (collection.size() > 0)) {
+                // the collection is new
+                overwrite(key, collection);
+            }
+            return 0;
         } else {
             // overwrite the old collection
             // read old information
@@ -167,14 +179,15 @@ public class kelondroCollectionIndex {
             int oldchunkcount = (int) oldindexrow.getColLongB256(idx_col_chunkcount);
             int oldrownumber  = (int) oldindexrow.getColLongB256(idx_col_indexpos);
             int oldPartitionNumber = arrayIndex(oldchunkcount);
+            int oldSerialNumber = 0;
             
-            if (join) {
+            if (merge) {
                 // load the old collection and join it with the old
                 // open array entry
-                kelondroFixedWidthArray oldarray = getArray(oldPartitionNumber, oldchunksize);
+                kelondroFixedWidthArray oldarray = getArray(oldPartitionNumber, oldSerialNumber, oldchunksize);
                 //System.out.println("joining for key " + new String(key) + ", oldrow=" + oldrownumber + ", oldchunkcount=" + oldchunkcount + ", array file=" + oldarray.filename);
                 kelondroRow.Entry oldarrayrow = oldarray.get(oldrownumber);
-                if (oldarrayrow == null) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, oldchunksize, oldPartitionNumber).toString(), "array does not contain expected row");
+                if (oldarrayrow == null) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, oldchunksize, oldPartitionNumber, oldSerialNumber).toString(), "array does not contain expected row");
 
                 // read the row and define a collection
                 kelondroRowSet oldcollection = new kelondroRowSet(this.rowdef, oldarrayrow.getColBytes(1)); // FIXME: this does not yet work with different rowdef in case of several rowdef.objectsize()
@@ -184,14 +197,45 @@ public class kelondroCollectionIndex {
                 collection = oldcollection;
             }
             
+            int removed = 0;
+            if (removekeys != null) {
+                // load the old collection and remove keys
+                // open array entry
+                kelondroFixedWidthArray oldarray = getArray(oldPartitionNumber, oldSerialNumber, oldchunksize);
+                kelondroRow.Entry oldarrayrow = oldarray.get(oldrownumber);
+                if (oldarrayrow == null) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, oldchunksize, oldPartitionNumber, oldSerialNumber).toString(), "array does not contain expected row");
+
+                // read the row and define a collection
+                kelondroRowSet oldcollection = new kelondroRowSet(this.rowdef, oldarrayrow.getColBytes(1)); // FIXME: this does not yet work with different rowdef in case of several rowdef.objectsize()
+                
+                // remove the keys from the set
+                Iterator i = removekeys.iterator();
+                Object k;
+                while (i.hasNext()) {
+                    k = i.next();
+                    if (k instanceof byte[]) {if (oldcollection.remove((byte[]) k) != null) removed++;}
+                    if (k instanceof String) {if (oldcollection.remove(((String) k).getBytes()) != null) removed++;}
+                }
+                collection = oldcollection;
+            }
+            
+            if (collection.size() == 0) {
+                if (deletecomplete) {
+                    kelondroFixedWidthArray array = getArray(oldPartitionNumber, oldSerialNumber, oldchunksize);
+                    array.remove(oldrownumber);
+                }
+                return removed;
+            }
+            
             int newPartitionNumber = arrayIndex(collection.size());
+            int newSerialNumber = 0;
             
             // see if we need new space or if we can overwrite the old space
             if (oldPartitionNumber == newPartitionNumber) {
                 // we don't need a new slot, just write into the old one
 
                 // find array file
-                kelondroFixedWidthArray array = getArray(newPartitionNumber, this.rowdef.objectsize());
+                kelondroFixedWidthArray array = getArray(newPartitionNumber, newSerialNumber, this.rowdef.objectsize());
                 
                 // define row
                 kelondroRow.Entry arrayEntry = array.row().newEntry();
@@ -203,12 +247,12 @@ public class kelondroCollectionIndex {
                 
                 // update the index entry
                 oldindexrow.setColLongB256(idx_col_chunkcount, collection.size());
-                oldindexrow.setColLongB256(idx_col_update, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
+                oldindexrow.setColLongB256(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
                 index.put(oldindexrow);
             } else {
                 // we need a new slot, that means we must first delete the old entry
                 // find array file
-                kelondroFixedWidthArray array = getArray(oldPartitionNumber, oldchunksize);
+                kelondroFixedWidthArray array = getArray(oldPartitionNumber, oldSerialNumber, oldchunksize);
                 
                 // delete old entry
                 array.remove(oldrownumber);
@@ -216,6 +260,7 @@ public class kelondroCollectionIndex {
                 // write a new entry in the other array
                 overwrite(key, collection);
             }
+            return removed;
         }
     }
 
@@ -224,7 +269,7 @@ public class kelondroCollectionIndex {
         // simply store a collection without check if the collection existed before
         
         // find array file
-        kelondroFixedWidthArray array = getArray(arrayIndex(collection.size()), this.rowdef.objectsize());
+        kelondroFixedWidthArray array = getArray(arrayIndex(collection.size()), 0, this.rowdef.objectsize());
         
         // define row
         kelondroRow.Entry arrayEntry = array.row().newEntry();
@@ -240,55 +285,88 @@ public class kelondroCollectionIndex {
         indexEntry.setColLongB256(idx_col_chunksize, this.rowdef.objectsize());
         indexEntry.setColLongB256(idx_col_chunkcount, collection.size());
         indexEntry.setColLongB256(idx_col_indexpos, (long) newRowNumber);
-        indexEntry.setColLongB256(idx_col_update, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
+        indexEntry.setColLongB256(idx_col_lastread, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
+        indexEntry.setColLongB256(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
         index.put(indexEntry);
     }
     
-    public kelondroRowSet get(byte[] key) throws IOException {
+    public kelondroRowSet get(byte[] key, boolean deleteIfEmpty) throws IOException {
         // find an entry, if one exists
         kelondroRow.Entry indexrow = index.get(key);
         if (indexrow == null) return null;
-        
+        return getdelete(indexrow, false, deleteIfEmpty);
+    }
+    
+    public kelondroRowSet delete(byte[] key) throws IOException {
+        // find an entry, if one exists
+        kelondroRow.Entry indexrow = index.get(key);
+        if (indexrow == null) return null;
+        return getdelete(indexrow, true, false);
+    }
+    
+    private kelondroRowSet getdelete(kelondroRow.Entry indexrow, boolean remove, boolean deleteIfEmpty) throws IOException {
         // read values
         int chunksize  = (int) indexrow.getColLongB256(idx_col_chunksize);
         int chunkcount = (int) indexrow.getColLongB256(idx_col_chunkcount);
         int rownumber  = (int) indexrow.getColLongB256(idx_col_indexpos);
         int partitionnumber = arrayIndex(chunkcount);
+        int serialnumber = 0;
         
         // open array entry
-        kelondroFixedWidthArray array = getArray(partitionnumber, chunksize);
+        kelondroFixedWidthArray array = getArray(partitionnumber, serialnumber, chunksize);
         kelondroRow.Entry arrayrow = array.get(rownumber);
-        if (arrayrow == null) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, partitionnumber).toString(), "array does not contain expected row");
+        if (arrayrow == null) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, partitionnumber, serialnumber).toString(), "array does not contain expected row");
 
         // read the row and define a collection
         kelondroRowSet collection = new kelondroRowSet(this.rowdef, arrayrow.getColBytes(1)); // FIXME: this does not yet work with different rowdef in case of several rowdef.objectsize()
         int chunkcountInArray = collection.size();
-        if (chunkcountInArray != chunkcount) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, partitionnumber).toString(), "array has different chunkcount than index: index = " + chunkcount + ", array = " + chunkcountInArray);
+        if (chunkcountInArray != chunkcount) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, partitionnumber, serialnumber).toString(), "array has different chunkcount than index: index = " + chunkcount + ", array = " + chunkcountInArray);
+        
+        if ((remove) || ((chunkcountInArray == 0) && (deleteIfEmpty))) array.remove(rownumber);
+        
         return collection;
     }
+
+    public Iterator keycollections(byte[] startKey, boolean rot) {
+        // returns an iteration of {byte[], kelondroRowSet} Objects
+        try {
+            return new keycollectionIterator(startKey, rot);
+        } catch (IOException e) {
+            e.printStackTrace();
+            return null;
+        }
+    }
     
-    public int remove(byte[] key) throws IOException {
-        // returns the number of chunks that have been deleted with the removed collection
+    public class keycollectionIterator implements Iterator {
         
-        // find an entry, if one exists
-        kelondroRow.Entry indexrow = index.get(key);
-        if (indexrow == null) return 0;
-
-        // read values
-        int chunksize  = (int) indexrow.getColLongB256(idx_col_chunksize);
-        int chunkcount = (int) indexrow.getColLongB256(idx_col_chunkcount);
-        int rownumber  = (int) indexrow.getColLongB256(idx_col_indexpos);
-        int partitionnumber = arrayIndex(chunkcount);
+        Iterator indexRowIterator;
         
-        // open array entry
-        kelondroFixedWidthArray array = getArray(partitionnumber, chunksize);
+        public keycollectionIterator(byte[] startKey, boolean rot) throws IOException {
+            // iterator of {byte[], kelondroRowSet} Objects
+            indexRowIterator = index.rows(true, rot, startKey);
+        }
         
-        // remove array entry
-        array.remove(rownumber);
+        public boolean hasNext() {
+            return indexRowIterator.hasNext();
+        }
+
+        public Object next() {
+            kelondroRow.Entry indexrow = (kelondroRow.Entry) indexRowIterator.next();
+            if (indexrow == null) return null;
+            try {
+                return new Object[]{indexrow.getColBytes(0), getdelete(indexrow, false, false)};
+            } catch (IOException e) {
+                e.printStackTrace();
+                return null;
+            }
+        }
+
+        public void remove() {
+            indexRowIterator.remove();
+        }
         
-        return chunkcount;
     }
-
+    
     public void close() throws IOException {
         this.index.close();
         Iterator i = arrays.values().iterator();
@@ -300,7 +378,7 @@ public class kelondroCollectionIndex {
     public static void main(String[] args) {
 
         // define payload structure
-        kelondroRow rowdef = new kelondroRow("byte[] eins-10, byte[] zwei-80");
+        kelondroRow rowdef = new kelondroRow("byte[] a-10, byte[] b-80");
         
         File path = new File(args[0]);
         String filenameStub = args[1];
@@ -323,6 +401,7 @@ public class kelondroCollectionIndex {
                 for (int j = 0; j < i; j++) {
                     collection.add(rowdef.newEntry(new byte[][]{("abc" + j).getBytes(), "xxx".getBytes()}));
                 }
+                System.out.println("put key-" + i + ": " + collection.toString());
                 collectionIndex.put(("key-" + i).getBytes(), collection);
             }
             
@@ -332,7 +411,7 @@ public class kelondroCollectionIndex {
                 for (int j = 0; j < i; j++) {
                     collection.add(rowdef.newEntry(new byte[][]{("def" + j).getBytes(), "xxx".getBytes()}));
                 }
-                collectionIndex.join(("key-" + i).getBytes(), collection);
+                collectionIndex.merge(("key-" + i).getBytes(), collection);
             }
             
             collectionIndex.close();

source/de/anomic/kelondro/kelondroColumn.java

@@ -138,7 +138,7 @@ public class kelondroColumn {
         }
         
         // check length constraints
-        if (this.cellwidth <= 0) throw new kelondroException("kelondroColumn - no cell width given for " + this.nickname);
+        if (this.cellwidth < 0) throw new kelondroException("kelondroColumn - no cell width given for " + this.nickname);
         if (((typename.equals("boolean")) && (this.cellwidth > 1)) ||
             ((typename.equals("byte")) && (this.cellwidth > 1)) ||
             ((typename.equals("short")) && (this.cellwidth > 2)) ||

source/de/anomic/kelondro/kelondroFlexTable.java

@@ -44,11 +44,16 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
         System.out.println("*** Last Startup time: " + stt + " milliseconds");
         long start = System.currentTimeMillis();
 
+        if ((preloadTime < 0) && (indexfile.exists())) {
+            // delete existing index file
+            System.out.println("*** Delete File index " + indexfile);
+            indexfile.delete();
+        }
         if (indexfile.exists()) {
             // use existing index file
             System.out.println("*** Using File index " + indexfile);
             ki = new kelondroTree(indexfile, buffersize, preloadTime, 10);
-        } else if (stt > preloadTime) {
+        } else if ((preloadTime >= 0) && (stt > preloadTime)) {
             // generate new index file
             System.out.print("*** Generating File index for " + size() + " entries from " + indexfile);
             ki = initializeTreeIndex(indexfile, buffersize, preloadTime, objectOrder);
@@ -153,4 +158,37 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
         }
     }
 
+    public Iterator rows(boolean up, boolean rotating, byte[] firstKey) throws IOException {
+        return new rowIterator(up, rotating, firstKey);
+    }
+    
+    public class rowIterator implements Iterator {
+
+        Iterator indexIterator;
+        
+        public rowIterator(boolean up, boolean rotating, byte[] firstKey) throws IOException {
+            indexIterator = index.rows(up, rotating, firstKey);
+        }
+        
+        public boolean hasNext() {
+            return indexIterator.hasNext();
+        }
+
+        public Object next() {
+            kelondroRow.Entry idxEntry = (kelondroRow.Entry) indexIterator.next();
+            int idx = (int) idxEntry.getColLongB256(1);
+            try {
+                return get(idx);
+            } catch (IOException e) {
+                e.printStackTrace();
+                return null;
+            }
+        }
+
+        public void remove() {
+            indexIterator.remove();
+        }
+        
+    }
+
 }

source/de/anomic/kelondro/kelondroIndex.java

@@ -51,6 +51,7 @@
 package de.anomic.kelondro;
 
 import java.io.IOException;
+import java.util.Iterator;
 
 public interface kelondroIndex {
 
@@ -59,6 +60,6 @@ public interface kelondroIndex {
     public kelondroRow.Entry get(byte[] key) throws IOException;
     public kelondroRow.Entry put(kelondroRow.Entry row) throws IOException;
     public kelondroRow.Entry remove(byte[] key) throws IOException;
-    //public Iterator rows(boolean up, boolean rotating, byte[] firstKey) throws IOException;
+    public Iterator rows(boolean up, boolean rotating, byte[] firstKey) throws IOException;
     public void close() throws IOException;
 }

source/de/anomic/kelondro/kelondroRowCollection.java

@@ -48,6 +48,17 @@ public class kelondroRowCollection {
         this(rowdef, 0);
     }
     
+    public kelondroRowCollection(kelondroRowCollection rc) {
+        this.rowdef = rc.rowdef;
+        this.chunkcache = rc.chunkcache;
+        this.chunkcount = rc.chunkcount;
+        this.sortColumn = rc.sortColumn;
+        this.sortOrder = rc.sortOrder;
+        this.sortBound = rc.sortBound;
+        this.lastTimeRead = rc.lastTimeRead;
+        this.lastTimeWrote = rc.lastTimeWrote;
+    }
+    
     public kelondroRowCollection(kelondroRow rowdef, int objectCount) {
         this.rowdef = rowdef;
         this.chunkcache = new byte[objectCount * rowdef.objectsize()];
@@ -111,7 +122,6 @@ public class kelondroRowCollection {
     
     public byte[] exportCollection() {
         // returns null if the collection is empty
-        if (size() == 0) return null;
         trim();
         kelondroRow row = exportRow(chunkcache.length);
         kelondroRow.Entry entry = row.newEntry();

source/de/anomic/kelondro/kelondroRowSet.java

@@ -37,6 +37,12 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
     private kelondroProfile profile;
     private TreeSet removeMarker;
 
+    public kelondroRowSet(kelondroRowSet rs) {
+        super(rs);
+        this.profile = rs.profile;
+        this.removeMarker = rs.removeMarker;
+    }
+    
     public kelondroRowSet(kelondroRow rowdef) {
         super(rowdef);
         this.removeMarker = new TreeSet();

source/de/anomic/kelondro/kelondroSplittedTree.java

@@ -153,9 +153,8 @@ public class kelondroSplittedTree implements kelondroIndex {
     public kelondroRow.Entry remove(byte[] key) throws IOException {
         return ktfs[partition(key)].remove(key);
     }
-
-    public Iterator rows(boolean up, boolean rotating) throws IOException {
-        return new ktfsIterator(up, rotating);
+    public Iterator rows(boolean up, boolean rotating, byte[] firstKey) throws IOException {
+        return new ktfsIterator(up, rotating, firstKey);
     }
     
     public class ktfsIterator implements Iterator {
@@ -164,11 +163,12 @@ public class kelondroSplittedTree implements kelondroIndex {
         Iterator ktfsI;
         boolean up, rot;
         
-        public ktfsIterator(boolean up, boolean rotating) throws IOException {
+        public ktfsIterator(boolean up, boolean rotating, byte[] firstKey) throws IOException {
             this.up = up;
             this.rot = rotating;
             c = (up) ? 0 : (ff - 1);
-            ktfsI = ktfs[c].rows(up, false, null);
+            if (firstKey != null) throw new UnsupportedOperationException("ktfsIterator does not work with a start key");
+            ktfsI = ktfs[c].rows(up, false, firstKey); // FIXME: this works only correct with firstKey == null
         }
         
         public boolean hasNext() {

source/de/anomic/plasma/plasmaWordIndex.java

@@ -405,7 +405,7 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI {
     }
     
     public static final int RL_RAMCACHE    = 0;
-    public static final int RL_FILECACHE   = 1;
+    public static final int RL_COLLECTIONS = 1; // the 'new' index structure
     public static final int RL_ASSORTMENTS = 2;
     public static final int RL_WORDFILES   = 3;