replaced old keyIterator and rowIterator by buffered iterators

that are synchronized with database access
Main change is done in kelondroTree, other classes are only adoptions

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1918 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/dbtest.java

@@ -213,7 +213,7 @@ public class dbtest {
             }
             
             if (command.equals("list")) {
-                Iterator i = table.rows(true, false);
+                Iterator i = table.rows(true, false, null);
                 byte[][] row;
                 while (i.hasNext()) {
                     row = (byte[][]) i.next();
@@ -377,7 +377,7 @@ final class dbTable implements kelondroIndex {
         return null;
     }
     
-    public Iterator rows(boolean up, boolean rotating) throws IOException {
+    public Iterator rows(boolean up, boolean rotating, byte[] startKey) throws IOException {
         // Objects are of type byte[][]
         return null;
     }

source/de/anomic/kelondro/kelondroAbstractOrder.java

@@ -50,9 +50,14 @@ import de.anomic.kelondro.kelondroRecords.Node;
 public abstract class kelondroAbstractOrder implements kelondroOrder {
 
     protected byte[] zero = null;
+    protected boolean asc = true;
     
     public abstract Object clone();
 
+    public void direction(boolean ascending) {
+        asc = ascending;
+    }
+    
     public long partition(byte[] key, int forks) {
         final long d = (Long.MAX_VALUE / forks) + ((Long.MAX_VALUE % forks) + 1) / forks;
         return cardinal(key) / d;

source/de/anomic/kelondro/kelondroBase64Order.java

@@ -70,7 +70,6 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
     public static final kelondroBase64Order enhancedCoder = new kelondroBase64Order(true, false);
 
     private boolean rfc1113compliant;
-    private boolean asc;
     private final char[] alpha;
     private final byte[] ahpla;
 

source/de/anomic/kelondro/kelondroDyn.java

@@ -189,7 +189,7 @@ public class kelondroDyn extends kelondroTree {
     public synchronized dynKeyIterator dynKeys(boolean up, boolean rotating) throws IOException {
         // iterates only the keys of the Nodes
         // enumerated objects are of type String
-        return new dynKeyIterator(super.rows(up, rotating));
+        return new dynKeyIterator(super.rows(up, rotating, null));
     }
 
     public synchronized dynKeyIterator dynKeys(boolean up, boolean rotating, byte[] firstKey) throws IOException {

source/de/anomic/kelondro/kelondroFScoreCluster.java

@@ -135,7 +135,7 @@ public class kelondroFScoreCluster {
         Iterator iterator;
         
         public scoreIterator(boolean up, boolean rotating) throws IOException {
-            iterator = countrefDB.rows(up, rotating);
+            iterator = countrefDB.rows(up, rotating, null);
         }
        
         public boolean hasNext() {

source/de/anomic/kelondro/kelondroIndex.java

@@ -61,5 +61,5 @@ public interface kelondroIndex {
     public byte[][] get(byte[] key) throws IOException;
     public byte[][] put(byte[][] row) throws IOException;
     public byte[][] remove(byte[] key) throws IOException;
-    public Iterator rows(boolean up, boolean rotating) throws IOException; // Objects are of type byte[][]
+    public Iterator rows(boolean up, boolean rotating, byte[] startKey) throws IOException; // Objects are of type byte[][]
 }

source/de/anomic/kelondro/kelondroNaturalOrder.java

@@ -49,8 +49,6 @@ import java.util.Comparator;
 
 public class kelondroNaturalOrder extends kelondroAbstractOrder implements kelondroOrder, Comparator, Cloneable {
     
-    boolean asc;
-    
     public static final kelondroOrder naturalOrder = new kelondroNaturalOrder(true);
     
     public kelondroNaturalOrder(boolean ascending) {

source/de/anomic/kelondro/kelondroOrder.java

@@ -51,6 +51,8 @@ public interface kelondroOrder extends Comparator {
 
     public Object clone();
     
+    public void direction(boolean ascending); // the ordering direction can be changed at any time
+    
     public String signature(); // returns a signature String so that different orderings have different signatures
     
     public long partition(byte[] key, int forkes);

source/de/anomic/kelondro/kelondroSplittedTree.java

@@ -153,8 +153,8 @@ public class kelondroSplittedTree implements kelondroIndex {
         return ktfs[partition(key)].remove(key);
     }
 
-    public Iterator rows(boolean up, boolean rotating) throws IOException {
+    public Iterator rows(boolean up, boolean rotating, byte[] startKey) throws IOException {
-        return new ktfsIterator(up, rotating);
+        return new ktfsIterator(up, rotating, startKey);
     }
     
     public class ktfsIterator implements Iterator {
@@ -162,12 +162,14 @@ public class kelondroSplittedTree implements kelondroIndex {
         int c = 0;
         Iterator ktfsI;
         boolean up, rot;
+        byte[] start;
         
-        public ktfsIterator(boolean up, boolean rotating) throws IOException {
+        public ktfsIterator(boolean up, boolean rotating, byte[] startKey) throws IOException {
             this.up = up;
             this.rot = rotating;
+            this.start = startKey;
             c = (up) ? 0 : (ff - 1);
-            ktfsI = ktfs[c].rows(up, false);
+            ktfsI = ktfs[c].rows(up, false, null);
         }
         
         public boolean hasNext() {
@@ -183,7 +185,7 @@ public class kelondroSplittedTree implements kelondroIndex {
                 if (c < (ff - 1)) {
                     c++;
                     try {
-                        ktfsI = ktfs[c].rows(true, false);
+                        ktfsI = ktfs[c].rows(true, false, start);
                     } catch (IOException e) {
                         return null;
                     }
@@ -192,7 +194,7 @@ public class kelondroSplittedTree implements kelondroIndex {
                     if (rot) {
                         c = 0;
                         try {
-                            ktfsI = ktfs[c].rows(true, false);
+                            ktfsI = ktfs[c].rows(true, false, start);
                         } catch (IOException e) {
                             return null;
                         }
@@ -204,7 +206,7 @@ public class kelondroSplittedTree implements kelondroIndex {
                 if (c > 0) {
                     c--;
                     try {
-                        ktfsI = ktfs[c].rows(false, false);
+                        ktfsI = ktfs[c].rows(false, false, start);
                     } catch (IOException e) {
                         return null;
                     }
@@ -213,7 +215,7 @@ public class kelondroSplittedTree implements kelondroIndex {
                     if (rot) {
                         c = ff - 1;
                         try {
-                            ktfsI = ktfs[c].rows(false, false);
+                            ktfsI = ktfs[c].rows(false, false, start);
                         } catch (IOException e) {
                             return null;
                         }

source/de/anomic/kelondro/kelondroTables.java

@@ -169,12 +169,6 @@ public class kelondroTables {
         return tree.rows(up, rotating, firstKey);
     }
     
-    public synchronized Iterator /* of byte[][]-Elements */ rows(String tablename, boolean up, boolean rotating) throws IOException {
-        kelondroTree tree = (kelondroTree) tTables.get(tablename);
-        if (tree == null) throw new RuntimeException("kelondroTables.selectOrderBy: tree table '" + tablename + "' does not exist.");
-        return tree.rows(up, rotating);
-    }
-    
     // if you need the long-values from a row-iteration, please use kelondroRecords.bytes2long to convert from byte[] to long
     
     public synchronized void delete(String tablename, String key) throws IOException {

source/de/anomic/kelondro/kelondroTree.java

@@ -54,9 +54,11 @@ import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.StringTokenizer;
+import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.Vector;
 import java.util.logging.Logger;
+import java.util.Map;
 
 public class kelondroTree extends kelondroRecords implements kelondroIndex {
 	
@@ -807,26 +809,6 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
 	return node;
     }
     
-    /*
-    private synchronized Iterator nodeIterator(boolean up, boolean rotating) {
-        // iterates the elements in a sorted way. returns Node - type Objects
-        try {
-            return new nodeIterator(up, rotating);
-        } catch (IOException e) {
-            throw new RuntimeException("error creating an iteration: " + e.getMessage());
-        }
-    }
-
-    private synchronized Iterator nodeIterator(boolean up, boolean rotating, byte[] firstKey) {
-        // iterates the elements in a sorted way. returns Node - type Objects
-        try {
-            return new nodeIterator(up, rotating, firstKey, true);
-        } catch (IOException e) {
-            throw new RuntimeException("error creating an iteration: " + e.getMessage());
-        }
-    }
-    */
-    
     private class nodeIterator implements Iterator {
         // we implement an iteration! (not a recursive function as the structure would suggest...)
         // the iterator iterates Node objects
@@ -1009,78 +991,165 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
         }
     }
 
-    public synchronized Iterator rows(boolean up, boolean rotating) throws IOException {
+    public TreeMap rows(boolean up, boolean rotating, byte[] firstKey, boolean including, int count) throws IOException {
+        // returns an ordered map of keys/row relations; key objects are of type String, value objects are of type byte[][]
+        kelondroOrder setOrder = (kelondroOrder) objectOrder.clone();
+        setOrder.direction(up);
+        setOrder.rotate(firstKey);
+        TreeMap rows = new TreeMap(setOrder);
+        Node n;
+        synchronized (this) {
+            Iterator i = (firstKey == null) ? new nodeIterator(up, rotating) : new nodeIterator(up, rotating, firstKey, including);
+            while ((rows.size() < count) && (i.hasNext())) {
+                n = (Node) i.next();
+                if (n != null) rows.put(new String(n.getKey()), n.getValues());
+            }
+        }
+        return rows;
+    }
+    
+    public TreeSet keys(boolean up, boolean rotating, byte[] firstKey, boolean including, int count) throws IOException {
+        // returns an ordered set of keys; objects are of type String
+        kelondroOrder setOrder = (kelondroOrder) objectOrder.clone();
+        setOrder.direction(up);
+        setOrder.rotate(firstKey);
+        TreeSet set = new TreeSet(setOrder);
+        Node n;
+        synchronized (this) {
+            Iterator i = (firstKey == null) ? new nodeIterator(up, rotating) : new nodeIterator(up, rotating, firstKey, including);
+            while ((set.size() < count) && (i.hasNext())) {
+                n = (Node) i.next();
+                if (n != null) set.add(new String(n.getKey()));
+            }
+        }
+        return set;
+    }
+    
+    public Iterator rows(boolean up, boolean rotating, byte[] firstKey) throws IOException {
         // iterates the rows of the Nodes
         // enumerated objects are of type byte[][]
         // iterates the elements in a sorted way.
-	return new rowIterator(new nodeIterator(up, rotating));
+        // if iteration should start at smallest element, set firstKey to null
+        return new rowIterator(up, rotating, firstKey, this.size());
     }
     
-    public synchronized Iterator rows(boolean up, boolean rotating, byte[] firstKey) throws IOException {
+    public Iterator keys(boolean up, boolean rotating, byte[] firstKey) throws IOException {
-        return new rowIterator((firstKey == null) ? new nodeIterator(up, rotating) : new nodeIterator(up, rotating, firstKey, true));
+        // iterates only the keys of the Nodes
+        // enumerated objects are of type String
+        // iterates the elements in a sorted way.
+        // if iteration should start at smallest element, set firstKey to null
+        return new keyIterator(up, rotating, firstKey, this.size());
     }
     
     public class rowIterator implements Iterator {
         
-        Iterator nodeIterator;
+        int chunkSize;
-        
+        byte[] start;
-        public rowIterator(Iterator nodeIterator) {
+        boolean inc, rot;
-            this.nodeIterator = nodeIterator;
+        long count;
+        byte[] lastKey;
+        TreeMap rowBuffer;
+        Iterator bufferIterator;
+        
+        public rowIterator(boolean up, boolean rotating, byte[] firstKey, long guessedCountLimit) throws IOException {
+            start = firstKey;
+            inc = up;
+            rot = rotating;
+            count = 0;
+            lastKey = null;
+            chunkSize = (int) Math.min(100, guessedCountLimit);
+            rowBuffer = rows(inc, rot, start, true, chunkSize);
+            bufferIterator = rowBuffer.entrySet().iterator();
         }
         
         public boolean hasNext() {
-            return (nodeIterator.hasNext());
+            return ((bufferIterator != null) && (bufferIterator.hasNext()) && ((rot) || (count < size())));
         }
         
         public Object next() {
+            if (!(bufferIterator.hasNext())) return null;
+            Map.Entry entry = (Map.Entry) bufferIterator.next();
+            lastKey = ((String) entry.getKey()).getBytes();
+            
+            // check if this was the last entry in the rowBuffer
+            if (!(bufferIterator.hasNext())) {
+                // assign next buffer chunk
                 try {
-		Node nextNode = (Node) nodeIterator.next();
+                    rowBuffer = rows(inc, rot, lastKey, false, chunkSize);
-		if (nextNode == null) throw new kelondroException(filename, "no more elements available");
+                    bufferIterator = rowBuffer.entrySet().iterator();
-                return nextNode.getValues();
                 } catch (IOException e) {
-                throw new kelondroException(filename, "io-error: " + e.getMessage());
+                    rowBuffer = null;
+                    bufferIterator = null;
                 }
             }
             
-        public void remove() {
+            // return the row
+            count++;
+            return entry.getValue();
         }
         
+        public void remove() {
+            if (lastKey != null) try {
+                kelondroTree.this.remove(lastKey);
+            } catch (IOException e) {
+                // do nothing
             }
-
-    public synchronized keyIterator keys(boolean up, boolean rotating) throws IOException {
-        // iterates only the keys of the Nodes
-        // enumerated objects are of type String
-        // iterates the elements in a sorted way.
-        return new keyIterator(new nodeIterator(up, rotating));
         }
         
-    public Iterator keys(boolean up, boolean rotating, byte[] firstKey) throws IOException {
-        return new keyIterator(new nodeIterator(up, rotating, firstKey, true));
     }
     
     public class keyIterator implements Iterator {
         
-        Iterator nodeIterator;
+        int chunkSize;
-        
+        byte[] start;
-        public keyIterator(Iterator nodeIterator) {
+        boolean inc, rot;
-            this.nodeIterator = nodeIterator;
+        long count;
-        }
+        String lastKey;
-        
+        TreeSet keyBuffer;
-        public void finalize() {
+        Iterator bufferIterator;
-            nodeIterator = null;
+        
+        public keyIterator(boolean up, boolean rotating, byte[] firstKey, long guessedCountLimit) throws IOException {
+            start = firstKey;
+            inc = up;
+            rot = rotating;
+            count = 0;
+            lastKey = null;
+            chunkSize = (int) Math.min(100, guessedCountLimit);
+            keyBuffer = keys(inc, rot, start, true, chunkSize);
+            bufferIterator = keyBuffer.iterator();
         }
                 
         public boolean hasNext() {
-            return (nodeIterator.hasNext());
+            return ((bufferIterator != null) && (bufferIterator.hasNext()) && ((rot) || (count < size())));
         }
         
         public Object next() {
-            Node nextNode = (Node) nodeIterator.next();
+            if (!(bufferIterator.hasNext())) return null;
-            if (nextNode == null) throw new kelondroException(filename, "no more elements available");
+            lastKey = (String) bufferIterator.next();
-            return new String(nextNode.getKey());
+            
+            // check if this was the last entry in the rowBuffer
+            if (!(bufferIterator.hasNext())) {
+                // assign next buffer chunk
+                try {
+                    keyBuffer = keys(inc, rot, lastKey.getBytes(), false, chunkSize);
+                    bufferIterator = keyBuffer.iterator();
+                } catch (IOException e) {
+                    keyBuffer = null;
+                    bufferIterator = null;
+                }
+            }
+            
+            // return the row
+            count++;
+            return lastKey;
         }
         
         public void remove() {
+            if (lastKey != null) try {
+                kelondroTree.this.remove(lastKey.getBytes());
+            } catch (IOException e) {
+                // do nothing
+            }
         }
         
     }
@@ -1469,7 +1538,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
             for (int i = 0; i < 100; i++) {
                 b = ("T" + i).getBytes(); tt.put(b, b);
             }
-            Iterator i = tt.keys(true, false);
+            Iterator i = tt.keys(true, false, null);
             while (i.hasNext()) System.out.print((String) i.next() + ", ");
             System.out.println();
 
@@ -1558,7 +1627,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
     public static int countElements(kelondroTree t) {
         int count = 0;
         try {
-            Iterator iter = t.rows(true, false);
+            Iterator iter = t.rows(true, false, null);
             byte[][] row;
             while (iter.hasNext()) {
                 count++;

source/de/anomic/plasma/plasmaCrawlEURL.java

@@ -261,7 +261,7 @@ public class plasmaCrawlEURL extends plasmaURL {
         // enumerates entry elements
         Iterator i;
         public kenum(boolean up, boolean rotating) throws IOException {
-            i = urlHashCache.rows(up, rotating);
+            i = urlHashCache.rows(up, rotating, null);
         }
         public boolean hasMoreElements() {
             return i.hasNext();

source/de/anomic/plasma/plasmaCrawlLURL.java

@@ -755,7 +755,7 @@ public final class plasmaCrawlLURL extends plasmaURL {
         boolean error = false;
         
         public kiter(boolean up, boolean rotating) throws IOException {
-            i = urlHashCache.rows(up, rotating);
+            i = urlHashCache.rows(up, rotating, null);
             error = false;
         }
 

source/de/anomic/plasma/plasmaCrawlStacker.java

@@ -542,7 +542,7 @@ public final class plasmaCrawlStacker {
                 }
                 try {
                     // loop through the list and fill the messageList with url hashs
-                    Iterator iter = this.urlEntryCache.keys(true, false);
+                    Iterator iter = this.urlEntryCache.keys(true, false, null);
                     String urlHash;
                     while (iter.hasNext()) {
                         urlHash = (String) iter.next();

source/de/anomic/plasma/plasmaWordIndex.java

@@ -380,7 +380,7 @@ public final class plasmaWordIndex {
     
     public synchronized TreeSet wordHashes(String startHash, int resourceLevel, boolean rot, int count) throws IOException {
         kelondroOrder hashOrder = (kelondroOrder) indexOrder.clone();
-        if (rot) hashOrder.rotate(startHash.getBytes()); else hashOrder.rotate(null);
+        hashOrder.rotate(startHash.getBytes());
         TreeSet hashes = new TreeSet(hashOrder);
         Iterator i = wordHashes(startHash, resourceLevel, rot);
         String hash;

source/de/anomic/plasma/plasmaWordIndexEntity.java

@@ -214,7 +214,7 @@ public final class plasmaWordIndexEntity {
             if (theIndex == null) {
                 i = null;
             } else try {
-                i = theIndex.rows(up, false);
+                i = theIndex.rows(up, false, null);
             } catch (kelondroException e) {
                 e.printStackTrace();
                 theIndex.file().delete();

source/de/anomic/yacy/yacyNewsDB.java

@@ -137,18 +137,18 @@ public class yacyNewsDB {
 
     public class recordIterator implements Iterator {
 
-        Iterator nodeIterator;
+        Iterator rowIterator;
 
         public recordIterator() throws IOException {
-            nodeIterator = news.rows(true, false);
+            rowIterator = news.rows(true, false, null);
         }
 
         public boolean hasNext() {
-            return nodeIterator.hasNext();
+            return rowIterator.hasNext();
         }
 
         public Object next() {
-            return b2r((byte[][]) nodeIterator.next());
+            return b2r((byte[][]) rowIterator.next());
         }
 
         public void remove() {

source/de/anomic/yacy/yacySeedDB.java

@@ -818,11 +818,6 @@ public final class yacySeedDB {
             if ((it == null) || (!(it.hasNext()))) return null;
             Map dna = (Map) it.next();
             String hash = (String) dna.remove("key");
-            /* Users of SVN 1498 and 1499 might have extries with shortened hashes in their DBs.
-             * Those are no problem since they should be eliminated from the active DB quickly.
-             * They might stay in the passive and potential DB indefinatly, but are not affecting
-             * normal operations there.
-             */
             //while (hash.length() < commonHashLength) { hash = hash + "_"; }
             return new yacySeed(hash, dna);
         }

source/yacy.java

@@ -95,7 +95,6 @@ import de.anomic.tools.enumerateFiles;
 import de.anomic.yacy.yacyClient;
 import de.anomic.yacy.yacyCore;
 import de.anomic.yacy.yacySeedDB;
-import de.anomic.data.listManager;
 
 /**
 * This is the main class of YaCy. Several threads are started from here: