- corrected return value of put() methods (not used anywhere, so it did not harm before)

- added use of LookAheadIterator which should prevent mistakes when coding iterators with embedded iterators
- added a fail-safe reaction in case of database corruption using iterators over database elements (no interruption then)


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7154 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/yacy/graphics/NetworkGraph.java

@@ -36,6 +36,7 @@ import java.util.Date;
 import java.util.Iterator;
 
 import net.yacy.cora.document.Hit;
+import net.yacy.kelondro.logging.Log;
 import net.yacy.visualization.PrintTool;
 import net.yacy.visualization.RasterPlotter;
 
@@ -196,10 +197,13 @@ public class NetworkGraph {
         Iterator<yacySeed> e = seedDB.seedsConnected(true, false, null, (float) 0.0);
         while (e.hasNext() && count < maxCount) {
             seed = e.next();
-            if (seed != null) {
-                drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seed, COL_ACTIVE_DOT, COL_ACTIVE_LINE, COL_ACTIVE_TEXT, coronaangle);
-                count++;
+            if (seed == null) {
+                Log.logWarning("NetworkGraph", "connected seed == null");
+                continue;
             }
+            //Log.logInfo("NetworkGraph", "drawing peer " + seed.getName());
+            drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seed, COL_ACTIVE_DOT, COL_ACTIVE_LINE, COL_ACTIVE_TEXT, coronaangle);
+            count++;
         }
         totalCount += count;
 
@@ -208,12 +212,16 @@ public class NetworkGraph {
         e = seedDB.seedsSortedDisconnected(false, yacySeed.LASTSEEN);
         while (e.hasNext() && count < maxCount) {
             seed = e.next();
-            if (seed != null) {
-                lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenUTC()) / 1000 / 60);
-                if (lastseen > passiveLimit) break; // we have enough, this list is sorted so we don't miss anything
-                drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seed, COL_PASSIVE_DOT, COL_PASSIVE_LINE, COL_PASSIVE_TEXT, coronaangle);
-                count++;
+            if (seed == null) {
+                Log.logWarning("NetworkGraph", "disconnected seed == null");
+                continue;
             }
+            lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenUTC()) / 1000 / 60);
+            if (lastseen > passiveLimit) {
+                break; // we have enough, this list is sorted so we don't miss anything
+            }
+            drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seed, COL_PASSIVE_DOT, COL_PASSIVE_LINE, COL_PASSIVE_TEXT, coronaangle);
+            count++;
         }
         totalCount += count;
 
@@ -222,12 +230,16 @@ public class NetworkGraph {
         e = seedDB.seedsSortedPotential(false, yacySeed.LASTSEEN);
         while (e.hasNext() && count < maxCount) {
             seed = e.next();
-            if (seed != null) {
-                lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenUTC()) / 1000 / 60);
-                if (lastseen > potentialLimit) break; // we have enough, this list is sorted so we don't miss anything
-                drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seed, COL_POTENTIAL_DOT, COL_POTENTIAL_LINE, COL_POTENTIAL_TEXT, coronaangle);
-                count++;
+            if (seed == null) {
+                Log.logWarning("NetworkGraph", "potential seed == null");
+                continue;
+            }
+            lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenUTC()) / 1000 / 60);
+            if (lastseen > potentialLimit) {
+                break; // we have enough, this list is sorted so we don't miss anything
             }
+            drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seed, COL_POTENTIAL_DOT, COL_POTENTIAL_LINE, COL_POTENTIAL_TEXT, coronaangle);
+            count++;
         }
         totalCount += count;
 

source/de/anomic/yacy/graphics/WebStructureGraph.java

@@ -46,6 +46,7 @@ import net.yacy.kelondro.order.Base64Order;
 import net.yacy.kelondro.order.MicroDate;
 import net.yacy.kelondro.util.DateFormatter;
 import net.yacy.kelondro.util.FileUtils;
+import net.yacy.kelondro.util.LookAheadIterator;
 
 
 public class WebStructureGraph {
@@ -463,21 +464,15 @@ public class WebStructureGraph {
         return new structureIterator(latest);
     }
     
-    public class structureIterator implements Iterator<structureEntry> {
+    public class structureIterator extends LookAheadIterator<structureEntry> implements Iterator<structureEntry> {
 
         private final Iterator<Map.Entry<String, String>> i;
-        private structureEntry nextentry;
         
         public structureIterator(final boolean latest) {
             i = ((latest) ? structure_new : structure_old).entrySet().iterator();
-            next0();
         }
         
-        public boolean hasNext() {
-            return nextentry != null;
-        }
-
-        private void next0() {
+        public structureEntry next0() {
             Map.Entry<String, String> entry = null;
             String dom = null, ref = "";
             while (i.hasNext()) {
@@ -488,24 +483,10 @@ public class WebStructureGraph {
                 if (dom.length() >= 8) break;
                 dom = null;
             }
-            if ((entry == null) || (dom == null)) {
-                nextentry = null;
-                return;
-            }
+            if (entry == null || dom == null) return null;
             assert (ref.length() - 8) % 10 == 0 : "refs = " + ref + ", length = " + ref.length();
-            nextentry = new structureEntry(dom.substring(0, 6), dom.substring(7), ref.substring(0, 8), refstr2map(ref));
-        }
-        
-        public structureEntry next() {
-            final structureEntry r = nextentry;
-            next0();
-            return r;
-        }
-
-        public void remove() {
-            throw new UnsupportedOperationException("not implemented");
+            return new structureEntry(dom.substring(0, 6), dom.substring(7), ref.substring(0, 8), refstr2map(ref));
         }
-        
     }
     
     public static class structureEntry {

source/de/anomic/yacy/yacySeedDB.java

@@ -1018,7 +1018,7 @@ public final class yacySeedDB implements AlternativeDomainNames {
         }
         
         private yacySeed internalNext() {
-            if ((it == null) || (!(it.hasNext()))) return null;
+            if (it == null || !(it.hasNext())) return null;
             try {
                 ConcurrentHashMap<String, String> dna;
                 while (it.hasNext()) {

source/net/yacy/kelondro/blob/HeapReader.java

@@ -45,6 +45,7 @@ import net.yacy.kelondro.order.CloneableIterator;
 import net.yacy.kelondro.order.NaturalOrder;
 import net.yacy.kelondro.order.RotateIterator;
 import net.yacy.kelondro.util.FileUtils;
+import net.yacy.kelondro.util.LookAheadIterator;
 import net.yacy.kelondro.util.MemoryControl;
 
 
@@ -571,7 +572,7 @@ public class HeapReader {
      * static iterator of entries in BLOBHeap files:
      * this is used to import heap dumps into a write-enabled index heap
      */
-    public static class entries implements
+    public static class entries extends LookAheadIterator<Map.Entry<byte[], byte[]>> implements
         CloneableIterator<Map.Entry<byte[], byte[]>>,
         Iterator<Map.Entry<byte[], byte[]>>,
         Iterable<Map.Entry<byte[], byte[]>> {
@@ -579,14 +580,12 @@ public class HeapReader {
         DataInputStream is;
         int keylen;
         private final File blobFile;
-        Map.Entry<byte[], byte[]> nextEntry;
         
         public entries(final File blobFile, final int keylen) throws IOException {
             if (!(blobFile.exists())) throw new IOException("file " + blobFile + " does not exist");
             this.is = new DataInputStream(new BufferedInputStream(new FileInputStream(blobFile), 8*1024*1024));
             this.keylen = keylen;
             this.blobFile = blobFile;
-            this.nextEntry = next0();
         }
 
         public CloneableIterator<Entry<byte[], byte[]>> clone(Object modifier) {
@@ -601,14 +600,7 @@ public class HeapReader {
             }
         }
         
-        public boolean hasNext() {
-            if (is == null) return false;
-            if  (this.nextEntry != null) return true;
-            close();
-            return false;
-        }
-
-        private Map.Entry<byte[], byte[]> next0() {
+        public Map.Entry<byte[], byte[]> next0() {
             try {
                 byte b;
                 int len;
@@ -637,10 +629,6 @@ public class HeapReader {
                     // so far we have read this.keylen - 1 + 1 = this.keylen bytes.
                     // there must be a remaining number of len - this.keylen bytes left for the BLOB
                     if (len < this.keylen) return null;    // a strange case that can only happen in case of corrupted data
-//                    if (is.available() < (len - this.keylen)) { // this really indicates corrupted data but doesn't work for >2GB Blobs
-//                        Log.logWarning("HeapReader", "corrupted data by entry of " + len + " bytes at available of: " + is.available() + " in " + this.blobFile.getName());
-//                        return null;
-//                    }
                     payload = new byte[len - this.keylen]; // the remaining record entries
                     if (is.read(payload) < payload.length) return null;
                     return new entry(key, payload);
@@ -650,20 +638,6 @@ public class HeapReader {
             }
         }
         
-        public Map.Entry<byte[], byte[]> next() {
-            final Map.Entry<byte[], byte[]> n = this.nextEntry;
-            this.nextEntry = next0();
-            return n;
-        }
-
-        public void remove() {
-            throw new UnsupportedOperationException("blobs cannot be altered during read-only iteration");
-        }
-
-        public Iterator<Map.Entry<byte[], byte[]>> iterator() {
-            return this;
-        }
-        
         public void close() {
             if (is != null) try { is.close(); } catch (final IOException e) {}
             is = null;

source/net/yacy/kelondro/blob/MapDataMining.java

@@ -38,6 +38,7 @@ import net.yacy.kelondro.index.RowSpaceExceededException;
 import net.yacy.kelondro.logging.Log;
 import net.yacy.kelondro.order.ByteOrder;
 import net.yacy.kelondro.order.CloneableIterator;
+import net.yacy.kelondro.util.LookAheadIterator;
 import net.yacy.kelondro.util.ScoreCluster;
 
 
@@ -363,29 +364,17 @@ public class MapDataMining extends MapHeap {
         super.close();
     }
     
-    public class mapIterator implements Iterator<Map<String, String>> {
+    public class mapIterator extends LookAheadIterator<Map<String, String>> implements Iterator<Map<String, String>> {
         // enumerates Map-Type elements
         // the key is also included in every map that is returned; it's key is 'key'
 
         Iterator<byte[]> keyIterator;
-        Map<String, String> n;
 
         public mapIterator(final Iterator<byte[]> keyIterator) {
             this.keyIterator = keyIterator;
-            this.n = next0();
-        }
-
-        public boolean hasNext() {
-            return this.n != null;
-        }
-
-        public Map<String, String> next() {
-            final Map<String, String> n1 = n;
-            n = next0();
-            return n1;
         }
         
-        private Map<String, String> next0() {
+        public Map<String, String> next0() {
             if (keyIterator == null) return null;
             byte[] nextKey;
             Map<String, String> map;
@@ -394,7 +383,8 @@ public class MapDataMining extends MapHeap {
                 try {
                     map = get(nextKey);
                 } catch (final IOException e) {
-                    break;
+                    Log.logWarning("MapDataMining", e.getMessage());
+                    continue;
                 } catch (RowSpaceExceededException e) {
                     Log.logException(e);
                     continue;
@@ -405,9 +395,5 @@ public class MapDataMining extends MapHeap {
             }
             return null;
         }
-
-        public void remove() {
-            throw new UnsupportedOperationException();
-        }
     } // class mapIterator
 }

source/net/yacy/kelondro/data/word/Word.java

@@ -55,7 +55,7 @@ public class Word {
     public static final int commonHashLength = 12;
     
     private static final int hashCacheSize = Math.max(10000, Math.min(100000, (int) (MemoryControl.available() / 20000L)));
-    private static final ARC<String, byte[]> hashCache = new ConcurrentARC<String, byte[]>(hashCacheSize, Runtime.getRuntime().availableProcessors());
+    private static final ARC<String, byte[]> hashCache = new ConcurrentARC<String, byte[]>(hashCacheSize, Runtime.getRuntime().availableProcessors() + 1);
     
     // object carries statistics for words and sentences
     public  int      count;       // number of occurrences

source/net/yacy/kelondro/index/BufferedObjectIndex.java

@@ -149,6 +149,13 @@ public class BufferedObjectIndex implements Index, Iterable<Row.Entry> {
         }
     }
 
+    /**
+     * Adds the row to the index. The row is identified by the primary key of the row.
+     * @param row a index row
+     * @return true if this set did _not_ already contain the given row. 
+     * @throws IOException
+     * @throws RowSpaceExceededException
+     */
     public boolean put(Entry row) throws IOException, RowSpaceExceededException {
         synchronized (this.backend) {
             checkBuffer();

source/net/yacy/kelondro/index/HandleMap.java

@@ -194,6 +194,14 @@ public final class HandleMap implements Iterable<Row.Entry> {
         return indexentry.getColLong(1);
     }
     
+    /**
+     * Adds the key-value pair to the index.
+     * @param key the index key
+     * @param l the value
+     * @return the previous entry of the index
+     * @throws IOException
+     * @throws RowSpaceExceededException
+     */
     public final synchronized long put(final byte[] key, final long l) throws RowSpaceExceededException {
         assert l >= 0 : "l = " + l;
         assert (key != null);

source/net/yacy/kelondro/index/HandleSet.java

@@ -143,6 +143,13 @@ public final class HandleSet implements Iterable<byte[]>, Cloneable {
         for (byte[] b: aset) put(b);
     }
 
+    /**
+     * Adds the key to the set
+     * @param key
+     * @return true if this set did _not_ already contain the given key. 
+     * @throws IOException
+     * @throws RowSpaceExceededException
+     */
     public final synchronized boolean put(final byte[] key) throws RowSpaceExceededException {
         assert (key != null);
         final Row.Entry newentry = index.row().newEntry();

source/net/yacy/kelondro/index/Index.java

@@ -44,6 +44,14 @@ public interface Index extends Iterable<Row.Entry> {
     public boolean has(byte[] key); // use this only if there is no get in case that has returns true
     public Row.Entry get(byte[] key) throws IOException;
     public Row.Entry replace(Row.Entry row) throws RowSpaceExceededException, IOException;
+    
+    /**
+     * Adds the row to the index. The row is identified by the primary key of the row.
+     * @param row a index row
+     * @return true if this set did _not_ already contain the given row. 
+     * @throws IOException
+     * @throws RowSpaceExceededException
+     */
     public boolean put(Row.Entry row) throws IOException, RowSpaceExceededException;
     public void addUnique(Row.Entry row) throws RowSpaceExceededException, IOException; // no double-check
     public ArrayList<RowCollection> removeDoubles() throws IOException, RowSpaceExceededException; // removes all elements that are double (to be used after all addUnique)

source/net/yacy/kelondro/index/RAMIndex.java

@@ -157,33 +157,44 @@ public final class RAMIndex implements Index, Iterable<Row.Entry> {
         // else place it in the index1
         return index1.replace(entry);
     }
-   
-	public final synchronized boolean put(final Row.Entry entry) throws RowSpaceExceededException {
+	
+	/**
+     * Adds the row to the index. The row is identified by the primary key of the row.
+     * @param row a index row
+     * @return true if this set did _not_ already contain the given row. 
+     * @throws IOException
+     * @throws RowSpaceExceededException
+     */
+	public final boolean put(final Row.Entry entry) throws RowSpaceExceededException {
         assert (entry != null);
-        if (entry == null) return false;
-        finishInitialization();
-        // if the new entry is within the initialization part, just overwrite it
-        assert index0.isSorted();
-        final byte[] key = entry.getPrimaryKeyBytes();
-        if (index0.has(key)) {
-            // replace the entry
-            index0.put(entry);
-            return true;
+        if (entry == null) return true;
+        synchronized (this) {
+            finishInitialization();
+            // if the new entry is within the initialization part, just overwrite it
+            assert index0.isSorted();
+            final byte[] key = entry.getPrimaryKeyBytes();
+            if (index0.has(key)) {
+                // replace the entry
+                index0.put(entry);
+                return false;
+            }
+            // else place it in the index1
+            return index1.put(entry);
         }
-        // else place it in the index1
-        return index1.put(entry);
     }
    
-    public final synchronized void addUnique(final Row.Entry entry) throws RowSpaceExceededException {
+    public final void addUnique(final Row.Entry entry) throws RowSpaceExceededException {
     	assert (entry != null);
     	if (entry == null) return;
-        if (index1 == null) {
-            // we are in the initialization phase
-        	index0.addUnique(entry);
-        	return;
-        }
-        // initialization is over, add to secondary index
-        index1.addUnique(entry);
+    	synchronized (this) {
+            if (index1 == null) {
+                // we are in the initialization phase
+            	index0.addUnique(entry);
+            	return;
+            }
+            // initialization is over, add to secondary index
+            index1.addUnique(entry);
+    	}
     }
 
 	public final void addUnique(final List<Entry> rows) throws RowSpaceExceededException {

source/net/yacy/kelondro/index/RAMIndexCluster.java

@@ -114,6 +114,7 @@ public final class RAMIndexCluster implements Index, Iterable<Row.Entry>, Clonea
     
     public final void addUnique(final Entry row) throws RowSpaceExceededException {
         final int i = indexFor(row);
+        assert i >= 0 : "i = " + i;
         if (i < 0) return;
         accessArray(i).addUnique(row);
     }
@@ -171,10 +172,18 @@ public final class RAMIndexCluster implements Index, Iterable<Row.Entry>, Clonea
             return MergeIterator.cascade(col, this.rowdef.objectOrder, MergeIterator.simpleMerge, up);
         }            
     }
-
+    
+    /**
+     * Adds the row to the index. The row is identified by the primary key of the row.
+     * @param row a index row
+     * @return true if this set did _not_ already contain the given row. 
+     * @throws IOException
+     * @throws RowSpaceExceededException
+     */
     public final boolean put(final Entry row) throws RowSpaceExceededException {
         final int i = indexFor(row);
-        if (i < 0) return false;
+        assert i >= 0 : "i = " + i;
+        if (i < 0) return true;
         return accessArray(i).put(row);
     }
 
@@ -236,6 +245,7 @@ public final class RAMIndexCluster implements Index, Iterable<Row.Entry>, Clonea
     
     public final Entry replace(final Entry row) throws RowSpaceExceededException {
         final int i = indexFor(row);
+        assert i >= 0 : "i = " + i;
         if (i < 0) return null;
         return accessArray(i).replace(row);
     }

source/net/yacy/kelondro/index/RowSet.java

@@ -24,6 +24,7 @@
 
 package net.yacy.kelondro.index;
 
+import java.io.IOException;
 import java.util.Iterator;
 import java.util.Random;
 
@@ -113,6 +114,13 @@ public class RowSet extends RowCollection implements Index, Iterable<Row.Entry>
         return get(index, true);
     }
     
+    /**
+     * Adds the row to the index. The row is identified by the primary key of the row.
+     * @param row a index row
+     * @return true if this set did _not_ already contain the given row. 
+     * @throws IOException
+     * @throws RowSpaceExceededException
+     */
     public final synchronized boolean put(final Row.Entry entry) throws RowSpaceExceededException {
         assert (entry != null);
         assert (entry.getPrimaryKeyBytes() != null);
@@ -124,12 +132,12 @@ public class RowSet extends RowCollection implements Index, Iterable<Row.Entry>
         final int index = find(entry.bytes(), 0);
         if (index < 0) {
             super.addUnique(entry);
-            return false;
+            return true;
         } else {
             final int sb = this.sortBound; // save the sortBound, because it is not altered (we replace at the same place)
             set(index, entry);       // this may alter the sortBound, which we will revert in the next step
             this.sortBound = sb;     // revert a sortBound altering
-            return true;
+            return false;
         }
     }
 

source/net/yacy/kelondro/table/ChunkIterator.java

@@ -34,8 +34,9 @@ import java.io.IOException;
 import java.util.Iterator;
 
 import net.yacy.kelondro.logging.Log;
+import net.yacy.kelondro.util.LookAheadIterator;
 
-public class ChunkIterator implements Iterator<byte[]> {
+public class ChunkIterator extends LookAheadIterator<byte[]> implements Iterator<byte[]> {
 
     private final int chunksize;
     
@@ -52,7 +53,6 @@ public class ChunkIterator implements Iterator<byte[]> {
     
     
     private final DataInputStream stream;
-    private byte[] nextBytes;
     private final int recordsize;
     
     public ChunkIterator(final File file, final int recordsize, final int chunksize) throws FileNotFoundException {
@@ -61,13 +61,8 @@ public class ChunkIterator implements Iterator<byte[]> {
         this.recordsize = recordsize;
         this.chunksize = chunksize;
         this.stream = new DataInputStream(new BufferedInputStream(new FileInputStream(file), 64 * 1024));
-        this.nextBytes = next0();
     }
     
-    public boolean hasNext() {
-        return nextBytes != null;
-    }
-
     public byte[] next0() {
         final byte[] chunk = new byte[chunksize];
         int r, s;
@@ -91,14 +86,4 @@ public class ChunkIterator implements Iterator<byte[]> {
             return null;
         }
     }
-
-    public byte[] next() {
-        final byte[] n = this.nextBytes;
-        this.nextBytes = next0();
-        return n;
-    }
-    
-    public void remove() {
-        throw new UnsupportedOperationException("no remove in ChunkIterator possible");
-    }
 }

source/net/yacy/kelondro/table/SplitTable.java

@@ -354,6 +354,13 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
         return null;
     }
     
+    /**
+     * Adds the row to the index. The row is identified by the primary key of the row.
+     * @param row a index row
+     * @return true if this set did _not_ already contain the given row. 
+     * @throws IOException
+     * @throws RowSpaceExceededException
+     */
     public boolean put(final Row.Entry row) throws IOException, RowSpaceExceededException {
         assert row.objectsize() <= this.rowdef.objectsize;
         Index keeper = keeperOf(row.getColBytes(0, true));
@@ -362,7 +369,9 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
             assert this.current == null || this.tables.get(this.current) != null : "this.current = " + this.current;
             keeper = (this.current == null) ? newTable() : checkTable(this.tables.get(this.current));
         }
-        return keeper.put(row);
+        boolean b = keeper.put(row);
+        assert b;
+        return b;
     }
     
 

source/net/yacy/kelondro/table/Table.java

@@ -543,12 +543,19 @@ public class Table implements Index, Iterable<Row.Entry> {
         return rowdef.newEntry(b);
     }
     
+    /**
+     * Adds the row to the index. The row is identified by the primary key of the row.
+     * @param row a index row
+     * @return true if this set did _not_ already contain the given row. 
+     * @throws IOException
+     * @throws RowSpaceExceededException
+     */
     public synchronized boolean put(final Entry row) throws IOException, RowSpaceExceededException {
         assert file == null || file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size() + ", file = " + this.filename();
         assert table == null || table.size() == index.size() : "table.size() = " + table.size() + ", index.size() = " + index.size() + ", file = " + this.filename();
         assert row != null;
         assert row.bytes() != null;
-        if (file == null || row == null || row.bytes() == null) return false;
+        if (file == null || row == null || row.bytes() == null) return true;
         final int i = (int) index.get(row.getPrimaryKeyBytes());
         if (i == -1) {
             try {
@@ -558,7 +565,7 @@ public class Table implements Index, Iterable<Row.Entry> {
                 this.table = null;
                 addUnique(row);
             }
-            return false;
+            return true;
         }
         
         if (table == null) {
@@ -576,7 +583,7 @@ public class Table implements Index, Iterable<Row.Entry> {
         }
         assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size();
         assert table == null || table.size() == index.size() : "table.size() = " + table.size() + ", index.size() = " + index.size();
-        return true;
+        return false;
     }
 
     public Entry put(final Entry row, final Date entryDate) throws IOException, RowSpaceExceededException {