- smaller caches to save memory

- close cloneable iterators to free memory

source/net/yacy/cora/order/CloneableIterator.java

@@ -28,8 +28,17 @@ import java.util.Iterator;
 
 public interface CloneableIterator<E> extends Iterator<E> {
 
-    // clone the iterator using a modifier
-    // the modifier can be i.e. a re-start position
+    /**
+     * clone the iterator using a modifier
+     * the modifier can be i.e. a re-start position
+     * @param modifier
+     * @return
+     */
     public CloneableIterator<E> clone(Object modifier);
 
+    /**
+     * a CloneableIterator should be closed after usage to free resources
+     */
+    public void close();
+
 }

source/net/yacy/cora/order/CloneableMapIterator.java

@@ -81,4 +81,8 @@ public class CloneableMapIterator<E> implements CloneableIterator<E> {
 		this.map.remove(this.last);
 	}
 
+    @Override
+    public void close() {
+    }
+
 }

source/net/yacy/kelondro/blob/ArrayStack.java

@@ -1010,6 +1010,7 @@ public class ArrayStack implements BLOB {
             final HeapWriter writer = new HeapWriter(tmpFile, newFile, keylength, order, writeBuffer);
             rewrite(i, order, writer);
             writer.close(true);
+            i.close();
         } catch (final IOException e) {
             Log.logSevere("ArrayStack", "cannot writing or close writing rewrite, newFile = " + newFile.toString() + ", tmpFile = " + tmpFile.toString() + ": " + e.getMessage(), e);
             FileUtils.deletedelete(tmpFile);

source/net/yacy/kelondro/blob/BEncodedHeap.java

@@ -121,6 +121,10 @@ public class BEncodedHeap implements MapStore {
                     return this;
                 }
 
+                @Override
+                public void close() {
+                }
+
             };
         }
     }

source/net/yacy/kelondro/blob/MapHeap.java

@@ -385,14 +385,15 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
         final boolean up, rotating;
         final byte[] firstKey, secondKey;
         Iterator<byte[]> iterator;
+        final private CloneableIterator<byte[]> blobkeys;
 
         public KeyIterator(final boolean up, final boolean rotating, final byte[] firstKey, final byte[] secondKey) throws IOException {
             this.up = up;
             this.rotating = rotating;
             this.firstKey = firstKey;
             this.secondKey = secondKey;
-            final CloneableIterator<byte[]> i = MapHeap.this.blob.keys(up, firstKey);
-            this.iterator = rotating ? new RotateIterator<byte[]>(i, secondKey, MapHeap.this.blob.size()) : i;
+            this.blobkeys = MapHeap.this.blob.keys(up, firstKey);
+            this.iterator = rotating ? new RotateIterator<byte[]>(this.blobkeys, secondKey, MapHeap.this.blob.size()) : this.blobkeys;
         }
 
         @Override
@@ -421,6 +422,10 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
             }
         }
 
+        @Override
+        public void close() {
+            this.blobkeys.close();
+        }
     }
 
     public synchronized Iterator<Map.Entry<byte[], Map<String, String>>> entries(final boolean up, final boolean rotating) throws IOException {

source/net/yacy/kelondro/data/word/Word.java

@@ -56,7 +56,7 @@ public class Word {
      */
     public static final int commonHashLength = 12;
 
-    private static final int hashCacheSize = Math.max(200000, Math.min(10000000, (int) (MemoryControl.available() / 20000L)));
+    private static final int hashCacheSize = Math.max(100000, Math.min(1000000, (int) (MemoryControl.available() / 40000L)));
     private static ARC<String, byte[]> hashCache = null;
     static {
         try {

source/net/yacy/kelondro/index/RowSet.java

@@ -411,6 +411,10 @@ public class RowSet extends RowCollection implements Index, Iterable<Row.Entry>,
         public final void remove() {
             throw new UnsupportedOperationException();
         }
+
+        @Override
+        public void close() {
+        }
     }
 
     @Override
@@ -478,6 +482,10 @@ public class RowSet extends RowCollection implements Index, Iterable<Row.Entry>,
         public final void remove() {
             throw new UnsupportedOperationException();
         }
+
+        @Override
+        public void close() {
+        }
     }
 
     /**

source/net/yacy/kelondro/order/Digest.java

@@ -56,7 +56,7 @@ public class Digest {
 
 	public static BlockingQueue<MessageDigest> digestPool = new LinkedBlockingDeque<MessageDigest>();
 
-    private static final int md5CacheSize = Math.max(200000, Math.min(10000000, (int) (MemoryControl.available() / 20000L)));
+    private static final int md5CacheSize = Math.max(100000, Math.min(1000000, (int) (MemoryControl.available() / 40000L)));
     private static ARC<String, byte[]> md5Cache = null;
     static {
         try {

source/net/yacy/kelondro/order/MergeIterator.java

@@ -61,6 +61,12 @@ public class MergeIterator<E> implements CloneableIterator<E> {
         nextb();
     }
 
+    @Override
+    public void close() {
+        this.a.close();
+        this.b.close();
+    }
+
     @Override
     public MergeIterator<E> clone(final Object modifier) {
         assert this.a != null;
@@ -159,6 +165,9 @@ public class MergeIterator<E> implements CloneableIterator<E> {
             public CloneableIterator<A> clone(Object modifier) {
                 return this;
             }
+            @Override
+            public void close() {
+            }
         };
         return cascade(iterators.iterator(), c, merger, up);
     }
@@ -180,6 +189,9 @@ public class MergeIterator<E> implements CloneableIterator<E> {
             public CloneableIterator<A> clone(Object modifier) {
                 return this;
             }
+            @Override
+            public void close() {
+            }
         };
         final CloneableIterator<A> one = iiterators.next();
         if (!(iiterators.hasNext())) return one;

source/net/yacy/kelondro/order/RotateIterator.java

@@ -45,28 +45,38 @@ public class RotateIterator<E> implements CloneableIterator<E> {
         this.nempty = this.clone.hasNext();
     }
 
+	@Override
     public RotateIterator<E> clone(final Object modifier) {
-        return new RotateIterator<E>(a, modifier, terminationCount - 1);
+        return new RotateIterator<E>(this.a, modifier, this.terminationCount - 1);
     }
 
+    @Override
     public boolean hasNext() {
-        return (terminationCount > 0) && (this.nempty);
+        return (this.terminationCount > 0) && (this.nempty);
     }
 
+    @Override
     public E next() {
     	// attention: this iterator has no termination - on purpose.
     	// it must be taken care that a calling method has a termination predicate different
     	// from the hasNext() method
-        if (!(a.hasNext())) {
-            a = clone.clone(modifier);
-            assert a.hasNext();
+        if (!(this.a.hasNext())) {
+            this.a = this.clone.clone(this.modifier);
+            assert this.a.hasNext();
         }
-        terminationCount--;
-        return a.next();
+        this.terminationCount--;
+        return this.a.next();
     }
 
+    @Override
     public void remove() {
-        a.remove();
+        this.a.remove();
+    }
+
+    @Override
+    public void close() {
+        this.a.close();
+        this.clone.close();
     }
 
 }

source/net/yacy/kelondro/order/StackIterator.java

@@ -111,6 +111,9 @@ public class StackIterator<E> implements CloneableIterator<E> {
             public CloneableIterator<A> clone(Object modifier) {
                 return null;
             }
+            @Override
+            public void close() {
+            }
         };
         if (iterators.length == 1) {
             return iterators[0];
@@ -126,4 +129,10 @@ public class StackIterator<E> implements CloneableIterator<E> {
         if (a == null) return stack(iterators0);
         return new StackIterator<A>(a, stack(iterators0));
     }
+
+    @Override
+    public void close() {
+        this.a.close();
+        this.b.close();
+    }
 }

source/net/yacy/kelondro/rwi/ReferenceContainerArray.java

@@ -194,6 +194,11 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
             return this;
         }
 
+        @Override
+        public void close() {
+            this.iterator.close();
+        }
+
     }
 
     /**
@@ -276,6 +281,11 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
             return this;
         }
 
+        @Override
+        public void close() {
+            this.iterator.close();
+        }
+
     }
 
     /**
@@ -472,6 +482,7 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
                     lastlog = System.currentTimeMillis();
                 }
             }
+            ei.close();
         }
         references.trim();
         System.out.println("CELL REFERENCE COLLECTION finished");

source/net/yacy/kelondro/rwi/ReferenceContainerCache.java

@@ -319,6 +319,9 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
             return this;
         }
 
+        @Override
+        public void close() {
+        }
     }
 
     @Override
@@ -401,6 +404,9 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
             return this;
         }
 
+        @Override
+        public void close() {
+        }
     }
 
     /**

source/net/yacy/kelondro/table/Table.java

@@ -809,6 +809,7 @@ public class Table implements Index, Iterable<Row.Entry> {
         return this.rowdef.newEntry(b);
     }
 
+    @Override
     public synchronized Entry removeOne() throws IOException {
         assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size();
         assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size();
@@ -835,6 +836,7 @@ public class Table implements Index, Iterable<Row.Entry> {
         return lr;
     }
 
+    @Override
     public List<Row.Entry> top(int count) throws IOException {
         final ArrayList<Row.Entry> list = new ArrayList<Row.Entry>();
         if ((this.file == null) || (this.index == null)) return list;
@@ -849,6 +851,7 @@ public class Table implements Index, Iterable<Row.Entry> {
         return list;
     }
 
+    @Override
     public synchronized void clear() throws IOException {
         final File f = this.file.filename();
         this.file.close();
@@ -873,19 +876,23 @@ public class Table implements Index, Iterable<Row.Entry> {
         this.index.clear();
     }
 
+    @Override
     public Row row() {
         return this.rowdef;
     }
 
+    @Override
     public int size() {
         if (this.index == null) return 0;
         return this.index.size();
     }
 
+    @Override
     public boolean isEmpty() {
         return this.index.isEmpty();
     }
 
+    @Override
     public Iterator<Entry> iterator() {
         try {
             return rows();
@@ -894,6 +901,7 @@ public class Table implements Index, Iterable<Row.Entry> {
         }
     }
 
+    @Override
     public synchronized CloneableIterator<Entry> rows() throws IOException {
         this.file.flushBuffer();
         return new rowIteratorNoOrder();
@@ -910,14 +918,17 @@ public class Table implements Index, Iterable<Row.Entry> {
             this.i = Table.this.index.iterator();
         }
 
+        @Override
         public CloneableIterator<Entry> clone(final Object modifier) {
             return new rowIteratorNoOrder();
         }
 
+        @Override
         public boolean hasNext() {
             return this.i != null && this.i.hasNext();
         }
 
+        @Override
         public Entry next() {
             final Row.Entry entry = this.i.next();
             if (entry == null) return null;
@@ -931,6 +942,7 @@ public class Table implements Index, Iterable<Row.Entry> {
             }
         }
 
+        @Override
         public void remove() {
             if (this.key != null) {
                 try {
@@ -942,14 +954,22 @@ public class Table implements Index, Iterable<Row.Entry> {
             }
         }
 
+        @Override
+        public void close() {
+            if (this.i instanceof CloneableIterator) {
+                ((CloneableIterator<Entry>) this.i).close();
+            }
+        }
+
     }
 
+    @Override
     public synchronized CloneableIterator<Entry> rows(final boolean up, final byte[] firstKey) throws IOException {
         return new rowIterator(up, firstKey);
     }
 
     private class rowIterator implements CloneableIterator<Entry> {
-        private final Iterator<byte[]> i;
+        private final CloneableIterator<byte[]> i;
         private final boolean up;
         private final byte[] fk;
         private int c;
@@ -961,14 +981,17 @@ public class Table implements Index, Iterable<Row.Entry> {
             this.c = -1;
         }
 
+        @Override
         public CloneableIterator<Entry> clone(final Object modifier) {
             return new rowIterator(this.up, this.fk);
         }
 
+        @Override
         public boolean hasNext() {
             return this.i.hasNext();
         }
 
+        @Override
         public Entry next() {
             final byte[] k = this.i.next();
             assert k != null;
@@ -995,10 +1018,15 @@ public class Table implements Index, Iterable<Row.Entry> {
             return Table.this.rowdef.newEntry(b);
         }
 
+        @Override
         public void remove() {
             throw new UnsupportedOperationException("no remove in Table.rowIterator");
         }
 
+        @Override
+        public void close() {
+            this.i.close();
+        }
     }
 
     private static byte[] testWord(final char c) {
@@ -1125,6 +1153,7 @@ public class Table implements Index, Iterable<Row.Entry> {
         */
     }
 
+    @Override
     public void deleteOnExit() {
         this.file.deleteOnExit();
     }

source/net/yacy/search/index/MetadataRepository.java

@@ -281,7 +281,7 @@ public final class MetadataRepository implements /*Metadata,*/ Iterable<byte[]>
 
     public class kiter implements CloneableIterator<URIMetadataRow> {
         // enumerates entry elements
-        private final Iterator<Row.Entry> iter;
+        private final CloneableIterator<Row.Entry> iter;
         private final boolean error;
         boolean up;
 
@@ -326,6 +326,11 @@ public final class MetadataRepository implements /*Metadata,*/ Iterable<byte[]>
         public final void remove() {
             this.iter.remove();
         }
+
+        @Override
+        public void close() {
+            this.iter.close();
+        }
     }
 
 

source/net/yacy/search/ranking/BlockRank.java

@@ -144,6 +144,7 @@ public class BlockRank {
                 final ReferenceContainer<HostReference> references = ri.next();
                 index.add(references);
             }
+            ri.close();
         } catch (final IOException e) {
             Log.logException(e);
         } catch (final RowSpaceExceededException e) {