- enhanced concurrency in database index access for multicore

- added statistics about database index caches in PerformanceMemory_p.html
- adoped many classes to use the new statistics
- added missing close statements

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7018 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/PerformanceMemory_p.html

@@ -99,6 +99,30 @@
       </tr>
     </table>
     
+    <p><strong>Object Index Caches:</strong></p>
+    <table border="0" cellpadding="2" cellspacing="1">
+      <tr class="TableHeader" valign="bottom">
+        <td>Table</td>
+        <td>Size</td>
+        <td>Chunk Size</td>
+        <td>Needed Memory</td>
+        <td>Used Memory</td>
+        <!-- <td>Flushes</td> -->
+      </tr>
+      #{indexcache}#
+      <tr class="TableCellLight">
+        <td align="left" class="TableCellDark">#[Name]#</td>
+        <td align="right">#[Count]#</td>
+        <td align="right">#[ChunkSize]#</td>
+        <td align="right">#[NeededMem]#</td>
+        <td align="right">#[UsedMem]#</td>
+      </tr>
+      #{/indexcache}#
+      <tr class="TableCellDark">
+        <td colspan="9">Total Mem: #[indexcacheTotalMem]# MB</td>
+      </tr>
+    </table>
+    
     <p><strong>Object Read Caches:</strong></p>
     <table border="0" cellpadding="2" cellspacing="1">
       <tr class="TableHeader" valign="bottom">

htroot/PerformanceMemory_p.java

@@ -30,6 +30,7 @@ import java.util.Iterator;
 import java.util.Map;
 
 import net.yacy.kelondro.index.Cache;
+import net.yacy.kelondro.index.ObjectIndexCache;
 import net.yacy.kelondro.table.Table;
 import net.yacy.kelondro.util.Domains;
 import net.yacy.kelondro.util.FileUtils;
@@ -116,10 +117,36 @@ public class PerformanceMemory_p {
         prop.put("EcoList", c);
         prop.putNum("EcoIndexTotalMem", totalmem / (1024 * 1024d));
         
+        // write object cache table
+        Iterator<Map.Entry<String, ObjectIndexCache>> oi = ObjectIndexCache.objects();
+        c = 0;
+        mem = 0;
+        Map.Entry<String, ObjectIndexCache> oie;
+        ObjectIndexCache cache;
+        long hitmem, totalhitmem = 0;
+        while (oi.hasNext()) {
+            oie = oi.next();
+            filename = oie.getKey();
+            cache = oie.getValue();
+            prop.put("indexcache_" + c + "_Name", ((p = filename.indexOf("DATA")) < 0) ? filename : filename.substring(p));
+            
+            hitmem = cache.mem();
+            totalhitmem += hitmem;
+            prop.put("indexcache_" + c + "_ChunkSize", cache.row().objectsize);
+            prop.putNum("indexcache_" + c + "_Count", cache.size());
+            prop.put("indexcache_" + c + "_NeededMem", cache.size() * cache.row().objectsize);
+            prop.put("indexcache_" + c + "_UsedMem", hitmem);
+            
+            c++;
+        }
+        prop.put("indexcache", c);
+        prop.putNum("indexcacheTotalMem", totalhitmem / (1024 * 1024d));
+        
         // write object cache table
         i = Cache.filenames();
         c = 0;
-        long hitmem, missmem, totalhitmem = 0, totalmissmem = 0;
+        long missmem, totalmissmem = 0;
+        totalhitmem = 0;
         while (i.hasNext()) {
             filename = i.next();
             map = Cache.memoryStats(filename);

source/de/anomic/crawler/Balancer.java

@@ -498,7 +498,9 @@ public class Balancer {
         Request request;
     	while (i.hasNext()) {
     	    handle = i.next();
-    	    request = new Request(this.urlFileIndex.get(handle));
+    	    Row.Entry entry = this.urlFileIndex.get(handle);
+    	    if (entry == null) continue;
+    	    request = new Request(entry);
     	    host = request.url().getHost();
     		try {
                 pushHashToDomainStacks(host, handle);

source/de/anomic/data/URLAnalysis.java

@@ -408,6 +408,7 @@ public class URLAnalysis {
             System.out.println("INDEX REFERENCE COLLECTION starting dump of statistics");
             idx.dump(new File(statisticPath));
             System.out.println("INDEX REFERENCE COLLECTION finished dump, wrote " + idx.size() + " entries to " + statisticPath);
+            idx.close();
         } catch (Exception e) {
             Log.logException(e);
         }
@@ -433,6 +434,7 @@ public class URLAnalysis {
                 update = System.currentTimeMillis();
             }
         }
+        idx.close();
         mr.close();
         System.out.println("INDEX DIFF URL-COL finished diff, starting dump to " + diffFile);
         count = hs.dump(new File(diffFile));

source/de/anomic/search/blockrank/CRProcess.java

@@ -217,7 +217,7 @@ public class CRProcess {
             } else {
                 // initialize counters and dates
                 acc_entry = acc.row().newEntry();
-                acc_entry.setCol("Referee", key, null);
+                acc_entry.setCol(0, key, null);
                 for (int i = 1; i < acc.row().columns(); i++) {
                     acc_entry.setCol(i, new_entry.getAttr(acc.row().column(i).nickname, 0));
                 }

source/de/anomic/yacy/dht/FlatWordPartitionScheme.java

@@ -111,7 +111,7 @@ public class FlatWordPartitionScheme implements PartitionScheme {
 
         System.gc(); // for resource measurement
         long a = MemoryControl.available();
-        HandleMap idx = new HandleMap(12, Base64Order.enhancedCoder, 4, 150000);
+        HandleMap idx = new HandleMap(12, Base64Order.enhancedCoder, 4, 150000, "test");
         for (int i = 0; i < count; i++) {
             try {
                 idx.inc(FlatWordPartitionScheme.positionToHash(r.nextInt(count)));
@@ -126,7 +126,7 @@ public class FlatWordPartitionScheme implements PartitionScheme {
         long memk = a - MemoryControl.available();
         System.out.println("Used Memory: " + memk + " bytes");
         System.out.println("x " + idx.get(FlatWordPartitionScheme.positionToHash(0)));
-        idx = null;
+        idx.close();
         
         r = new Random(0);
         start = System.currentTimeMillis();

source/net/yacy/document/parser/html/TransformerWriter.java

@@ -245,7 +245,7 @@ public final class TransformerWriter extends Writer {
 					} catch (IOException e) {
 					    Log.logException(e);
 					}
-                    filterCont = new CharBuffer();
+                    if (filterCont == null) filterCont = new CharBuffer(); else filterCont.reset();
                     return new char[0];
                 } else {
                      // we ignore that thing and return it again

source/net/yacy/kelondro/blob/HeapReader.java

@@ -189,7 +189,7 @@ public class HeapReader {
         Log.logInfo("HeapReader", "generating index for " + heapFile.toString() + ", " + (file.length() / 1024 / 1024) + " MB. Please wait.");
         
         this.free = new Gap();
-        HandleMap.initDataConsumer indexready = HandleMap.asynchronusInitializer(keylength, this.ordering, 8, Math.max(10, (int) (Runtime.getRuntime().freeMemory() / (10 * 1024 * 1024))));
+        HandleMap.initDataConsumer indexready = HandleMap.asynchronusInitializer(this.name() + ".initializer", keylength, this.ordering, 8, Math.max(10, (int) (Runtime.getRuntime().freeMemory() / (10 * 1024 * 1024))));
         byte[] key = new byte[keylength];
         int reclen;
         long seek = 0;
@@ -248,7 +248,6 @@ public class HeapReader {
             Log.logException(e);
         }
         Log.logInfo("HeapReader", "finished index generation for " + heapFile.toString() + ", " + index.size() + " entries, " + free.size() + " gaps.");
-        
     }
     
     private void mergeFreeEntries() throws IOException {

source/net/yacy/kelondro/blob/HeapWriter.java

@@ -80,7 +80,7 @@ public final class HeapWriter {
         this.heapFileTMP = temporaryHeapFile;
         this.heapFileREADY = readyHeapFile;
         this.keylength = keylength;
-        this.index = new HandleMap(keylength, ordering, 8, 100000);
+        this.index = new HandleMap(keylength, ordering, 8, 100000, readyHeapFile.getAbsolutePath());
         this.os = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(temporaryHeapFile), outBuffer));
         this.seek = 0;
     }

source/net/yacy/kelondro/index/BufferedObjectIndex.java

@@ -72,6 +72,10 @@ public class BufferedObjectIndex implements ObjectIndex, Iterable<Row.Entry> {
         }
     }
     
+    public long mem() {
+        return this.backend.mem() + this.buffer.mem();
+    }
+    
     /**
      * check size of buffer in such a way that a put into the buffer is possible
      * afterwards without exceeding the given maximal buffersize

source/net/yacy/kelondro/index/Cache.java

@@ -98,6 +98,10 @@ public final class Cache implements ObjectIndex, Iterable<Row.Entry> {
         this.hasnotDelete = 0;
     }
     
+    public long mem() {
+        return this.index.mem() + this.readHitCache.mem() + this.readMissCache.mem();
+    }
+    
     public final int writeBufferSize() {
         return 0;
     }

source/net/yacy/kelondro/index/HandleMap.java

@@ -63,9 +63,9 @@ public final class HandleMap implements Iterable<Row.Entry> {
      * @param objectOrder
      * @param space
      */
-    public HandleMap(final int keylength, final ByteOrder objectOrder, final int idxbytes, final int expectedspace) {
+    public HandleMap(final int keylength, final ByteOrder objectOrder, final int idxbytes, final int expectedspace, String name) {
         this.rowdef = new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, keylength, "key"), new Column("long c-" + idxbytes + " {b256}")}, objectOrder);
-        this.index = new RowSetArray(rowdef, spread(expectedspace));
+        this.index = new RowSetArray(name, rowdef, spread(expectedspace));
     }
 
     /**
@@ -77,7 +77,7 @@ public final class HandleMap implements Iterable<Row.Entry> {
      * @throws RowSpaceExceededException 
      */
     public HandleMap(final int keylength, final ByteOrder objectOrder, final int idxbytes, final File file) throws IOException, RowSpaceExceededException {
-        this(keylength, objectOrder, idxbytes, (int) (file.length() / (keylength + idxbytes)));
+        this(keylength, objectOrder, idxbytes, (int) (file.length() / (keylength + idxbytes)), file.getAbsolutePath());
         // read the index dump and fill the index
         InputStream is = new BufferedInputStream(new FileInputStream(file), 1024 * 1024);
         if (file.getName().endsWith(".gz")) is = new GZIPInputStream(is);
@@ -95,8 +95,12 @@ public final class HandleMap implements Iterable<Row.Entry> {
         assert this.index.size() == file.length() / (keylength + idxbytes);
     }
     
+    public long mem() {
+        return index.mem();
+    }
+    
     private static final int spread(int expectedspace) {
-        return Math.min(Runtime.getRuntime().availableProcessors() * 2, Math.max(1, expectedspace / 3000));
+        return Math.min(Runtime.getRuntime().availableProcessors(), Math.max(1, expectedspace / 3000));
     }
     
     public final int[] saturation() {
@@ -304,8 +308,8 @@ public final class HandleMap implements Iterable<Row.Entry> {
      * @param bufferSize
      * @return
      */
-    public final static initDataConsumer asynchronusInitializer(final int keylength, final ByteOrder objectOrder, final int idxbytes, final int expectedspace) {
-        final initDataConsumer initializer = new initDataConsumer(new HandleMap(keylength, objectOrder, idxbytes, expectedspace));
+    public final static initDataConsumer asynchronusInitializer(String name, final int keylength, final ByteOrder objectOrder, final int idxbytes, final int expectedspace) {
+        final initDataConsumer initializer = new initDataConsumer(new HandleMap(keylength, objectOrder, idxbytes, expectedspace, name));
         final ExecutorService service = Executors.newSingleThreadExecutor();
         initializer.setResult(service.submit(initializer));
         service.shutdown();
@@ -389,6 +393,9 @@ public final class HandleMap implements Iterable<Row.Entry> {
             return map;
         }
         
+        public void close() {
+            this.map.close();
+        }
     }
 
 	public Iterator<Row.Entry> iterator() {

source/net/yacy/kelondro/index/IndexTest.java

@@ -113,7 +113,7 @@ public class IndexTest {
         Runtime.getRuntime().gc();
         final long freeStartKelondro = MemoryControl.available();
         HandleMap ii = null;
-        ii = new HandleMap(12, Base64Order.enhancedCoder, 4, count);
+        ii = new HandleMap(12, Base64Order.enhancedCoder, 4, count, "test");
         for (int i = 0; i < count; i++)
             try {
                 ii.putUnique(tests[i], 1);
@@ -128,7 +128,7 @@ public class IndexTest {
         for (int i = 0; i < count; i++) if (ii.get(tests[i]) != 1) bugs++;
         Runtime.getRuntime().gc();
         final long freeEndKelondro = MemoryControl.available();
-        ii.clear(); ii = null;
+        ii.clear(); ii.close(); ii = null;
         final long t7 = System.currentTimeMillis();
         System.out.println("time   for HandleMap<byte[]> test: " + (t7 - t6) + ", " + bugs + " bugs");
         System.out.println("memory for HandleMap<byte[]>: " + (freeStartKelondro - freeEndKelondro) / mb + " MB\n");

source/net/yacy/kelondro/index/ObjectIndex.java

@@ -43,6 +43,7 @@ public interface ObjectIndex extends Iterable<Row.Entry> {
 
     public String filename(); // returns a unique identified for this index; can be a real or artificial file name
     public int size();
+    public long mem();
     public boolean isEmpty();
     public Row row();
     public byte[] smallestKey();

source/net/yacy/kelondro/index/ObjectIndexCache.java

@@ -28,6 +28,8 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
 
 import net.yacy.kelondro.index.Row.Entry;
 import net.yacy.kelondro.order.CloneableIterator;
@@ -37,35 +39,39 @@ import net.yacy.kelondro.order.StackIterator;
 
 public final class ObjectIndexCache implements ObjectIndex, Iterable<Row.Entry> {
     
+    private static final TreeMap<String, ObjectIndexCache> objectTracker = new TreeMap<String, ObjectIndexCache>();
+    
+    private final String name;
     private final Row rowdef;
     private RowSet index0;
     private RowSet index1;
     private final Row.EntryComparator entryComparator;
     //private final int spread;
     
-    public ObjectIndexCache(final Row rowdef, final int expectedspace) {
+    public ObjectIndexCache(String name, final Row rowdef, final int expectedspace) {
+        this.name = name;
         this.rowdef = rowdef;
         this.entryComparator = new Row.EntryComparator(rowdef.objectOrder);
         //this.spread = Math.max(10, expectedspace / 3000);
         reset();
+        objectTracker.put(name, this);
     }
     
-    public ObjectIndexCache(final Row rowdef, final int expectedspace, final int initialspace) throws RowSpaceExceededException {
-        this.rowdef = rowdef;
-        this.entryComparator = new Row.EntryComparator(rowdef.objectOrder);
-        //this.spread = Math.max(10, expectedspace / 3000);
-        reset(initialspace);
-    }
-    
-    private ObjectIndexCache(final Row rowdef, RowSet index0, RowSet index1, Row.EntryComparator entryComparator) {
+    private ObjectIndexCache(String name, final Row rowdef, RowSet index0, RowSet index1, Row.EntryComparator entryComparator) {
+        this.name = name;
         this.rowdef = rowdef;
         this.index0 = index0;
         this.index1 = index1;
         this.entryComparator = entryComparator;
+        objectTracker.put(name, this);
+    }
+
+    public static final Iterator<Map.Entry<String, ObjectIndexCache>> objects() {
+        return objectTracker.entrySet().iterator();
     }
     
     public ObjectIndexCache clone() {
-        return new ObjectIndexCache(this.rowdef, index0.clone(), index1.clone(), entryComparator);
+        return new ObjectIndexCache(this.name + ".clone", this.rowdef, index0.clone(), index1.clone(), entryComparator);
     }
     
     public void clear() {
@@ -247,6 +253,17 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable<Row.Entry>
         return list;
     }
     
+    public long mem() {
+        if (index0 != null && index1 == null) {
+            return index0.mem();
+        }
+        if (index0 == null && index1 != null) {
+            return index1.mem();
+        }
+        assert (index0 != null && index1 != null);
+        return index0.mem() + index1.mem();
+    }
+    
     public final synchronized int size() {
         if (index0 != null && index1 == null) {
             return index0.size();
@@ -359,6 +376,7 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable<Row.Entry>
     public final synchronized void close() {
         if (index0 != null) index0.close();
         if (index1 != null) index1.close();
+        objectTracker.remove(this.name);
     }
 
 	public final String filename() {

source/net/yacy/kelondro/index/Row.java

@@ -388,14 +388,6 @@ public final class Row {
             return rowinstance[offset + colstart[column]] == 0;
         }
         
-        @Deprecated
-        public final void setCol(final String nickname, final char c) {
-            if (nickref == null) genNickRef();
-            final Object[] ref = nickref.get(nickname);
-            if (ref == null) return;
-            rowinstance[offset + ((Integer) ref[1]).intValue()] = (byte) c;
-        }
-        
         @Deprecated
         public final void setCol(final String nickname, final byte[] cell) {
             if (nickref == null) genNickRef();
@@ -448,17 +440,6 @@ public final class Row {
                 }
         }
         
-        public final void setCol(final String nick, final String cell, final String encoding) {
-            if (encoding == null)
-                setCol(nick, cell.getBytes());
-            else
-                try {
-                    setCol(nick, cell.getBytes(encoding));
-                } catch (final UnsupportedEncodingException e) {
-                    Log.logSevere("Row", "", e);
-                }
-        }
-        
         @Deprecated
         public final void setCol(final String nickname, final long cell) {
             if (nickref == null) genNickRef();
@@ -506,26 +487,6 @@ public final class Row {
             throw new kelondroException("ROW", "addCol did not find appropriate encoding");
         }
         
-        @Deprecated
-        public final byte[] getCol(final String nickname, final byte[] dflt) {
-            if (nickref == null) genNickRef();
-            final Object[] ref = nickref.get(nickname);
-            if (ref == null) return dflt;
-            final Column col = (Column) ref[0];
-            final byte[] cell = new byte[col.cellwidth];
-            System.arraycopy(rowinstance, offset + ((Integer) ref[1]).intValue(), cell, 0, cell.length);
-            return cell;
-        }
-        
-        @Deprecated
-        public final String getColString(final String nickname, final String dflt, final String encoding) {
-            if (nickref == null) genNickRef();
-            final Object[] ref = nickref.get(nickname);
-            if (ref == null) return dflt;
-            final Column col = (Column) ref[0];
-            return getColString(((Integer) ref[1]).intValue(), col.cellwidth, encoding);
-        }
-        
         public final String getColString(final int column, final String encoding) {
             return getColString(colstart[column], row[column].cellwidth, encoding);
         }
@@ -545,16 +506,6 @@ public final class Row {
             }
         }
         
-        @Deprecated
-        public final long getColLong(final String nickname, final long dflt) {
-            if (nickref == null) genNickRef();
-            final Object[] ref = nickref.get(nickname);
-            if (ref == null) return dflt;
-            final Column col = (Column) ref[0];
-            final int clstrt = ((Integer) ref[1]).intValue();
-            return getColLong(col.encoder, clstrt, col.cellwidth);
-        }
-        
         public final long getColLong(final int column) {
             // uses the column definition to choose the right encoding
             return getColLong(row[column].encoder, colstart[column], row[column].cellwidth);
@@ -582,14 +533,6 @@ public final class Row {
             throw new kelondroException("ROW", "getColLong did not find appropriate encoding");
         }
         
-        @Deprecated
-        public final byte getColByte(final String nickname, final byte dflt) {
-            if (nickref == null) genNickRef();
-            final Object[] ref = nickref.get(nickname);
-            if (ref == null) return dflt;
-            return rowinstance[offset + ((Integer) ref[1]).intValue()];
-        }
-        
         public final byte getColByte(final int column) {
             return rowinstance[offset + colstart[column]];
         }
@@ -619,13 +562,6 @@ public final class Row {
             return c;
         }
         
-        public final char[] getColChars(final int column) {
-            final int w = row[column].cellwidth;
-            final char[] c = new char[w];
-            System.arraycopy(rowinstance, offset + colstart[column], c, 0, w);
-            return c;
-        }
-        
         public final void writeToArray(final int column, final byte[] target, final int targetOffset) {
             // this method shall replace the getColBytes where possible, bacause it may reduce the number of new byte[] allocations
             assert (targetOffset + row[column].cellwidth <= target.length) : "targetOffset = " + targetOffset + ", target.length = " + target.length + ", row[column].cellwidth() = " + row[column].cellwidth;

source/net/yacy/kelondro/index/RowCollection.java

@@ -167,6 +167,14 @@ public class RowCollection implements Iterable<Row.Entry>, Cloneable {
         this.chunkcount = 0;
         this.sortBound = 0;
 	}
+	
+	/**
+	 * calculate the memory that the structure occupies in ram
+	 * @return number of bytes in use
+	 */
+	public long mem() {
+	    return this.chunkcache.length;
+	}
    
     private static final Row exportMeasureRow = exportRow(0 /* no relevance */);
 

source/net/yacy/kelondro/index/RowSetArray.java

@@ -36,19 +36,22 @@ import net.yacy.kelondro.order.StackIterator;
 
 public final class RowSetArray implements ObjectIndex, Iterable<Row.Entry>, Cloneable {
 
-    private final Row      rowdef;
+    private final String             name;
+    private final Row                rowdef;
     private final ObjectIndexCache[] array;
     
-    public RowSetArray(final Row rowdef, final int arraySize) {
+    public RowSetArray(String name, final Row rowdef, final int arraySize) {
         //assert arraySize < 100 : arraySize;
+        this.name = name;
         this.array = new ObjectIndexCache[arraySize];
         this.rowdef = rowdef;
         for (int i = 0; i < arraySize; i++) {
-            this.array[i] = new ObjectIndexCache(rowdef, 0);
+            this.array[i] = new ObjectIndexCache(name + "." + i, rowdef, 0);
         }
     }
     
-    private RowSetArray(final Row rowdef, final ObjectIndexCache[] array) {
+    private RowSetArray(String name, final Row rowdef, final ObjectIndexCache[] array) {
+        this.name = name;
         this.array = array;
         this.rowdef = rowdef;
     }
@@ -58,15 +61,15 @@ public final class RowSetArray implements ObjectIndex, Iterable<Row.Entry>, Clon
         for (int i = 0; i < this.array.length; i++) {
             a[i] = this.array[i].clone();
         }
-        return new RowSetArray(this.rowdef, a);
+        return new RowSetArray(this.name + ".clone", this.rowdef, a);
     }
     
     private final int indexFor(final byte[] key) {
-        return (int) (this.rowdef.objectOrder.cardinal(key) % ((long) array.length));
+        return (int) ((this.rowdef.objectOrder.cardinal(key) / 17) % ((long) array.length));
     }
     
     private final int indexFor(final Entry row) {
-        return (int) (this.rowdef.objectOrder.cardinal(row.bytes(), 0, row.getPrimaryKeyLength()) % ((long) array.length));
+        return (int) ((this.rowdef.objectOrder.cardinal(row.bytes(), 0, row.getPrimaryKeyLength()) / 17) % ((long) array.length));
     }
     
     public final byte[] smallestKey() {
@@ -96,7 +99,7 @@ public final class RowSetArray implements ObjectIndex, Iterable<Row.Entry>, Clon
     private final ObjectIndexCache accessArray(final int i) {
         ObjectIndexCache r = this.array[i];
         if (r == null) synchronized (this.array) {
-            r = new ObjectIndexCache(this.rowdef, 0);
+            r = new ObjectIndexCache(name + "." + i, this.rowdef, 0);
             this.array[i] = r;
         }
         return r;
@@ -114,15 +117,15 @@ public final class RowSetArray implements ObjectIndex, Iterable<Row.Entry>, Clon
 
     public final void clear() {
         synchronized (this.array) {
-            for (int i = 0; i < this.array.length; i++) {
-                if (this.array[i] != null) this.array[i].clear();
-                this.array[i] = null;
-            }
+            for (ObjectIndexCache c: this.array) if (c != null) c.clear();
         }
     }
 
     public final void close() {
         clear();
+        synchronized (this.array) {
+            for (ObjectIndexCache c: this.array) if (c != null) c.close();
+        }
     }
 
     public final void deleteOnExit() {
@@ -265,6 +268,19 @@ public final class RowSetArray implements ObjectIndex, Iterable<Row.Entry>, Clon
         return c;
     }
 
+
+    public long mem() {
+        long m = 0;
+        synchronized (this.array) {
+            for (int i = 0; i < this.array.length; i++) {
+                if (this.array[i] != null) {
+                    m += this.array[i].mem();
+                }
+            }
+        }
+        return m;
+    }
+    
     public final boolean isEmpty() {
         synchronized (this.array) {
             for (int i = 0; i < this.array.length; i++) {

source/net/yacy/kelondro/io/CharBuffer.java

@@ -123,7 +123,7 @@ public final class CharBuffer extends Writer {
 
     private void grow() {
         int newsize = buffer.length * 2 + 1;
-        if (newsize < 256) newsize = 256;
+        if (newsize < 32) newsize = 32;
         char[] tmp = new char[newsize];
         System.arraycopy(buffer, offset, tmp, 0, length);
         buffer = tmp;
@@ -173,7 +173,9 @@ public final class CharBuffer extends Writer {
     }
 
     public CharBuffer append(final String s) {
-        return append(s,0,s.length());
+        final char[] temp = new char[s.length()];
+        s.getChars(0, temp.length, temp, 0);        
+        return append(temp);
     }    
     
     public CharBuffer append(final String s, final int off, final int len) {

source/net/yacy/kelondro/rwi/ReferenceContainerArray.java

@@ -331,7 +331,7 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
                             final Row payloadrow) throws IOException, RowSpaceExceededException {
        
         System.out.println("CELL REFERENCE COLLECTION startup");
-        HandleMap references = new HandleMap(payloadrow.primaryKeyLength, termOrder, 4, 1000000);
+        HandleMap references = new HandleMap(payloadrow.primaryKeyLength, termOrder, 4, 1000000, heapLocation.getAbsolutePath());
         String[] files = heapLocation.list();
         for (String f: files) {
             if (f.length() < 22 || !f.startsWith("text.index") || !f.endsWith(".blob")) continue;

source/net/yacy/kelondro/table/SQLTable.java

@@ -102,7 +102,11 @@ public class SQLTable implements ObjectIndex, Iterable<Row.Entry> {
         }
         
     }
-
+    
+    public long mem() {
+        return 0;
+    }
+    
     public byte[] smallestKey() {
         return null;
     }

source/net/yacy/kelondro/table/SplitTable.java

@@ -80,7 +80,7 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
     private long  fileSizeLimit;
     private boolean useTailCache;
     private boolean exceed134217727;
-    
+
     public SplitTable(
             final File path, 
             final String tablename, 
@@ -109,6 +109,12 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
         init();
     }
     
+    public long mem() {
+        long m = 0;
+        for (ObjectIndex i: tables.values()) m += i.mem();
+        return m;
+    }
+    
     public final byte[] smallestKey() {
         HandleSet keysort = new HandleSet(this.rowdef.primaryKeyLength, this.rowdef.objectOrder, this.tables.size());
         for (ObjectIndex oi: this.tables.values()) try {

source/net/yacy/kelondro/table/Table.java

@@ -124,7 +124,7 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
             // initialize index and copy table
             final int  records = Math.max(fileSize, initialSpace);
             final long neededRAM4table = (records) * ((rowdef.objectsize) + 4L) * 3L;
-            table = ((exceed134217727 || neededRAM4table < maxarraylength) &&
+            this.table = ((exceed134217727 || neededRAM4table < maxarraylength) &&
                      (useTailCache && MemoryControl.available() > neededRAM4table + 200 * 1024 * 1024)) ?
                     new RowSet(taildef, records) : null;
             Log.logInfo("TABLE", "initialization of " + tablefile.getName() + ". table copy: " + ((table == null) ? "no" : "yes") + ", available RAM: " + (MemoryControl.available() / 1024 / 1024) + "MB, needed: " + (neededRAM4table/1024/1024 + 200) + "MB, allocating space for " + records + " entries");
@@ -134,18 +134,18 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
                 // there is now not enough memory left for the index. So delete the table again to free the memory
                 // for the index
                 Log.logSevere("TABLE", tablefile.getName() + ": not enough RAM (" + (MemoryControl.available() / 1024 / 1024) + "MB) left for index, deleting allocated table space to enable index space allocation (needed: " + (neededRAM4index / 1024 / 1024) + "MB)");
-                table = null; System.gc();
+                this.table = null; System.gc();
                 Log.logSevere("TABLE", tablefile.getName() + ": RAM after releasing the table: " + (MemoryControl.available() / 1024 / 1024) + "MB");
             }
-            index = new HandleMap(rowdef.primaryKeyLength, rowdef.objectOrder, 4, records);
-            HandleMap errors = new HandleMap(rowdef.primaryKeyLength, NaturalOrder.naturalOrder, 4, records);
+            this.index = new HandleMap(rowdef.primaryKeyLength, rowdef.objectOrder, 4, records, tablefile.getAbsolutePath());
+            HandleMap errors = new HandleMap(rowdef.primaryKeyLength, NaturalOrder.naturalOrder, 4, records, tablefile.getAbsolutePath() + ".errors");
             Log.logInfo("TABLE", tablefile + ": TABLE " + tablefile.toString() + " has table copy " + ((table == null) ? "DISABLED" : "ENABLED"));
 
             // read all elements from the file into the copy table
             Log.logInfo("TABLE", "initializing RAM index for TABLE " + tablefile.getName() + ", please wait.");
             int i = 0;
             byte[] key;
-            if (table == null) {
+            if (this.table == null) {
                 final Iterator<byte[]> ki = new ChunkIterator(tablefile, rowdef.objectsize, rowdef.primaryKeyLength);
                 while (ki.hasNext()) {
                     key = ki.next();
@@ -153,7 +153,7 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
                     assert key != null;
                     if (key == null) {i++; continue;}
                     if (rowdef.objectOrder.wellformed(key)) {
-                        index.putUnique(key, i++);
+                        this.index.putUnique(key, i++);
                     } else {
                         errors.putUnique(key, i++);
                     }
@@ -173,13 +173,13 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
                         index.putUnique(key, i++);
                         // write the tail into the table
                         try {
-                            table.addUnique(taildef.newEntry(record, rowdef.primaryKeyLength, true));
+                            this.table.addUnique(taildef.newEntry(record, rowdef.primaryKeyLength, true));
                         } catch (RowSpaceExceededException e) {
-                            table = null;
+                            this.table = null;
                             break;
                         }
                         if (abandonTable()) {
-                            table = null;
+                            this.table = null;
                             break;
                         }
                     } else {
@@ -201,7 +201,8 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
                 Log.logWarning("Table", "removing not well-formed entry " + idx + " with key: " + NaturalOrder.arrayList(key, 0, key.length) + ", " + errorcc++ + "/" + errorc);
                 removeInFile(idx);
             }
-            assert index.size() == this.file.size() : "index.size() = " + index.size() + ", this.file.size() = " + this.file.size();
+            errors.close();
+            assert this.index.size() == this.file.size() : "index.size() = " + index.size() + ", this.file.size() = " + this.file.size();
             
             // remove doubles
             if (!freshFile) {
@@ -215,9 +216,9 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
                     final byte[] record = new byte[rowdef.objectsize];
                     key = new byte[rowdef.primaryKeyLength];
                     for (final Long[] ds: doubles) {
-                        file.get(ds[0].intValue(), record, 0);
+                        this.file.get(ds[0].intValue(), record, 0);
                         System.arraycopy(record, 0, key, 0, rowdef.primaryKeyLength);
-                        index.putUnique(key, ds[0].intValue());
+                        this.index.putUnique(key, ds[0].intValue());
                     }
                     // then remove the other doubles by removing them from the table, but do a re-indexing while doing that
                     // first aggregate all the delete positions because the elements from the top positions must be removed first
@@ -247,6 +248,10 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
         tableTracker.put(tablefile.toString(), this);
     }
     
+    public long mem() {
+        return index.mem() + ((table == null) ? 0 : table.mem());
+    }
+    
     private boolean abandonTable() {
         // check if not enough memory is there to maintain a memory copy of the table
         return MemoryControl.available() < minmemremaining;
@@ -427,7 +432,9 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
     public void close() {
         this.file.close();
         this.file = null;
+        if (this.table != null) this.table.close();
         this.table = null;
+        if (this.index != null) this.index.close();
         this.index = null;
     }
     
@@ -783,7 +790,7 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
         
         // initialize index and copy table
         table = (table == null) ? null : new RowSet(taildef);
-        index = new HandleMap(rowdef.primaryKeyLength, rowdef.objectOrder, 4, 100000);        
+        index.clear();        
     }
 
     public Row row() {