- less restrictions for usage of Table RAM copy

- new limit to use the table copy (instead of flag): 400MB available. If
less is available, then a copy is never used. If more is available, then
it can be used if there is a remaining space of at least 200MB
- flush caches more often: flush the Digest cache

defaults/yacy.init

@@ -623,10 +623,11 @@ cleanup.failedSearchURLtimeout = 86400000
 javastart_Xmx=Xmx600m
 javastart_Xms=Xms600m
 
-# YaCy is able to use RAM copies of database tables. This needs a lot of RAM
-# To switch copying of file tables int RAM on, use this property
-# this value is automatically set to true, if more than one gigabyte RAM is available
-ramcopy=false
+# YaCy is able to use RAM copies of database tables. This needs a lot of RAM.
+# To switch on copying of file tables int RAM, there must be enough memory
+# The memory that is available at startup time is used to switch the feature on
+# The tableCachingLimit is the amount of free RAM at startup time to switch on the feature
+tableCachingLimit=419430400
 
 # some java versions may be limited to a specific array size
 # of 134217727 entries. To prevent that tables of that size are generated,

source/net/yacy/kelondro/order/Digest.java

@@ -60,12 +60,19 @@ public class Digest {
     private static ARC<String, byte[]> md5Cache = null;
     static {
         try {
-            md5Cache = new ConcurrentARC<String, byte[]>(md5CacheSize, Math.max(32, 4 * Runtime.getRuntime().availableProcessors()));
+            md5Cache = new ConcurrentARC<String, byte[]>(md5CacheSize, Math.max(8, 2 * Runtime.getRuntime().availableProcessors()));
         } catch (final OutOfMemoryError e) {
-            md5Cache = new ConcurrentARC<String, byte[]>(1000, Math.max(8, 2 * Runtime.getRuntime().availableProcessors()));
+            md5Cache = new ConcurrentARC<String, byte[]>(1000, Math.max(2, Runtime.getRuntime().availableProcessors()));
         }
     }
 
+    /**
+     * clean the md5 cache
+     */
+    public static void cleanup() {
+    	md5Cache.clear();
+    }
+    
     public static String encodeHex(final long in, final int length) {
         String s = Long.toHexString(in);
         while (s.length() < length) s = "0" + s;
@@ -114,7 +121,7 @@ public class Digest {
         // generate a hex representation from the md5 of a byte-array
         return encodeHex(encodeMD5Raw(b));
     }
-
+    
     public static byte[] encodeMD5Raw(final String key) {
 
         byte[] h = md5Cache.get(key);

source/net/yacy/kelondro/table/Table.java

@@ -93,7 +93,7 @@ public class Table implements Index, Iterable<Row.Entry> {
 
         this.rowdef = rowdef;
         this.buffersize = buffersize;
-        this.minmemremaining = Math.max(400 * 1024 * 1024, MemoryControl.available() / 10);
+        this.minmemremaining = Math.max(200L * 1024L * 1024L, MemoryControl.available() / 10);
         //this.fail = 0;
         // define the taildef, a row like the rowdef but without the first column
         final Column[] cols = new Column[rowdef.columns() - 1];
@@ -123,19 +123,18 @@ public class Table implements Index, Iterable<Row.Entry> {
 
             // initialize index and copy table
             final int  records = Math.max(fileSize, initialSpace);
-            final long neededRAM4table = (records) * ((rowdef.objectsize) + 4L) * 3L;
-            this.table = ((exceed134217727 || neededRAM4table < maxarraylength) &&
-                     (useTailCache && MemoryControl.available() > neededRAM4table + 200 * 1024 * 1024)) ?
+            final long neededRAM4table = 200L * 1024L * 1024L + records * (rowdef.objectsize + rowdef.primaryKeyLength) * 3L / 2L;
+            this.table = ((exceed134217727 || neededRAM4table < maxarraylength) && useTailCache && MemoryControl.available() > neededRAM4table) ?
                     new RowSet(this.taildef, records) : null;
             Log.logInfo("TABLE", "initialization of " + tablefile.getName() + ". table copy: " + ((this.table == null) ? "no" : "yes") + ", available RAM: " + (MemoryControl.available() / 1024 / 1024) + "MB, needed: " + (neededRAM4table/1024/1024 + 200) + "MB, allocating space for " + records + " entries");
-            final long neededRAM4index = 400 * 1024 * 1024 + records * (rowdef.primaryKeyLength + 4) * 3 / 2;
-            if (!MemoryControl.request(neededRAM4index, false)) {
+            final long neededRAM4index = 200L * 1024L * 1024L + records * rowdef.primaryKeyLength * 3L / 2L;
+            if (!MemoryControl.request(neededRAM4index, true)) {
                 // despite calculations seemed to show that there is enough memory for the table AND the index
                 // there is now not enough memory left for the index. So delete the table again to free the memory
                 // for the index
-                Log.logSevere("TABLE", tablefile.getName() + ": not enough RAM (" + (MemoryControl.available() / 1024 / 1024) + "MB) left for index, deleting allocated table space to enable index space allocation (needed: " + (neededRAM4index / 1024 / 1024) + "MB)");
+                Log.logSevere("TABLE", tablefile.getName() + ": not enough RAM (" + (MemoryControl.available() / 1024L / 1024L) + "MB) left for index, deleting allocated table space to enable index space allocation (needed: " + (neededRAM4index / 1024 / 1024) + "MB)");
                 this.table = null; System.gc();
-                Log.logSevere("TABLE", tablefile.getName() + ": RAM after releasing the table: " + (MemoryControl.available() / 1024 / 1024) + "MB");
+                Log.logSevere("TABLE", tablefile.getName() + ": RAM after releasing the table: " + (MemoryControl.available() / 1024L / 1024L) + "MB");
             }
             this.index = new HandleMap(rowdef.primaryKeyLength, rowdef.objectOrder, 4, records, tablefile.getAbsolutePath());
             final HandleMap errors = new HandleMap(rowdef.primaryKeyLength, NaturalOrder.naturalOrder, 4, records, tablefile.getAbsolutePath() + ".errors");
@@ -178,14 +177,14 @@ public class Table implements Index, Iterable<Row.Entry> {
                             this.table = null;
                             break;
                         }
-                        if (abandonTable()) {
-                            this.table = null;
-                            break;
-                        }
                     } else {
                         errors.putUnique(key, i++);
                     }
                 }
+                Runtime.getRuntime().gc();
+                if (abandonTable()) {
+                    this.table = null;
+                }
             }
             this.index.trim();
 

source/net/yacy/search/Switchboard.java

@@ -302,12 +302,12 @@ public final class Switchboard extends serverSwitch
         initRemoteProxy();
 
         // memory configuration
-        this.useTailCache = getConfigBool("ramcopy", true);
-        if ( MemoryControl.available() > 1024 * 1024 * 1024 * 1 ) {
+        long tableCachingLimit = getConfigLong("tableCachingLimit", 419430400L);
+        if ( MemoryControl.available() > tableCachingLimit ) {
             this.useTailCache = true;
         }
         this.exceed134217727 = getConfigBool("exceed134217727", true);
-        if ( MemoryControl.available() > 1024 * 1024 * 1024 * 2 ) {
+        if ( MemoryControl.available() > 1024L * 1024L * 1024L * 2L ) {
             this.exceed134217727 = true;
         }
 
@@ -1911,6 +1911,7 @@ public final class Switchboard extends serverSwitch
         try {
         	// flush the document compressor cache
         	Cache.commit();
+        	Digest.cleanup(); // don't let caches become permanent memory leaks
         	
             // clear caches if necessary
             if ( !MemoryControl.request(8000000L, false) ) {

source/net/yacy/search/index/MetadataRepository.java

@@ -97,6 +97,7 @@ public final class MetadataRepository implements /*Metadata,*/ Iterable<byte[]>
 
     public void clearCache() {
         if (this.urlIndexFile instanceof Cache) ((Cache) this.urlIndexFile).clearCache();
+        if (this.statsDump != null) this.statsDump.clear();
         this.statsDump = null;
     }