diff --git a/defaults/yacy.init b/defaults/yacy.init index 6bb101e7c..671d72a7d 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -623,10 +623,11 @@ cleanup.failedSearchURLtimeout = 86400000 javastart_Xmx=Xmx600m javastart_Xms=Xms600m -# YaCy is able to use RAM copies of database tables. This needs a lot of RAM -# To switch copying of file tables int RAM on, use this property -# this value is automatically set to true, if more than one gigabyte RAM is available -ramcopy=false +# YaCy is able to use RAM copies of database tables. This needs a lot of RAM. +# To switch on copying of file tables int RAM, there must be enough memory +# The memory that is available at startup time is used to switch the feature on +# The tableCachingLimit is the amount of free RAM at startup time to switch on the feature +tableCachingLimit=419430400 # some java versions may be limited to a specific array size # of 134217727 entries. To prevent that tables of that size are generated, diff --git a/source/net/yacy/kelondro/order/Digest.java b/source/net/yacy/kelondro/order/Digest.java index d1efe21d7..b088a0889 100644 --- a/source/net/yacy/kelondro/order/Digest.java +++ b/source/net/yacy/kelondro/order/Digest.java @@ -60,12 +60,19 @@ public class Digest { private static ARC md5Cache = null; static { try { - md5Cache = new ConcurrentARC(md5CacheSize, Math.max(32, 4 * Runtime.getRuntime().availableProcessors())); + md5Cache = new ConcurrentARC(md5CacheSize, Math.max(8, 2 * Runtime.getRuntime().availableProcessors())); } catch (final OutOfMemoryError e) { - md5Cache = new ConcurrentARC(1000, Math.max(8, 2 * Runtime.getRuntime().availableProcessors())); + md5Cache = new ConcurrentARC(1000, Math.max(2, Runtime.getRuntime().availableProcessors())); } } + /** + * clean the md5 cache + */ + public static void cleanup() { + md5Cache.clear(); + } + public static String encodeHex(final long in, final int length) { String s = Long.toHexString(in); while (s.length() < length) s = "0" + s; @@ -114,7 +121,7 @@ public class Digest { // generate a hex representation from the md5 of a byte-array return encodeHex(encodeMD5Raw(b)); } - + public static byte[] encodeMD5Raw(final String key) { byte[] h = md5Cache.get(key); diff --git a/source/net/yacy/kelondro/table/Table.java b/source/net/yacy/kelondro/table/Table.java index b2483be11..8ea1f6cbc 100644 --- a/source/net/yacy/kelondro/table/Table.java +++ b/source/net/yacy/kelondro/table/Table.java @@ -93,7 +93,7 @@ public class Table implements Index, Iterable { this.rowdef = rowdef; this.buffersize = buffersize; - this.minmemremaining = Math.max(400 * 1024 * 1024, MemoryControl.available() / 10); + this.minmemremaining = Math.max(200L * 1024L * 1024L, MemoryControl.available() / 10); //this.fail = 0; // define the taildef, a row like the rowdef but without the first column final Column[] cols = new Column[rowdef.columns() - 1]; @@ -123,19 +123,18 @@ public class Table implements Index, Iterable { // initialize index and copy table final int records = Math.max(fileSize, initialSpace); - final long neededRAM4table = (records) * ((rowdef.objectsize) + 4L) * 3L; - this.table = ((exceed134217727 || neededRAM4table < maxarraylength) && - (useTailCache && MemoryControl.available() > neededRAM4table + 200 * 1024 * 1024)) ? + final long neededRAM4table = 200L * 1024L * 1024L + records * (rowdef.objectsize + rowdef.primaryKeyLength) * 3L / 2L; + this.table = ((exceed134217727 || neededRAM4table < maxarraylength) && useTailCache && MemoryControl.available() > neededRAM4table) ? new RowSet(this.taildef, records) : null; Log.logInfo("TABLE", "initialization of " + tablefile.getName() + ". table copy: " + ((this.table == null) ? "no" : "yes") + ", available RAM: " + (MemoryControl.available() / 1024 / 1024) + "MB, needed: " + (neededRAM4table/1024/1024 + 200) + "MB, allocating space for " + records + " entries"); - final long neededRAM4index = 400 * 1024 * 1024 + records * (rowdef.primaryKeyLength + 4) * 3 / 2; - if (!MemoryControl.request(neededRAM4index, false)) { + final long neededRAM4index = 200L * 1024L * 1024L + records * rowdef.primaryKeyLength * 3L / 2L; + if (!MemoryControl.request(neededRAM4index, true)) { // despite calculations seemed to show that there is enough memory for the table AND the index // there is now not enough memory left for the index. So delete the table again to free the memory // for the index - Log.logSevere("TABLE", tablefile.getName() + ": not enough RAM (" + (MemoryControl.available() / 1024 / 1024) + "MB) left for index, deleting allocated table space to enable index space allocation (needed: " + (neededRAM4index / 1024 / 1024) + "MB)"); + Log.logSevere("TABLE", tablefile.getName() + ": not enough RAM (" + (MemoryControl.available() / 1024L / 1024L) + "MB) left for index, deleting allocated table space to enable index space allocation (needed: " + (neededRAM4index / 1024 / 1024) + "MB)"); this.table = null; System.gc(); - Log.logSevere("TABLE", tablefile.getName() + ": RAM after releasing the table: " + (MemoryControl.available() / 1024 / 1024) + "MB"); + Log.logSevere("TABLE", tablefile.getName() + ": RAM after releasing the table: " + (MemoryControl.available() / 1024L / 1024L) + "MB"); } this.index = new HandleMap(rowdef.primaryKeyLength, rowdef.objectOrder, 4, records, tablefile.getAbsolutePath()); final HandleMap errors = new HandleMap(rowdef.primaryKeyLength, NaturalOrder.naturalOrder, 4, records, tablefile.getAbsolutePath() + ".errors"); @@ -178,14 +177,14 @@ public class Table implements Index, Iterable { this.table = null; break; } - if (abandonTable()) { - this.table = null; - break; - } } else { errors.putUnique(key, i++); } } + Runtime.getRuntime().gc(); + if (abandonTable()) { + this.table = null; + } } this.index.trim(); diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 1f5b4ee88..ba26ce6e7 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -302,12 +302,12 @@ public final class Switchboard extends serverSwitch initRemoteProxy(); // memory configuration - this.useTailCache = getConfigBool("ramcopy", true); - if ( MemoryControl.available() > 1024 * 1024 * 1024 * 1 ) { + long tableCachingLimit = getConfigLong("tableCachingLimit", 419430400L); + if ( MemoryControl.available() > tableCachingLimit ) { this.useTailCache = true; } this.exceed134217727 = getConfigBool("exceed134217727", true); - if ( MemoryControl.available() > 1024 * 1024 * 1024 * 2 ) { + if ( MemoryControl.available() > 1024L * 1024L * 1024L * 2L ) { this.exceed134217727 = true; } @@ -1911,6 +1911,7 @@ public final class Switchboard extends serverSwitch try { // flush the document compressor cache Cache.commit(); + Digest.cleanup(); // don't let caches become permanent memory leaks // clear caches if necessary if ( !MemoryControl.request(8000000L, false) ) { diff --git a/source/net/yacy/search/index/MetadataRepository.java b/source/net/yacy/search/index/MetadataRepository.java index e2e03ad5f..61f6c82d6 100644 --- a/source/net/yacy/search/index/MetadataRepository.java +++ b/source/net/yacy/search/index/MetadataRepository.java @@ -97,6 +97,7 @@ public final class MetadataRepository implements /*Metadata,*/ Iterable public void clearCache() { if (this.urlIndexFile instanceof Cache) ((Cache) this.urlIndexFile).clearCache(); + if (this.statsDump != null) this.statsDump.clear(); this.statsDump = null; }