- better memory protection in kelondro caches: computation of needed memory for cache grow

- removed excessive gc calls
- step to 16 vertical DHT partitions

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5611 6c8d7289-2bf4-0310-a012-ef5d649a1542

defaults/yacy.network.freeworld.unit

@@ -14,7 +14,7 @@ network.unit.search.time = 4
 network.unit.dht = true
 network.unit.dhtredundancy.junior = 1
 network.unit.dhtredundancy.senior = 3
-network.unit.dht.partitionExponent = 2
+network.unit.dht.partitionExponent = 4
 network.unit.remotecrawl.speed = 6
 network.unit.bootstrap.seedlist0 = http://www.yacy.net/seed.txt
 network.unit.bootstrap.seedlist1 = http://home.arcor.de/hermens/yacy/seed.txt

source/de/anomic/kelondro/blob/Cache.java

@@ -1,4 +1,4 @@
-// kelondroCache.java
+// kelondro.blob.Cache.java
 // (C) 2006 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
 // first published 26.10.2006 on http://www.anomic.de
 //
@@ -58,16 +58,16 @@ public class Cache implements ObjectIndex {
 
     // static object tracker; stores information about object cache usage
     private static final TreeMap<String, Cache> objectTracker = new TreeMap<String, Cache>();
-    private static long memStopGrow    = 12 * 1024 * 1024; // a limit for the node cache to stop growing if less than this memory amount is available
-    private static long memStartShrink =  8 * 1024 * 1024; // a limit for the node cache to start with shrinking if less than this memory amount is available
+    private static final long memStopGrow    = 40 * 1024 * 1024; // a limit for the node cache to stop growing if less than this memory amount is available
+    private static final long memStartShrink = 20 * 1024 * 1024; // a limit for the node cache to start with shrinking if less than this memory amount is available
     
     // class objects
-    private RowSet readHitCache;
-    private RowSet readMissCache;
-    private final ObjectIndex  index;
-    private Row    keyrow;
-    private int            readHit, readMiss, writeUnique, writeDouble, cacheDelete, cacheFlush;
-    private int            hasnotHit, hasnotMiss, hasnotUnique, hasnotDouble, hasnotDelete;
+    private final ObjectIndex index;         // the back-end of the cache
+    private       RowSet      readHitCache;  // contains a complete copy of the cached objects
+    private       RowSet      readMissCache; // contains only the keys of the objects that had been a miss
+    private       Row         keyrow;
+    private       int         readHit, readMiss, writeUnique, writeDouble, cacheDelete, cacheFlush;
+    private       int         hasnotHit, hasnotMiss, hasnotUnique, hasnotDouble, hasnotDelete;
     
     public Cache(final ObjectIndex backupIndex) {
         this.index = backupIndex;
@@ -99,11 +99,6 @@ public class Cache implements ObjectIndex {
     public int writeBufferSize() {
         return 0;
     }
-
-    public static void setCacheGrowStati(final long memStopGrowNew, final long memStartShrinkNew) {
-        memStopGrow = memStopGrowNew;
-        memStartShrink =  memStartShrinkNew;
-    }
     
     public static long getMemStopGrow() {
         return memStopGrow ;
@@ -131,7 +126,7 @@ public class Cache implements ObjectIndex {
         final HashMap<String, String> map = new HashMap<String, String>();
         map.put("objectHitChunkSize", (readHitCache == null) ? "0" : Integer.toString(readHitCache.rowdef.objectsize));
         map.put("objectHitCacheCount", (readHitCache == null) ? "0" : Integer.toString(readHitCache.size()));
-        map.put("objectHitMem", (readHitCache == null) ? "0" : Integer.toString((int) (readHitCache.rowdef.objectsize * readHitCache.size() * RowCollection.growfactor)));
+        map.put("objectHitMem", (readHitCache == null) ? "0" : Long.toString(readHitCache.rowdef.objectsize * readHitCache.size()));
         map.put("objectHitCacheReadHit", Integer.toString(readHit));
         map.put("objectHitCacheReadMiss", Integer.toString(readMiss));
         map.put("objectHitCacheWriteUnique", Integer.toString(writeUnique));
@@ -141,7 +136,7 @@ public class Cache implements ObjectIndex {
         
         map.put("objectMissChunkSize", (readMissCache == null) ? "0" : Integer.toString(readMissCache.rowdef.objectsize));
         map.put("objectMissCacheCount", (readMissCache == null) ? "0" : Integer.toString(readMissCache.size()));
-        map.put("objectMissMem", (readMissCache == null) ? "0" : Integer.toString((int) (readMissCache.rowdef.objectsize * readMissCache.size() * RowCollection.growfactor)));
+        map.put("objectMissMem", (readMissCache == null) ? "0" : Long.toString(readMissCache.rowdef.objectsize * readMissCache.size()));
         map.put("objectMissCacheReadHit", Integer.toString(hasnotHit));
         map.put("objectMissCacheReadMiss", Integer.toString(hasnotMiss));
         map.put("objectMissCacheWriteUnique", Integer.toString(hasnotUnique));
@@ -153,13 +148,20 @@ public class Cache implements ObjectIndex {
         return map;
     }
     
-    private int cacheGrowStatus() {
+    private int cacheGrowStatus(long available) {
         return CachedRecords.cacheGrowStatus(MemoryControl.available(), memStopGrow, memStartShrink);
     }
     
     private boolean checkMissSpace() {
         // returns true if it is allowed to write into this cache
-        if (cacheGrowStatus() < 1) {
+    	long available = MemoryControl.available();
+    	if (available - 2 * 1024 * 1024 < readMissCache.memoryNeededForGrow()) {
+    		if (readMissCache != null) {
+    			readMissCache.clear();
+            }
+    		return false;
+    	}
+        if (cacheGrowStatus(available) < 1) {
             if (readMissCache != null) {
                 readMissCache.clear();
             }
@@ -170,7 +172,14 @@ public class Cache implements ObjectIndex {
     
     private boolean checkHitSpace() {
         // returns true if it is allowed to write into this cache
-        final int status = cacheGrowStatus();
+    	long available = MemoryControl.available();
+    	if (available - 2 * 1024 * 1024 < readHitCache.memoryNeededForGrow()) {
+    		if (readHitCache != null) {
+                readHitCache.clear();
+            }
+    		return false;
+    	}
+        final int status = cacheGrowStatus(available);
         if (status < 1) {
             if (readHitCache != null) {
                 readHitCache.clear();
@@ -462,4 +471,4 @@ public class Cache implements ObjectIndex {
         this.index.deleteOnExit();
     }
 
-}
+}

source/de/anomic/kelondro/index/RowCollection.java

@@ -194,6 +194,12 @@ public class RowCollection implements Iterable<Row.Entry> {
         chunkcache = newChunkcache;
     }
     
+    /**
+     * compute the needed memory in case of a cache extension. That is, if the cache is full and must
+     * be copied into a new cache which is larger. In such a case the Collection needs more than the double size
+     * than is necessary to store the data. This method coputes the extra memory that is needed to perform this task.
+     * @return
+     */
     public final long memoryNeededForGrow() {
         return (long) ((((long) (chunkcount + 1)) * ((long) rowdef.objectsize)) * growfactor);
     }

source/de/anomic/kelondro/table/CachedRecords.java

@@ -46,9 +46,9 @@ public class CachedRecords extends AbstractRecords implements RandomAccessRecord
     private   static final int element_in_cache = 4; // for kelondroCollectionObjectMap: 4; for HashMap: 52
     
     // static supervision objects: recognize and coordinate all activites
-    private static TreeMap<String, CachedRecords> recordTracker = new TreeMap<String, CachedRecords>();
-    private static long memStopGrow    = 10000000; // a limit for the node cache to stop growing if less than this memory amount is available
-    private static long memStartShrink =  6000000; // a limit for the node cache to start with shrinking if less than this memory amount is available
+    private static final TreeMap<String, CachedRecords> recordTracker = new TreeMap<String, CachedRecords>();
+    private static final long memStopGrow    = 40 * 1024 * 1024; // a limit for the node cache to stop growing if less than this memory amount is available
+    private static final long memStartShrink = 20 * 1024 * 1024; // a limit for the node cache to start with shrinking if less than this memory amount is available
     
     // caching buffer
     private IntBytesMap   cacheHeaders; // the cache; holds overhead values and key element
@@ -120,9 +120,9 @@ public class CachedRecords extends AbstractRecords implements RandomAccessRecord
         }
     }
 
-    int cacheGrowStatus() {
+    public int cacheGrowStatus() {
         final long available = MemoryControl.available();
-        if ((cacheHeaders != null) && (available < cacheHeaders.memoryNeededForGrow())) return 0;
+        if ((cacheHeaders != null) && (available  - 2 * 1024 * 1024 < cacheHeaders.memoryNeededForGrow())) return 0;
         return cacheGrowStatus(available, memStopGrow, memStartShrink);
     }
     
@@ -133,18 +133,13 @@ public class CachedRecords extends AbstractRecords implements RandomAccessRecord
         // 2: cache is allowed to grow and must not shrink
         if (available > stopGrow) return 2;
         if (available > startShrink) {
-            MemoryControl.gc(30000, "kelendroCacheRecords.cacheGrowStatus(...) 1"); // thq
+            MemoryControl.gc(60000, "kelendroCacheRecords.cacheGrowStatus(...) 1"); // thq
             return 1;
         }
-        MemoryControl.gc(3000, "kelendroCacheRecords.cacheGrowStatus(...) 0"); // thq
+        MemoryControl.gc(30000, "kelendroCacheRecords.cacheGrowStatus(...) 0"); // thq
         return 0;
     }
     
-    public static void setCacheGrowStati(final long memStopGrowNew, final long memStartShrinkNew) {
-        memStopGrow = memStopGrowNew;
-        memStartShrink =  memStartShrinkNew;
-    }
-    
     public static long getMemStopGrow() {
         return memStopGrow ;
     }

source/de/anomic/kelondro/util/MemoryControl.java

@@ -46,6 +46,7 @@ public class MemoryControl {
      * @param info additional info for log
      */
     public final synchronized static void gc(final int last, final String info) { // thq
+    	assert last >= 10000; // too many forced GCs will cause bad execution performance
         final long elapsed = System.currentTimeMillis() - lastGC;
         if (elapsed > last) {
             final long before = free();
@@ -71,7 +72,7 @@ public class MemoryControl {
      */
     private static long runGC(final boolean count) {
         final long memBefore = available();
-        gc(1000, "serverMemory.runGC(...)");
+        gc(10000, "serverMemory.runGC(...)");
         final long freed = available() - memBefore;
         if (count) {
             gcs[gcs_pos] = freed;
@@ -159,7 +160,7 @@ public class MemoryControl {
         final long avg = getAverageGCFree();
         if (force || avg == 0 || avg + avail >= size) {
             // this is only called if we expect that an allocation of <size> bytes would cause the jvm to call the GC anyway
-            final long freed = runGC(!force);
+            final long freed = runGC(false);
             avail = available();
             log.logInfo("performed " + ((force) ? "explicit" : "necessary") + " GC, freed " + (freed >> 10)
                     + " KB (requested/available/average: "

source/de/anomic/plasma/plasmaRankingCRProcess.java

@@ -365,7 +365,6 @@ public class plasmaRankingCRProcess {
         // finished. write to file
         cr = null;
         cr_in = null;
-        MemoryControl.gc(1000, "plasmaRankingCRProcess.genrci(...)"); // thq
         rci.toFile(rci_out);
         return count;
     }

source/de/anomic/plasma/plasmaSwitchboard.java

@@ -103,8 +103,6 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
-import java.util.Timer;
-import java.util.TimerTask;
 import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.regex.Pattern;
@@ -143,11 +141,9 @@ import de.anomic.http.httpdRobotsTxtConfig;
 import de.anomic.index.indexDocumentMetadata;
 import de.anomic.index.indexReferenceBlacklist;
 import de.anomic.index.indexURLReference;
-import de.anomic.kelondro.blob.Cache;
 import de.anomic.kelondro.order.DateFormatter;
 import de.anomic.kelondro.order.Digest;
 import de.anomic.kelondro.order.NaturalOrder;
-import de.anomic.kelondro.table.CachedRecords;
 import de.anomic.kelondro.util.FileUtils;
 import de.anomic.kelondro.util.Log;
 import de.anomic.kelondro.util.MemoryControl;
@@ -236,7 +232,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
     public  double                         totalQPM = 0d;
     public  TreeMap<String, String>        clusterhashes; // map of peerhash(String)/alternative-local-address as ip:port or only ip (String) or null if address in seed should be used
     public  URLLicense                     licensedURLs;
-    public  Timer                          moreMemory;
     public  List<Pattern>                  networkWhitelist, networkBlacklist;
     public  Dispatcher                     dhtDispatcher;
     public  List<String>                   trail;
@@ -458,13 +453,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
         //Init bookmarks DB
         initBookmarks();
         
-        // set a maximum amount of memory for the caches
-        // long memprereq = Math.max(getConfigLong(INDEXER_MEMPREREQ, 0), wordIndex.minMem());
-        // setConfig(INDEXER_MEMPREREQ, memprereq);
-        // setThreadPerformance(INDEXER, getConfigLong(INDEXER_IDLESLEEP, 0), getConfigLong(INDEXER_BUSYSLEEP, 0), memprereq);
-        CachedRecords.setCacheGrowStati(40 * 1024 * 1024, 20 * 1024 * 1024);
-        Cache.setCacheGrowStati(40 * 1024 * 1024, 20 * 1024 * 1024);
-        
         // make parser
         log.logConfig("Starting Parser");
         this.parser = new plasmaParser();
@@ -607,10 +595,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
         
         // deploy busy threads
         log.logConfig("Starting Threads");
-        MemoryControl.gc(1000, "plasmaSwitchboard, help for profiler"); // help for profiler - thq
-
-        moreMemory = new Timer(); // init GC Thread - thq
-        moreMemory.schedule(new MoreMemory(), 300000, 600000);
+        MemoryControl.gc(10000, "plasmaSwitchboard, help for profiler"); // help for profiler - thq
         
         deployThread(plasmaSwitchboardConstants.CLEANUP, "Cleanup", "simple cleaning process for monitoring information", null,
                      new serverInstantBusyThread(this, plasmaSwitchboardConstants.CLEANUP_METHOD_START, plasmaSwitchboardConstants.CLEANUP_METHOD_JOBCOUNT, plasmaSwitchboardConstants.CLEANUP_METHOD_FREEMEM), 600000); // all 5 Minutes, wait 10 minutes until first run
@@ -1089,7 +1074,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
     public void close() {
         log.logConfig("SWITCHBOARD SHUTDOWN STEP 1: sending termination signal to managed threads:");
         serverProfiling.stopSystemProfiling();
-        moreMemory.cancel();
         terminateAllThreads(true);
         log.logConfig("SWITCHBOARD SHUTDOWN STEP 2: sending termination signal to threaded indexing");
         // closing all still running db importer jobs
@@ -1150,14 +1134,10 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
     
     public void deQueueFreeMem() {
         // flush some entries from the RAM cache
-        webIndex.flushCacheFor(3000);
+        webIndex.flushCacheFor(5000);
         // empty some caches
         webIndex.clearCache();
         plasmaSearchEvent.cleanupEvents(true);
-        // adopt maximum cache size to current size to prevent that further OutOfMemoryErrors occur
-/*      int newMaxCount = Math.max(1200, Math.min((int) getConfigLong(WORDCACHE_MAX_COUNT, 1200), wordIndex.dhtOutCacheSize()));
-        setConfig(WORDCACHE_MAX_COUNT, Integer.toString(newMaxCount));
-        wordIndex.setMaxWordCount(newMaxCount); */
     }
     
     public IndexingStack.QueueEntry deQueue() {
@@ -2157,12 +2137,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
     }
 }
 
-class MoreMemory extends TimerTask {
-    public final void run() {
-        MemoryControl.gc(10000, "MoreMemory()");
-    }
-}
-
 class delayedShutdown extends Thread {
     private final plasmaSwitchboard sb;
     private final long delay;

source/de/anomic/server/serverCore.java

@@ -58,7 +58,6 @@ import javax.net.ssl.SSLSocketFactory;
 
 import de.anomic.icap.icapd;
 import de.anomic.kelondro.util.ByteBuffer;
-import de.anomic.kelondro.util.MemoryControl;
 import de.anomic.kelondro.util.Log;
 import de.anomic.tools.PKCS12Tool;
 import de.anomic.urlRedirector.urlRedirectord;
@@ -317,7 +316,6 @@ public final class serverCore extends serverAbstractBusyThread implements server
     
     public void freemem() {
         // FIXME: can we something here to flush memory? Idea: Reduce the size of some of our various caches.
-        MemoryControl.gc(2000, "serverCore.freemem()"); // thq
     }
     
     // class body

source/de/anomic/yacy/dht/Dispatcher.java

@@ -124,11 +124,11 @@ public class Dispatcher {
     }
     
     public int cloudSize() {
-    	return this.transmissionCloud.size();
+    	return (this.transmissionCloud == null) ? 0 : this.transmissionCloud.size();
     }
     
     public int transmissionSize() {
-    	return this.indexingTransmissionProcessor.queueSize();
+    	return (this.indexingTransmissionProcessor == null) ? 0 : this.indexingTransmissionProcessor.queueSize();
     }
     
     /**