- added security functions to flush url and search caches in case that memory is full

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4933 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/yacysearch.java

@@ -63,6 +63,7 @@ import de.anomic.plasma.plasmaSearchRankingProfile;
 import de.anomic.plasma.plasmaSnippetCache;
 import de.anomic.plasma.plasmaSwitchboard;
 import de.anomic.server.serverCore;
+import de.anomic.server.serverMemory;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverProfiling;
 import de.anomic.server.serverSwitch;
@@ -197,6 +198,11 @@ public class yacysearch {
         }
         
         if ((!block) && (post == null || post.get("cat", "href").equals("href"))) {
+            // check available memory and clean up if necessary
+            if (!serverMemory.request(8000000L, false)) {
+                sb.webIndex.clearCache();
+                plasmaSearchEvent.cleanupEvents(true);
+            }
             
             plasmaSearchRankingProfile ranking = sb.getRanking();
             final TreeSet<String>[] query = plasmaSearchQuery.cleanQuery(querystring); // converts also umlaute

source/de/anomic/index/indexRepositoryReference.java

@@ -61,9 +61,13 @@ public final class indexRepositoryReference {
     public indexRepositoryReference(File indexSecondaryPath) {
         super();
         this.location = new File(indexSecondaryPath, "TEXT");        
-        urlIndexFile = new kelondroSplitTable(this.location, "urls", indexURLReference.rowdef, false);
+        urlIndexFile = new kelondroCache(new kelondroSplitTable(this.location, "urls", indexURLReference.rowdef, false));
     }
 
+    public void clearCache() {
+        if (urlIndexFile instanceof kelondroCache) ((kelondroCache) urlIndexFile).clearCache();
+    }
+    
     public void clear() throws IOException {
         if (exportthread != null) exportthread.interrupt();
         urlIndexFile.clear();

source/de/anomic/kelondro/kelondroCache.java

@@ -51,8 +51,8 @@ public class kelondroCache implements kelondroIndex {
 
     // static object tracker; stores information about object cache usage
     private static final TreeMap<String, kelondroCache> objectTracker = new TreeMap<String, kelondroCache>();
-    private static long memStopGrow    = 10000000; // a limit for the node cache to stop growing if less than this memory amount is available
-    private static long memStartShrink =  6000000; // a limit for the node cache to start with shrinking if less than this memory amount is available
+    private static long memStopGrow    = 12 * 1024 * 1024; // a limit for the node cache to stop growing if less than this memory amount is available
+    private static long memStartShrink =  8 * 1024 * 1024; // a limit for the node cache to start with shrinking if less than this memory amount is available
     
     // class objects
     private kelondroRowSet readHitCache;
@@ -180,6 +180,11 @@ public class kelondroCache implements kelondroIndex {
         return true;
     }
     
+    public synchronized void clearCache() {
+        readMissCache.clear();
+        readHitCache.clear();
+    }
+    
     public synchronized void close() {
         index.close();
         readHitCache = null;
@@ -187,7 +192,27 @@ public class kelondroCache implements kelondroIndex {
     }
 
     public boolean has(byte[] key) throws IOException {
-        return (get(key) != null);
+        // first look into the miss cache
+        if (readMissCache != null) {
+            if (readMissCache.get(key) != null) {
+                this.hasnotHit++;
+                return false;
+            } else {
+                this.hasnotMiss++;
+            }
+        }
+
+        // then try the hit cache and the buffers
+        if (readHitCache != null) {
+            if (readHitCache.get(key) != null) {
+                this.readHit++;
+                return true;
+            }
+        }
+        
+        // finally ask the back-end index
+        this.readMiss++;
+        return index.has(key);
     }
     
     public synchronized Entry get(byte[] key) throws IOException {
@@ -212,7 +237,7 @@ public class kelondroCache implements kelondroIndex {
             }
         }
         
-        // finally ask the backend index
+        // finally ask the back-end index
         this.readMiss++;
         entry = index.get(key);
         // learn from result
@@ -289,9 +314,8 @@ public class kelondroCache implements kelondroIndex {
 
     public synchronized Entry put(Entry row, Date entryDate) throws IOException {
         // a put with a date is bad for the cache: the date cannot be handled
-        // The write buffer does not work here, because it does not store dates.
-        
-        throw new UnsupportedOperationException("put with date is inefficient in kelondroCache");
+        // we omit the date here and use the current Date everywhere
+        return this.put(row);
     }
 
     public synchronized boolean addUnique(Entry row) throws IOException {

source/de/anomic/plasma/plasmaSwitchboard.java

@@ -1661,6 +1661,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
     public void deQueueFreeMem() {
         // flush some entries from the RAM cache
         webIndex.flushCacheSome();
+        // empty some caches
+        webIndex.clearCache();
+        plasmaSearchEvent.cleanupEvents(true);
         // adopt maximum cache size to current size to prevent that further OutOfMemoryErrors occur
 /*      int newMaxCount = Math.max(1200, Math.min((int) getConfigLong(WORDCACHE_MAX_COUNT, 1200), wordIndex.dhtOutCacheSize()));
         setConfig(WORDCACHE_MAX_COUNT, Integer.toString(newMaxCount));
@@ -1821,6 +1824,12 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
         try {
             boolean hasDoneSomething = false;
             
+            // clear caches if necessary
+            if (!serverMemory.request(8000000L, false)) {
+                webIndex.clearCache();
+                plasmaSearchEvent.cleanupEvents(true);
+            }
+            
             // set a random password if no password is configured
             if (!this.acceptLocalURLs && getConfigBool("adminAccountForLocalhost", false) && getConfig(httpd.ADMIN_ACCOUNT_B64MD5, "").length() == 0) {
                 // make a 'random' password

source/de/anomic/plasma/plasmaWordIndex.java

@@ -93,7 +93,7 @@ public final class plasmaWordIndex implements indexRI {
     private final indexRAMRI               dhtOutCache, dhtInCache;
     private final indexCollectionRI        collections;          // new database structure to replace AssortmentCluster and FileCluster
     private       serverLog                log;
-    indexRepositoryReference referenceURL;
+    indexRepositoryReference               referenceURL;
     public        yacySeedDB               seedDB;
     public        yacyNewsPool             newsPool;
     private       File                     primaryRoot, secondaryRoot;
@@ -195,6 +195,10 @@ public final class plasmaWordIndex implements indexRI {
         this.peerActions = new yacyPeerActions(seedDB, newsPool);
     }
     
+    public void clearCache() {
+        referenceURL.clearCache();
+    }
+
     public void clear() {
         dhtInCache.clear();
         dhtOutCache.clear();