changed way of storage for search requests:

- the search request cache can now get as large as 1000 entries
- if more entries arrive, unused are deleted
- the elements may stay in the cache up to 10 minutes and longer if they are used
- the elements are deleted earlier that 10 minutes if the memory gets low
This commit was mainly done for metager-feeding peers that have a query load of 50000 queries each day. Also added:
- a monitor for cache hit/cache miss in PerformanceMemory_p.html (see at bottom of page)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7093 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/PerformanceMemory_p.html

@@ -184,19 +184,51 @@
     <table border="0" cellpadding="2" cellspacing="1">
       <tr class="TableHeader" valign="bottom">
         <td>Type</td>
-        <td>Amount</td>
+        <td>Size</td>
+        <td>Hit</td>
+        <td>Miss</td>
+        <td>Insert</td>
+        <td>Delete</td>
       </tr>
       <tr class="TableCellLight">
         <td>DNSCache</td>
-        <td>#[namecache.hit]#</td>
+        <td>#[namecacheHit.size]#</td>
+        <td></td>
+        <td></td>
+        <td></td>
+        <td></td>
+      </tr>
+      <tr class="TableCellLight">
+        <td>DNSCache</td>
+        <td>#[namecacheMiss.size]#</td>
+        <td></td>
+        <td></td>
+        <td></td>
+        <td></td>
       </tr>
       <tr class="TableCellDark">
         <td>DNSNoCache</td>
+        <td></td>
         <td>#[namecache.noCache]#</td>
+        <td></td>
+        <td></td>
+        <td></td>
       </tr>
       <tr class="TableCellLight">
         <td>HashBlacklistedCache</td>
         <td>#[blacklistcache.size]#</td>
+        <td></td>
+        <td></td>
+        <td></td>
+        <td></td>
+      </tr>
+      <tr class="TableCellLight">
+        <td>Search Event Cache</td>
+        <td>#[searchevent.size]#</td>
+        <td>#[searchevent.hit]#</td>
+        <td>#[searchevent.miss]#</td>
+        <td>#[searchevent.insert]#</td>
+        <td>#[searchevent.delete]#</td>
       </tr>
     </table>
     

htroot/PerformanceMemory_p.java

@@ -38,6 +38,7 @@ import net.yacy.kelondro.util.FileUtils;
 import net.yacy.kelondro.util.Formatter;
 import net.yacy.kelondro.util.MemoryControl;
 
+import de.anomic.search.SearchEventCache;
 import de.anomic.search.Switchboard;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
@@ -187,12 +188,15 @@ public class PerformanceMemory_p {
         prop.putNum("objectMissCacheTotalMem", totalmissmem / (1024 * 1024d));
         
         // other caching structures
-        long amount = Domains.nameCacheHitSize();
-        prop.putNum("namecache.hit", amount);
-        amount = Domains.nameCacheNoCachingListSize();
-        prop.putNum("namecache.noCache", amount);
-        amount = Switchboard.urlBlacklist.blacklistCacheSize();
-        prop.putNum("blacklistcache.size", amount);
+        prop.putNum("namecacheHit.size", Domains.nameCacheHitSize());
+        prop.putNum("namecacheMiss.size", Domains.nameCacheMissSize());
+        prop.putNum("namecache.noCache", Domains.nameCacheNoCachingListSize());
+        prop.putNum("blacklistcache.size", Switchboard.urlBlacklist.blacklistCacheSize());
+        prop.putNum("searchevent.size", SearchEventCache.size());
+        prop.putNum("searchevent.hit", SearchEventCache.cacheHit);
+        prop.putNum("searchevent.miss", SearchEventCache.cacheMiss);
+        prop.putNum("searchevent.insert", SearchEventCache.cacheInsert);
+        prop.putNum("searchevent.delete", SearchEventCache.cacheDelete);
         // return rewrite values for templates
         return prop;
     }

source/de/anomic/search/SearchEventCache.java

@@ -26,10 +26,14 @@
 
 package de.anomic.search;
 
-import java.util.Iterator;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
 import java.util.TreeMap;
-import java.util.concurrent.ConcurrentHashMap;
 
+import net.yacy.cora.storage.ARC;
+import net.yacy.cora.storage.ConcurrentARC;
+import net.yacy.kelondro.util.MemoryControl;
 import net.yacy.repository.LoaderDispatcher;
 
 import de.anomic.crawler.ResultURLs;
@@ -37,33 +41,43 @@ import de.anomic.yacy.yacySeedDB;
 
 public class SearchEventCache {
 
-    private static ConcurrentHashMap<String, SearchEvent> lastEvents = new ConcurrentHashMap<String, SearchEvent>(); // a cache for objects from this class: re-use old search requests
-    public static final long eventLifetime = 60000; // the time an event will stay in the cache, 1 Minute
-
+    private static ARC<String, SearchEvent> lastEvents = new ConcurrentARC<String, SearchEvent>(1000, Runtime.getRuntime().availableProcessors() * 4); // a cache for objects from this class: re-use old search requests
+    public static final long eventLifetime = 600000; // the time an event will stay in the cache, 10 Minutes
+    public static final long memlimit = 100 * 1024 * 1024; // 100 MB
     public static String lastEventID = "";
+    public static long cacheInsert = 0, cacheHit = 0, cacheMiss = 0, cacheDelete = 0;
+    
+    public static int size() {
+        return lastEvents.size();
+    }
     
     public static void put(String eventID, SearchEvent event) {
         lastEventID = eventID;
-        lastEvents.put(eventID, event);
+        SearchEvent oldEvent = lastEvents.put(eventID, event);
+        if (oldEvent == null) cacheInsert++;
     }
     
     public static void cleanupEvents(final boolean all) {
         // remove old events in the event cache
-        final Iterator<SearchEvent> i = lastEvents.values().iterator();
-        SearchEvent event;
-        while (i.hasNext()) {
-            event = i.next();
-            if (all || event.getEventTime() + eventLifetime < System.currentTimeMillis()) {
-                event.cleanup();
-                
-                // remove the event
-                i.remove();
+        List<String> delete = new ArrayList<String>();
+        // the less memory is there, the less time is acceptable for elements in the cache
+        long memx = MemoryControl.available();
+        long acceptTime = memx > memlimit ? eventLifetime : memx  * eventLifetime / memlimit;
+        for (Map.Entry<String, SearchEvent> event: lastEvents) {
+            if (all || event.getValue().getEventTime() + acceptTime < System.currentTimeMillis()) {
+                event.getValue().cleanup();
+                delete.add(event.getKey());
             }
         }
+        // remove the events
+        cacheDelete += delete.size();
+        for (String k: delete) lastEvents.remove(k);
     }
 
     public static SearchEvent getEvent(final String eventID) {
-        return lastEvents.get(eventID);
+        SearchEvent event = lastEvents.get(eventID);
+        if (event == null) cacheMiss++; else cacheHit++;
+        return event;
     }
     
     public static SearchEvent getEvent(
@@ -76,11 +90,13 @@ public class SearchEventCache {
         
         String id = query.id(false);
         SearchEvent event = SearchEventCache.lastEvents.get(id);
+        if (event == null) cacheMiss++; else cacheHit++;
         if (Switchboard.getSwitchboard() != null && !Switchboard.getSwitchboard().crawlQueues.noticeURL.isEmpty() && event != null && System.currentTimeMillis() - event.getEventTime() > 60000) {
             // if a local crawl is ongoing, don't use the result from the cache to use possibly more results that come from the current crawl
             // to prevent that this happens during a person switches between the different result pages, a re-search happens no more than
             // once a minute
             SearchEventCache.lastEvents.remove(id);
+            cacheDelete++;
             event = null;
         } else {
             if (event != null) {