new cache flush configuration settings

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2460 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/PerformanceQueues_p.html

@@ -112,14 +112,33 @@ Changes take effect immediately</td>
   </tr>
   <tr valign="top" class="TableCellDark">
     <td class=small>Maximum number of words in cache:</td>
-    <td class=small><input name="wordCacheMaxCount" type="text" size="20" maxlength="100" value="#[wordCacheMaxCount]#"></td>
-    <td class=small>cannot be set for DHT</td>
-    <td class=small rowspan="2">
+    <td class=small colspan="2"><input name="wordCacheMaxCount" type="text" size="20" maxlength="100" value="#[wordCacheMaxCount]#"></td>
+    <td class=small>
     This is is the number of word indexes that shall be held in the
     ram cache during indexing. When YaCy is shut down, this cache must be
     flushed to disc; this may last some minutes.
     </td>
   </tr>
+  <tr valign="top" class="TableCellDark">
+    <td class=small>Initial space of words in cache:</td>
+    <td class=small colspan="2"><input name="wordCacheInitCount" type="text" size="20" maxlength="100" value="#[wordCacheInitCount]#"></td>
+    <td class=small>
+    This is is the init size of space for words in cache.
+    </td>
+  </tr>
+  <tr valign="top" class="TableCellDark">
+    <td class=small>word-flush idle divisor:</td>
+    <td class=small colspan="2"><input name="wordFlushIdleDivisor" type="text" size="20" maxlength="100" value="#[wordFlushIdleDivisor]#"></td>
+    <td class=small rowspan="2">
+    The word flush divisor is applied when an indexing loop is executed.
+    The divisor is used to divide the number of words in the cache; the result is the number of words that are flushed.
+    The idleDivisor is used if the indexing queue is empty, the busyDivisor is used if the indexing queue contains at least one element.
+    </td>
+  </tr>
+  <tr valign="top" class="TableCellDark">
+    <td class=small>word-flush busy divisor:</td>
+    <td class=small colspan="2"><input name="wordFlushBusyDivisor" type="text" size="20" maxlength="100" value="#[wordFlushBusyDivisor]#"></td>
+  </tr>
   <tr valign="top" class="TableCellLight">
     <td class=small colspan="4">
     <input type="submit" name="cacheSizeSubmit" value="Enter New Cache Size">

htroot/PerformanceQueues_p.java

@@ -170,11 +170,21 @@ public class PerformanceQueues_p {
         prop.put("table", c);
         
         if ((post != null) && (post.containsKey("cacheSizeSubmit"))) {
-            int wordCacheMaxCount = post.getInt("wordCacheMaxCount", 10000);
+            int wordCacheMaxCount = post.getInt("wordCacheMaxCount", 20000);
             switchboard.setConfig("wordCacheMaxCount", Integer.toString(wordCacheMaxCount));
             switchboard.wordIndex.setMaxWordCount(wordCacheMaxCount);
+            
+            int wordCacheInitCount = post.getInt("wordCacheInitCount", 30000);
+            switchboard.setConfig("wordCacheInitCount", Integer.toString(wordCacheInitCount));
+            
             int maxWaitingWordFlush = post.getInt("maxWaitingWordFlush", 180);
             switchboard.setConfig("maxWaitingWordFlush", Integer.toString(maxWaitingWordFlush));
+            
+            int wordFlushIdleDivisor = post.getInt("wordFlushIdleDivisor", 420);
+            switchboard.setConfig("wordFlushIdleDivisor", Integer.toString(wordFlushIdleDivisor));
+            int wordFlushBusyDivisor = post.getInt("wordFlushBusyDivisor", 5000);
+            switchboard.setConfig("wordFlushBusyDivisor", Integer.toString(wordFlushBusyDivisor));
+            switchboard.wordIndex.setWordFlushDivisor(wordFlushIdleDivisor, wordFlushBusyDivisor);
         }
         
         if ((post != null) && (post.containsKey("poolConfig"))) {
@@ -255,7 +265,10 @@ public class PerformanceQueues_p {
         prop.put("maxAgeOfKCache", "" + (switchboard.wordIndex.maxAgeOfKCache() / 1000 / 60)); // minutes
         prop.put("minAgeOfKCache", "" + (switchboard.wordIndex.minAgeOfKCache() / 1000 / 60)); // minutes
         prop.put("maxWaitingWordFlush", switchboard.getConfig("maxWaitingWordFlush", "180"));
-        prop.put("wordCacheMaxCount", switchboard.wordIndex.getMaxWordCount());
+        prop.put("wordCacheMaxCount", switchboard.getConfigLong("wordCacheMaxCount", 20000));
+        prop.put("wordCacheInitCount", switchboard.getConfigLong("wordCacheInitCount", 30000));
+        prop.put("wordFlushIdleDivisor", switchboard.getConfigLong("wordFlushIdleDivisor", 420));
+        prop.put("wordFlushBusyDivisor", switchboard.getConfigLong("wordFlushBusyDivisor", 5000));
         prop.put("onlineCautionDelay", switchboard.getConfig("onlineCautionDelay", "30000"));
         prop.put("onlineCautionDelayCurrent", System.currentTimeMillis() - switchboard.proxyLastAccess);
         

htroot/yacy/transferRWI.java

@@ -114,7 +114,7 @@ public final class transferRWI {
             granted = false; // don't accept more words if there are too many words to flush
             result = "busy";
             pause = 60000;
-        } else if ((checkLimit && sb.wordIndex.wSize() > sb.wordIndex.getMaxWordCount()) || ((sb.wordIndex.busyCacheFlush) && (!shortCacheFlush))) {
+        } else if ((checkLimit && sb.wordIndex.wSize() > sb.getConfigLong("wordCacheMaxCount", 20000)) || ((sb.wordIndex.busyCacheFlush) && (!shortCacheFlush))) {
             // we are too busy flushing the ramCache to receive indexes
             sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". We are too busy (wordcachesize=" + sb.wordIndex.wSize() + ").");
             granted = false; // don't accept more words if there are too many words to flush

source/de/anomic/plasma/plasmaSwitchboard.java

@@ -413,7 +413,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
         wordIndex = new plasmaWordIndex(plasmaPath, indexPublicTextPath, ramRWI, ramRWI_time, log, getConfigBool("useCollectionIndex", false));
 
         // set a high maximum cache size to current size; this is adopted later automatically
-        int wordCacheMaxCount = Math.max(80000, (int) getConfigLong("wordCacheMaxCount", 80000));
+        int wordCacheMaxCount = Math.max((int) getConfigLong("wordCacheInitCount", 30000),
+                                         (int) getConfigLong("wordCacheMaxCount", 20000));
         setConfig("wordCacheMaxCount", Integer.toString(wordCacheMaxCount));
         wordIndex.setMaxWordCount(wordCacheMaxCount); 
         
@@ -949,9 +950,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
     
     public void deQueueFreeMem() {
         // flush some entries from the RAM cache
-        wordIndex.flushCacheSome();
+        wordIndex.flushCacheSome(false);
         // adopt maximum cache size to current size to prevent that further OutOfMemoryErrors occur
-        int newMaxCount = Math.max(2000, Math.min(wordIndex.getMaxWordCount(), wordIndex.wSize()));
+        int newMaxCount = Math.max(2000, Math.min((int) getConfigLong("wordCacheMaxCount", 20000), wordIndex.wSize()));
         setConfig("wordCacheMaxCount", Integer.toString(newMaxCount));
         wordIndex.setMaxWordCount(newMaxCount); 
     }
@@ -965,7 +966,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
 
         // flush some entries from the RAM cache
         // (new permanent cache flushing)
-        wordIndex.flushCacheSome();
+        wordIndex.flushCacheSome(sbQueue.size() != 0);
 
         boolean doneSomething = false;
         

source/de/anomic/plasma/plasmaWordIndex.java

@@ -90,6 +90,7 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI {
     private final plasmaWordIndexFileCluster       backend;              // old database structure, to be replaced by CollectionRI
     public        boolean                          busyCacheFlush;       // shows if a cache flush is currently performed
     public        boolean                          useCollectionIndex;   // flag for usage of new collectionIndex db
+    private       int idleDivisor, busyDivisor;
     
     public plasmaWordIndex(File oldDatabaseRoot, File newIndexRoot, int bufferkb, long preloadTime, serverLog log, boolean useCollectionIndex) {
         this.oldDatabaseRoot = oldDatabaseRoot;
@@ -111,6 +112,8 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI {
         
         busyCacheFlush = false;
         this.useCollectionIndex = useCollectionIndex;
+        this.busyDivisor = 5000;
+        this.idleDivisor = 420;
     }
 
     public File getRoot() {
@@ -169,8 +172,9 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI {
         ramCache.setMaxWordCount(maxWords);
     }
 
-    public int getMaxWordCount() {
-        return ramCache.getMaxWordCount();
+    public void setWordFlushDivisor(int idleDivisor, int busyDivisor) {
+       this.idleDivisor = idleDivisor;
+       this.busyDivisor = busyDivisor;
     }
 
     public void flushControl() {
@@ -198,10 +202,9 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI {
             return added;
     }
 
-    public void flushCacheSome() {
+    public void flushCacheSome(boolean busy) {
         synchronized (this) { ramCache.shiftK2W(); }
-        //int flushCount = ramCache.wSize() / 420;
-        int flushCount = ramCache.wSize() / 5000; // for testings
+        int flushCount = (busy) ? ramCache.wSize() / busyDivisor : ramCache.wSize() / idleDivisor;
         if (flushCount > 100) flushCount = 100;
         if (flushCount < 1) flushCount = Math.min(1, ramCache.wSize());
         flushCache(flushCount);

yacy.init

@@ -578,6 +578,9 @@ javastart_Xms=Xms10m
 # maxWaitingWordFlush gives the number of seconds that the shutdown
 # may last for the word flush
 wordCacheMaxCount = 20000
+wordCacheInitCount = 30000
+wordFlushIdleDivisor = 420;
+wordFlushBusyDivisor = 5000;
 
 # Specifies if yacy can be used as transparent http proxy.
 #