From 18b6876860142df3c1d5f1f6c95cfb11366b762c Mon Sep 17 00:00:00 2001 From: orbiter Date: Fri, 25 Aug 2006 22:31:21 +0000 Subject: [PATCH] new cache flush configuration settings git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2460 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/PerformanceQueues_p.html | 25 ++++++++++++++++--- htroot/PerformanceQueues_p.java | 17 +++++++++++-- htroot/yacy/transferRWI.java | 2 +- .../de/anomic/plasma/plasmaSwitchboard.java | 9 ++++--- source/de/anomic/plasma/plasmaWordIndex.java | 13 ++++++---- yacy.init | 3 +++ 6 files changed, 54 insertions(+), 15 deletions(-) diff --git a/htroot/PerformanceQueues_p.html b/htroot/PerformanceQueues_p.html index abc4f0552..8dbc4277e 100644 --- a/htroot/PerformanceQueues_p.html +++ b/htroot/PerformanceQueues_p.html @@ -112,14 +112,33 @@ Changes take effect immediately Maximum number of words in cache: - - cannot be set for DHT - + + This is is the number of word indexes that shall be held in the ram cache during indexing. When YaCy is shut down, this cache must be flushed to disc; this may last some minutes. + + Initial space of words in cache: + + + This is is the init size of space for words in cache. + + + + word-flush idle divisor: + + + The word flush divisor is applied when an indexing loop is executed. + The divisor is used to divide the number of words in the cache; the result is the number of words that are flushed. + The idleDivisor is used if the indexing queue is empty, the busyDivisor is used if the indexing queue contains at least one element. + + + + word-flush busy divisor: + + diff --git a/htroot/PerformanceQueues_p.java b/htroot/PerformanceQueues_p.java index d3df416f1..6bb18215e 100644 --- a/htroot/PerformanceQueues_p.java +++ b/htroot/PerformanceQueues_p.java @@ -170,11 +170,21 @@ public class PerformanceQueues_p { prop.put("table", c); if ((post != null) && (post.containsKey("cacheSizeSubmit"))) { - int wordCacheMaxCount = post.getInt("wordCacheMaxCount", 10000); + int wordCacheMaxCount = post.getInt("wordCacheMaxCount", 20000); switchboard.setConfig("wordCacheMaxCount", Integer.toString(wordCacheMaxCount)); switchboard.wordIndex.setMaxWordCount(wordCacheMaxCount); + + int wordCacheInitCount = post.getInt("wordCacheInitCount", 30000); + switchboard.setConfig("wordCacheInitCount", Integer.toString(wordCacheInitCount)); + int maxWaitingWordFlush = post.getInt("maxWaitingWordFlush", 180); switchboard.setConfig("maxWaitingWordFlush", Integer.toString(maxWaitingWordFlush)); + + int wordFlushIdleDivisor = post.getInt("wordFlushIdleDivisor", 420); + switchboard.setConfig("wordFlushIdleDivisor", Integer.toString(wordFlushIdleDivisor)); + int wordFlushBusyDivisor = post.getInt("wordFlushBusyDivisor", 5000); + switchboard.setConfig("wordFlushBusyDivisor", Integer.toString(wordFlushBusyDivisor)); + switchboard.wordIndex.setWordFlushDivisor(wordFlushIdleDivisor, wordFlushBusyDivisor); } if ((post != null) && (post.containsKey("poolConfig"))) { @@ -255,7 +265,10 @@ public class PerformanceQueues_p { prop.put("maxAgeOfKCache", "" + (switchboard.wordIndex.maxAgeOfKCache() / 1000 / 60)); // minutes prop.put("minAgeOfKCache", "" + (switchboard.wordIndex.minAgeOfKCache() / 1000 / 60)); // minutes prop.put("maxWaitingWordFlush", switchboard.getConfig("maxWaitingWordFlush", "180")); - prop.put("wordCacheMaxCount", switchboard.wordIndex.getMaxWordCount()); + prop.put("wordCacheMaxCount", switchboard.getConfigLong("wordCacheMaxCount", 20000)); + prop.put("wordCacheInitCount", switchboard.getConfigLong("wordCacheInitCount", 30000)); + prop.put("wordFlushIdleDivisor", switchboard.getConfigLong("wordFlushIdleDivisor", 420)); + prop.put("wordFlushBusyDivisor", switchboard.getConfigLong("wordFlushBusyDivisor", 5000)); prop.put("onlineCautionDelay", switchboard.getConfig("onlineCautionDelay", "30000")); prop.put("onlineCautionDelayCurrent", System.currentTimeMillis() - switchboard.proxyLastAccess); diff --git a/htroot/yacy/transferRWI.java b/htroot/yacy/transferRWI.java index ba501fa59..76245ab03 100644 --- a/htroot/yacy/transferRWI.java +++ b/htroot/yacy/transferRWI.java @@ -114,7 +114,7 @@ public final class transferRWI { granted = false; // don't accept more words if there are too many words to flush result = "busy"; pause = 60000; - } else if ((checkLimit && sb.wordIndex.wSize() > sb.wordIndex.getMaxWordCount()) || ((sb.wordIndex.busyCacheFlush) && (!shortCacheFlush))) { + } else if ((checkLimit && sb.wordIndex.wSize() > sb.getConfigLong("wordCacheMaxCount", 20000)) || ((sb.wordIndex.busyCacheFlush) && (!shortCacheFlush))) { // we are too busy flushing the ramCache to receive indexes sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". We are too busy (wordcachesize=" + sb.wordIndex.wSize() + ")."); granted = false; // don't accept more words if there are too many words to flush diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 4d2ce9343..c7a5cb17c 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -413,7 +413,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser wordIndex = new plasmaWordIndex(plasmaPath, indexPublicTextPath, ramRWI, ramRWI_time, log, getConfigBool("useCollectionIndex", false)); // set a high maximum cache size to current size; this is adopted later automatically - int wordCacheMaxCount = Math.max(80000, (int) getConfigLong("wordCacheMaxCount", 80000)); + int wordCacheMaxCount = Math.max((int) getConfigLong("wordCacheInitCount", 30000), + (int) getConfigLong("wordCacheMaxCount", 20000)); setConfig("wordCacheMaxCount", Integer.toString(wordCacheMaxCount)); wordIndex.setMaxWordCount(wordCacheMaxCount); @@ -949,9 +950,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser public void deQueueFreeMem() { // flush some entries from the RAM cache - wordIndex.flushCacheSome(); + wordIndex.flushCacheSome(false); // adopt maximum cache size to current size to prevent that further OutOfMemoryErrors occur - int newMaxCount = Math.max(2000, Math.min(wordIndex.getMaxWordCount(), wordIndex.wSize())); + int newMaxCount = Math.max(2000, Math.min((int) getConfigLong("wordCacheMaxCount", 20000), wordIndex.wSize())); setConfig("wordCacheMaxCount", Integer.toString(newMaxCount)); wordIndex.setMaxWordCount(newMaxCount); } @@ -965,7 +966,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // flush some entries from the RAM cache // (new permanent cache flushing) - wordIndex.flushCacheSome(); + wordIndex.flushCacheSome(sbQueue.size() != 0); boolean doneSomething = false; diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index 90fc41234..08bb390de 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -90,6 +90,7 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI { private final plasmaWordIndexFileCluster backend; // old database structure, to be replaced by CollectionRI public boolean busyCacheFlush; // shows if a cache flush is currently performed public boolean useCollectionIndex; // flag for usage of new collectionIndex db + private int idleDivisor, busyDivisor; public plasmaWordIndex(File oldDatabaseRoot, File newIndexRoot, int bufferkb, long preloadTime, serverLog log, boolean useCollectionIndex) { this.oldDatabaseRoot = oldDatabaseRoot; @@ -111,6 +112,8 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI { busyCacheFlush = false; this.useCollectionIndex = useCollectionIndex; + this.busyDivisor = 5000; + this.idleDivisor = 420; } public File getRoot() { @@ -169,8 +172,9 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI { ramCache.setMaxWordCount(maxWords); } - public int getMaxWordCount() { - return ramCache.getMaxWordCount(); + public void setWordFlushDivisor(int idleDivisor, int busyDivisor) { + this.idleDivisor = idleDivisor; + this.busyDivisor = busyDivisor; } public void flushControl() { @@ -198,10 +202,9 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI { return added; } - public void flushCacheSome() { + public void flushCacheSome(boolean busy) { synchronized (this) { ramCache.shiftK2W(); } - //int flushCount = ramCache.wSize() / 420; - int flushCount = ramCache.wSize() / 5000; // for testings + int flushCount = (busy) ? ramCache.wSize() / busyDivisor : ramCache.wSize() / idleDivisor; if (flushCount > 100) flushCount = 100; if (flushCount < 1) flushCount = Math.min(1, ramCache.wSize()); flushCache(flushCount); diff --git a/yacy.init b/yacy.init index fbcdb95fc..2e81d3943 100644 --- a/yacy.init +++ b/yacy.init @@ -578,6 +578,9 @@ javastart_Xms=Xms10m # maxWaitingWordFlush gives the number of seconds that the shutdown # may last for the word flush wordCacheMaxCount = 20000 +wordCacheInitCount = 30000 +wordFlushIdleDivisor = 420; +wordFlushBusyDivisor = 5000; # Specifies if yacy can be used as transparent http proxy. #