diff --git a/defaults/yacy.network.freeworld.unit b/defaults/yacy.network.freeworld.unit index d8e23e571..a8c6b90a5 100644 --- a/defaults/yacy.network.freeworld.unit +++ b/defaults/yacy.network.freeworld.unit @@ -14,7 +14,7 @@ network.unit.search.time = 4 network.unit.dht = true network.unit.dhtredundancy.junior = 1 network.unit.dhtredundancy.senior = 3 -network.unit.dht.partitionExponent = 2 +network.unit.dht.partitionExponent = 4 network.unit.remotecrawl.speed = 6 network.unit.bootstrap.seedlist0 = http://www.yacy.net/seed.txt network.unit.bootstrap.seedlist1 = http://home.arcor.de/hermens/yacy/seed.txt diff --git a/source/de/anomic/kelondro/blob/Cache.java b/source/de/anomic/kelondro/blob/Cache.java index dc6078d83..70e731d73 100644 --- a/source/de/anomic/kelondro/blob/Cache.java +++ b/source/de/anomic/kelondro/blob/Cache.java @@ -1,4 +1,4 @@ -// kelondroCache.java +// kelondro.blob.Cache.java // (C) 2006 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany // first published 26.10.2006 on http://www.anomic.de // @@ -58,16 +58,16 @@ public class Cache implements ObjectIndex { // static object tracker; stores information about object cache usage private static final TreeMap objectTracker = new TreeMap(); - private static long memStopGrow = 12 * 1024 * 1024; // a limit for the node cache to stop growing if less than this memory amount is available - private static long memStartShrink = 8 * 1024 * 1024; // a limit for the node cache to start with shrinking if less than this memory amount is available + private static final long memStopGrow = 40 * 1024 * 1024; // a limit for the node cache to stop growing if less than this memory amount is available + private static final long memStartShrink = 20 * 1024 * 1024; // a limit for the node cache to start with shrinking if less than this memory amount is available // class objects - private RowSet readHitCache; - private RowSet readMissCache; - private final ObjectIndex index; - private Row keyrow; - private int readHit, readMiss, writeUnique, writeDouble, cacheDelete, cacheFlush; - private int hasnotHit, hasnotMiss, hasnotUnique, hasnotDouble, hasnotDelete; + private final ObjectIndex index; // the back-end of the cache + private RowSet readHitCache; // contains a complete copy of the cached objects + private RowSet readMissCache; // contains only the keys of the objects that had been a miss + private Row keyrow; + private int readHit, readMiss, writeUnique, writeDouble, cacheDelete, cacheFlush; + private int hasnotHit, hasnotMiss, hasnotUnique, hasnotDouble, hasnotDelete; public Cache(final ObjectIndex backupIndex) { this.index = backupIndex; @@ -99,11 +99,6 @@ public class Cache implements ObjectIndex { public int writeBufferSize() { return 0; } - - public static void setCacheGrowStati(final long memStopGrowNew, final long memStartShrinkNew) { - memStopGrow = memStopGrowNew; - memStartShrink = memStartShrinkNew; - } public static long getMemStopGrow() { return memStopGrow ; @@ -131,7 +126,7 @@ public class Cache implements ObjectIndex { final HashMap map = new HashMap(); map.put("objectHitChunkSize", (readHitCache == null) ? "0" : Integer.toString(readHitCache.rowdef.objectsize)); map.put("objectHitCacheCount", (readHitCache == null) ? "0" : Integer.toString(readHitCache.size())); - map.put("objectHitMem", (readHitCache == null) ? "0" : Integer.toString((int) (readHitCache.rowdef.objectsize * readHitCache.size() * RowCollection.growfactor))); + map.put("objectHitMem", (readHitCache == null) ? "0" : Long.toString(readHitCache.rowdef.objectsize * readHitCache.size())); map.put("objectHitCacheReadHit", Integer.toString(readHit)); map.put("objectHitCacheReadMiss", Integer.toString(readMiss)); map.put("objectHitCacheWriteUnique", Integer.toString(writeUnique)); @@ -141,7 +136,7 @@ public class Cache implements ObjectIndex { map.put("objectMissChunkSize", (readMissCache == null) ? "0" : Integer.toString(readMissCache.rowdef.objectsize)); map.put("objectMissCacheCount", (readMissCache == null) ? "0" : Integer.toString(readMissCache.size())); - map.put("objectMissMem", (readMissCache == null) ? "0" : Integer.toString((int) (readMissCache.rowdef.objectsize * readMissCache.size() * RowCollection.growfactor))); + map.put("objectMissMem", (readMissCache == null) ? "0" : Long.toString(readMissCache.rowdef.objectsize * readMissCache.size())); map.put("objectMissCacheReadHit", Integer.toString(hasnotHit)); map.put("objectMissCacheReadMiss", Integer.toString(hasnotMiss)); map.put("objectMissCacheWriteUnique", Integer.toString(hasnotUnique)); @@ -153,13 +148,20 @@ public class Cache implements ObjectIndex { return map; } - private int cacheGrowStatus() { + private int cacheGrowStatus(long available) { return CachedRecords.cacheGrowStatus(MemoryControl.available(), memStopGrow, memStartShrink); } private boolean checkMissSpace() { // returns true if it is allowed to write into this cache - if (cacheGrowStatus() < 1) { + long available = MemoryControl.available(); + if (available - 2 * 1024 * 1024 < readMissCache.memoryNeededForGrow()) { + if (readMissCache != null) { + readMissCache.clear(); + } + return false; + } + if (cacheGrowStatus(available) < 1) { if (readMissCache != null) { readMissCache.clear(); } @@ -170,7 +172,14 @@ public class Cache implements ObjectIndex { private boolean checkHitSpace() { // returns true if it is allowed to write into this cache - final int status = cacheGrowStatus(); + long available = MemoryControl.available(); + if (available - 2 * 1024 * 1024 < readHitCache.memoryNeededForGrow()) { + if (readHitCache != null) { + readHitCache.clear(); + } + return false; + } + final int status = cacheGrowStatus(available); if (status < 1) { if (readHitCache != null) { readHitCache.clear(); @@ -462,4 +471,4 @@ public class Cache implements ObjectIndex { this.index.deleteOnExit(); } -} +} \ No newline at end of file diff --git a/source/de/anomic/kelondro/index/RowCollection.java b/source/de/anomic/kelondro/index/RowCollection.java index 7a32dc056..737343957 100644 --- a/source/de/anomic/kelondro/index/RowCollection.java +++ b/source/de/anomic/kelondro/index/RowCollection.java @@ -194,6 +194,12 @@ public class RowCollection implements Iterable { chunkcache = newChunkcache; } + /** + * compute the needed memory in case of a cache extension. That is, if the cache is full and must + * be copied into a new cache which is larger. In such a case the Collection needs more than the double size + * than is necessary to store the data. This method coputes the extra memory that is needed to perform this task. + * @return + */ public final long memoryNeededForGrow() { return (long) ((((long) (chunkcount + 1)) * ((long) rowdef.objectsize)) * growfactor); } diff --git a/source/de/anomic/kelondro/table/CachedRecords.java b/source/de/anomic/kelondro/table/CachedRecords.java index 48933c05f..823929405 100644 --- a/source/de/anomic/kelondro/table/CachedRecords.java +++ b/source/de/anomic/kelondro/table/CachedRecords.java @@ -46,9 +46,9 @@ public class CachedRecords extends AbstractRecords implements RandomAccessRecord private static final int element_in_cache = 4; // for kelondroCollectionObjectMap: 4; for HashMap: 52 // static supervision objects: recognize and coordinate all activites - private static TreeMap recordTracker = new TreeMap(); - private static long memStopGrow = 10000000; // a limit for the node cache to stop growing if less than this memory amount is available - private static long memStartShrink = 6000000; // a limit for the node cache to start with shrinking if less than this memory amount is available + private static final TreeMap recordTracker = new TreeMap(); + private static final long memStopGrow = 40 * 1024 * 1024; // a limit for the node cache to stop growing if less than this memory amount is available + private static final long memStartShrink = 20 * 1024 * 1024; // a limit for the node cache to start with shrinking if less than this memory amount is available // caching buffer private IntBytesMap cacheHeaders; // the cache; holds overhead values and key element @@ -120,9 +120,9 @@ public class CachedRecords extends AbstractRecords implements RandomAccessRecord } } - int cacheGrowStatus() { + public int cacheGrowStatus() { final long available = MemoryControl.available(); - if ((cacheHeaders != null) && (available < cacheHeaders.memoryNeededForGrow())) return 0; + if ((cacheHeaders != null) && (available - 2 * 1024 * 1024 < cacheHeaders.memoryNeededForGrow())) return 0; return cacheGrowStatus(available, memStopGrow, memStartShrink); } @@ -133,18 +133,13 @@ public class CachedRecords extends AbstractRecords implements RandomAccessRecord // 2: cache is allowed to grow and must not shrink if (available > stopGrow) return 2; if (available > startShrink) { - MemoryControl.gc(30000, "kelendroCacheRecords.cacheGrowStatus(...) 1"); // thq + MemoryControl.gc(60000, "kelendroCacheRecords.cacheGrowStatus(...) 1"); // thq return 1; } - MemoryControl.gc(3000, "kelendroCacheRecords.cacheGrowStatus(...) 0"); // thq + MemoryControl.gc(30000, "kelendroCacheRecords.cacheGrowStatus(...) 0"); // thq return 0; } - public static void setCacheGrowStati(final long memStopGrowNew, final long memStartShrinkNew) { - memStopGrow = memStopGrowNew; - memStartShrink = memStartShrinkNew; - } - public static long getMemStopGrow() { return memStopGrow ; } diff --git a/source/de/anomic/kelondro/util/MemoryControl.java b/source/de/anomic/kelondro/util/MemoryControl.java index 01ffdf140..f86a39da8 100644 --- a/source/de/anomic/kelondro/util/MemoryControl.java +++ b/source/de/anomic/kelondro/util/MemoryControl.java @@ -46,6 +46,7 @@ public class MemoryControl { * @param info additional info for log */ public final synchronized static void gc(final int last, final String info) { // thq + assert last >= 10000; // too many forced GCs will cause bad execution performance final long elapsed = System.currentTimeMillis() - lastGC; if (elapsed > last) { final long before = free(); @@ -71,7 +72,7 @@ public class MemoryControl { */ private static long runGC(final boolean count) { final long memBefore = available(); - gc(1000, "serverMemory.runGC(...)"); + gc(10000, "serverMemory.runGC(...)"); final long freed = available() - memBefore; if (count) { gcs[gcs_pos] = freed; @@ -159,7 +160,7 @@ public class MemoryControl { final long avg = getAverageGCFree(); if (force || avg == 0 || avg + avail >= size) { // this is only called if we expect that an allocation of bytes would cause the jvm to call the GC anyway - final long freed = runGC(!force); + final long freed = runGC(false); avail = available(); log.logInfo("performed " + ((force) ? "explicit" : "necessary") + " GC, freed " + (freed >> 10) + " KB (requested/available/average: " diff --git a/source/de/anomic/plasma/plasmaRankingCRProcess.java b/source/de/anomic/plasma/plasmaRankingCRProcess.java index 5b979e790..2b536c329 100644 --- a/source/de/anomic/plasma/plasmaRankingCRProcess.java +++ b/source/de/anomic/plasma/plasmaRankingCRProcess.java @@ -365,7 +365,6 @@ public class plasmaRankingCRProcess { // finished. write to file cr = null; cr_in = null; - MemoryControl.gc(1000, "plasmaRankingCRProcess.genrci(...)"); // thq rci.toFile(rci_out); return count; } diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 3b9a6162d..7d5ac923e 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -103,8 +103,6 @@ import java.util.Locale; import java.util.Map; import java.util.Properties; import java.util.Set; -import java.util.Timer; -import java.util.TimerTask; import java.util.TreeMap; import java.util.TreeSet; import java.util.regex.Pattern; @@ -143,11 +141,9 @@ import de.anomic.http.httpdRobotsTxtConfig; import de.anomic.index.indexDocumentMetadata; import de.anomic.index.indexReferenceBlacklist; import de.anomic.index.indexURLReference; -import de.anomic.kelondro.blob.Cache; import de.anomic.kelondro.order.DateFormatter; import de.anomic.kelondro.order.Digest; import de.anomic.kelondro.order.NaturalOrder; -import de.anomic.kelondro.table.CachedRecords; import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.Log; import de.anomic.kelondro.util.MemoryControl; @@ -236,7 +232,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch clusterhashes; // map of peerhash(String)/alternative-local-address as ip:port or only ip (String) or null if address in seed should be used public URLLicense licensedURLs; - public Timer moreMemory; public List networkWhitelist, networkBlacklist; public Dispatcher dhtDispatcher; public List trail; @@ -458,13 +453,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch