diff --git a/htroot/PerformanceMemory_p.html b/htroot/PerformanceMemory_p.html index 82f42a46c..78ddae6cd 100644 --- a/htroot/PerformanceMemory_p.html +++ b/htroot/PerformanceMemory_p.html @@ -81,7 +81,7 @@ Database
  DB Size
  -Node Cache +Node Cache Object Cache Memory Occupation (KBytes) Description
  @@ -91,10 +91,11 @@ Chunk Size
high/med/low
(bytes) Empty
(avail.)
Slots Used: High, Medium, Low Prio +Node-Cache
Hit:Miss
Uniq:Doub
Flush
 
Size
Max  
Size
Current -Cache
Hit:Miss
Unique:Double
Flush
-Miss
Hit:Miss
Unique:Double
Flush
+Hit-Cache
Hit:Miss
Uniq:Doub
Flush
+Miss-Cache
Hit:Miss
Uniq:Doub
Flush
Used Now Assigned Max Default Max @@ -108,6 +109,7 @@ #[chunkRWI]# #[slempRWI]# #[slhigRWI]#
#[slmedRWI]#
#[sllowRWI]# +#[slhittmissRWI]#
#[sluniqdoubRWI]#
#[slflushRWI]# #[ochunkmaxRWI]# #[ochunkcurRWI]# #[ohittmissRWI]#
#[ouniqdoubRWI]#
#[oflushRWI]# @@ -128,6 +130,7 @@ cache will speed up crawls with a depth > 3. #[chunkHTTP]# #[slempHTTP]# #[slhigHTTP]#
#[slmedHTTP]#
#[sllowHTTP]# +#[slhittmissHTTP]#
#[sluniqdoubHTTP]#
#[slflushHTTP]# #[ochunkmaxHTTP]# #[ochunkcurHTTP]# #[ohittmissHTTP]#
#[ouniqdoubHTTP]#
#[oflushHTTP]# @@ -148,6 +151,7 @@ Increasing this cache will be most important for a fast proxy mode. #[chunkLURL]# #[slempLURL]# #[slhigLURL]#
#[slmedLURL]#
#[sllowLURL]# +#[slhittmissLURL]#
#[sluniqdoubLURL]#
#[slflushLURL]# #[ochunkmaxLURL]# #[ochunkcurLURL]# #[ohittmissLURL]#
#[ouniqdoubLURL]#
#[oflushLURL]# @@ -167,6 +171,7 @@ This cache is very important for a fast search process. Increasing the cache siz #[chunkNURL]# #[slempNURL]# #[slhigNURL]#
#[slmedNURL]#
#[sllowNURL]# +#[slhittmissNURL]#
#[sluniqdoubNURL]#
#[slflushNURL]# #[ochunkmaxNURL]# #[ochunkcurNURL]# #[ohittmissNURL]#
#[ouniqdoubNURL]#
#[oflushNURL]# @@ -186,6 +191,7 @@ Increasing the cache size will result in faster double-check during URL recognit #[chunkEURL]# #[slempEURL]# #[slhigEURL]#
#[slmedEURL]#
#[sllowEURL]# +#[slhittmissEURL]#
#[sluniqdoubEURL]#
#[slflushEURL]# #[ochunkmaxEURL]# #[ochunkcurEURL]# #[ohittmissEURL]#
#[ouniqdoubEURL]#
#[oflushEURL]# @@ -205,6 +211,7 @@ Increasing the cache size will most probably speed up crawling slightly, but not #[chunkDHT]# #[slempDHT]# #[slhigDHT]#
#[slmedDHT]#
#[sllowDHT]# +#[slhittmissDHT]#
#[sluniqdoubDHT]#
#[slflushDHT]# #[ochunkmaxDHT]# #[ochunkcurDHT]# #[ohittmissDHT]#
#[ouniqdoubDHT]#
#[oflushDHT]# @@ -225,6 +232,7 @@ Increasing this cache may speed up many functions, but we need to test this to s #[chunkMessage]# #[slempMessage]# #[slhigMessage]#
#[slmedMessage]#
#[sllowMessage]# +#[slhittmissMessage]#
#[sluniqdoubMessage]#
#[slflushMessage]# #[ochunkmaxMessage]# #[ochunkcurMessage]# #[ohittmissMessage]#
#[ouniqdoubMessage]#
#[oflushMessage]# @@ -243,6 +251,7 @@ Increasing this cache may speed up many functions, but we need to test this to s #[chunkWiki]# #[slempWiki]# #[slhigWiki]#
#[slmedWiki]#
#[sllowWiki]# +#[slhittmissWiki]#
#[sluniqdoubWiki]#
#[slflushWiki]# #[ochunkmaxWiki]# #[ochunkcurWiki]# #[ohittmissWiki]#
#[ouniqdoubWiki]#
#[oflushWiki]# @@ -263,6 +272,7 @@ Increasing this cache may speed up access to the wiki pages. #[chunkBlog]# #[slempBlog]# #[slhigBlog]#
#[slmedBlog]#
#[sllowBlog]# +#[slhittmissBlog]#
#[sluniqdoubBlog]#
#[slflushBlog]# #[ochunkmaxBlog]# #[ochunkcurBlog]# #[ohittmissBlog]#
#[ouniqdoubBlog]#
#[oflushBlog]# @@ -282,6 +292,7 @@ Increasing this cache may speed up access to the Blog. #[chunkNews]# #[slempNews]# #[slhigNews]#
#[slmedNews]#
#[sllowNews]# +#[slhittmissNews]#
#[sluniqdoubNews]#
#[slflushNews]# #[ochunkmaxNews]# #[ochunkcurNews]# #[ohittmissNews]#
#[ouniqdoubNews]#
#[oflushNews]# @@ -301,6 +312,7 @@ Increasing this cache may speed up the peer-ping. #[chunkRobots]# #[slempRobots]# #[slhigRobots]#
#[slmedRobots]#
#[sllowRobots]# +#[slhittmissRobots]#
#[sluniqdoubRobots]#
#[slflushRobots]# #[ochunkmaxRobots]# #[ochunkcurRobots]# #[ohittmissRobots]#
#[ouniqdoubRobots]#
#[oflushRobots]# @@ -320,6 +332,7 @@ Increasing this cache may speed up validation if crawling of the URL is allowed. #[chunkProfiles]# #[slempProfiles]# #[slhigProfiles]#
#[slmedProfiles]#
#[sllowProfiles]# +#[slhittmissProfiles]#
#[sluniqdoubProfiles]#
#[slflushProfiles]# #[ochunkmaxProfiles]# #[ochunkcurProfiles]# #[ohittmissProfiles]#
#[ouniqdoubProfiles]#
#[oflushProfiles]# @@ -334,7 +347,7 @@ Increasing this cache may speed up crawling, but not much space is needed, so th -Totals +Totals #[usedTotal]# MB #[currTotal]# MB #[dfltTotal]# MB @@ -344,7 +357,7 @@ Increasing this cache may speed up crawling, but not much space is needed, so th -Re-Configuration: +Re-Configuration:
these
custom
values
 
all
default
values
 
all
recom-
mended values diff --git a/htroot/PerformanceMemory_p.java b/htroot/PerformanceMemory_p.java index 987b28fa3..b7a9842af 100644 --- a/htroot/PerformanceMemory_p.java +++ b/htroot/PerformanceMemory_p.java @@ -48,9 +48,7 @@ import java.util.Map; import java.io.File; import de.anomic.http.httpHeader; -import de.anomic.index.indexURL; import de.anomic.plasma.plasmaSwitchboard; -import de.anomic.server.serverMemory; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.server.serverFileUtils; @@ -166,73 +164,73 @@ public class PerformanceMemory_p { req = sb.wordIndex.size(); chk = sb.wordIndex.assortmentsCacheChunkSizeAvg(); - slt = sb.wordIndex.assortmentsCacheFillStatusCml(); + slt = sb.wordIndex.assortmentsCacheNodeStatus(); ost = sb.wordIndex.assortmentsCacheObjectStatus(); putprop(prop, env, "RWI", set); req = sb.cacheManager.dbSize(); chk = sb.cacheManager.dbCacheChunkSize(); - slt = sb.cacheManager.dbCacheFillStatus(); + slt = sb.cacheManager.dbCacheStatus(); ost = sb.cacheManager.dbCacheObjectStatus(); putprop(prop, env, "HTTP", set); req = sb.urlPool.loadedURL.size(); chk = sb.urlPool.loadedURL.cacheNodeChunkSize(); - slt = sb.urlPool.loadedURL.cacheNodeFillStatus(); + slt = sb.urlPool.loadedURL.cacheNodeStatus(); ost = sb.urlPool.loadedURL.cacheObjectStatus(); putprop(prop, env, "LURL", set); req = sb.urlPool.noticeURL.size(); chk = sb.urlPool.noticeURL.cacheNodeChunkSize(); - slt = sb.urlPool.noticeURL.cacheNodeFillStatus(); + slt = sb.urlPool.noticeURL.cacheNodeStatus(); ost = sb.urlPool.noticeURL.cacheObjectStatus(); putprop(prop, env, "NURL", set); req = sb.urlPool.errorURL.size(); chk = sb.urlPool.errorURL.cacheNodeChunkSize(); - slt = sb.urlPool.errorURL.cacheNodeFillStatus(); + slt = sb.urlPool.errorURL.cacheNodeStatus(); ost = sb.urlPool.errorURL.cacheObjectStatus(); putprop(prop, env, "EURL", set); req = yacyCore.seedDB.sizeConnected() + yacyCore.seedDB.sizeDisconnected() + yacyCore.seedDB.sizePotential(); chk = yacyCore.seedDB.dbCacheNodeChunkSize(); - slt = yacyCore.seedDB.dbCacheNodeFillStatus(); + slt = yacyCore.seedDB.dbCacheNodeStatus(); ost = yacyCore.seedDB.dbCacheObjectStatus(); putprop(prop, env, "DHT", set); req = sb.messageDB.size(); chk = sb.messageDB.dbCacheNodeChunkSize(); - slt = sb.messageDB.dbCacheNodeFillStatus(); + slt = sb.messageDB.dbCacheNodeStatus(); ost = sb.messageDB.dbCacheObjectStatus(); putprop(prop, env, "Message", set); req = sb.wikiDB.sizeOfTwo(); chk = sb.wikiDB.dbCacheNodeChunkSize(); - slt = sb.wikiDB.dbCacheNodeFillStatus(); + slt = sb.wikiDB.dbCacheNodeStatus(); ost = sb.wikiDB.dbCacheObjectStatus(); putprop(prop, env, "Wiki", set); req = sb.blogDB.size(); chk = sb.blogDB.dbCacheNodeChunkSize(); - slt = sb.blogDB.dbCacheNodeFillStatus(); + slt = sb.blogDB.dbCacheNodeStatus(); ost = sb.blogDB.dbCacheObjectStatus(); putprop(prop, env, "Blog", set); req = yacyCore.newsPool.dbSize(); chk = yacyCore.newsPool.dbCacheNodeChunkSize(); - slt = yacyCore.newsPool.dbCacheNodeFillStatus(); + slt = yacyCore.newsPool.dbCacheNodeStatus(); ost = yacyCore.newsPool.dbCacheObjectStatus(); putprop(prop, env, "News", set); req = plasmaSwitchboard.robots.size(); chk = plasmaSwitchboard.robots.dbCacheNodeChunkSize(); - slt = plasmaSwitchboard.robots.dbCacheNodeFillStatus(); + slt = plasmaSwitchboard.robots.dbCacheNodeStatus(); ost = plasmaSwitchboard.robots.dbCacheObjectStatus(); putprop(prop, env, "Robots", set); req = sb.profiles.size(); chk = sb.profiles.dbCacheNodeChunkSize(); - slt = sb.profiles.dbCacheNodeFillStatus(); + slt = sb.profiles.dbCacheNodeStatus(); ost = sb.profiles.dbCacheObjectStatus(); putprop(prop, env, "Profiles", set); @@ -296,10 +294,13 @@ public class PerformanceMemory_p { if (set.equals("setBest")) env.setConfig("ramCache" + db, bst); prop.put("chunk" + db, chk[2] + "/" + chk[1] + "/" + chk[0]); prop.put("slreq" + db, req); - prop.put("slemp" + db, slt[0]); + prop.put("slemp" + db, slt[0] - slt[1] - slt[2] - slt[3]); prop.put("slhig" + db, slt[1]); prop.put("slmed" + db, slt[2]); prop.put("sllow" + db, slt[3]); + prop.put("slhittmiss" + db, slt[4] + ":" + slt[5]); + prop.put("sluniqdoub" + db, slt[6] + ":" + slt[7]); + prop.put("slflush" + db, slt[8]); prop.put("ochunkmax" + db, ost[0]); prop.put("ochunkcur" + db, ost[1]); prop.put("ohittmiss" + db, ost[5] + ":" + ost[6]); diff --git a/source/de/anomic/data/blogBoard.java b/source/de/anomic/data/blogBoard.java index 69fcc7cce..fa495d26a 100644 --- a/source/de/anomic/data/blogBoard.java +++ b/source/de/anomic/data/blogBoard.java @@ -90,8 +90,8 @@ public class blogBoard { return datbase.cacheNodeChunkSize(); } - public int[] dbCacheNodeFillStatus() { - return datbase.cacheNodeFillStatus(); + public int[] dbCacheNodeStatus() { + return datbase.cacheNodeStatus(); } public String[] dbCacheObjectStatus() { diff --git a/source/de/anomic/data/messageBoard.java b/source/de/anomic/data/messageBoard.java index 81bf0feb8..93fc724c2 100644 --- a/source/de/anomic/data/messageBoard.java +++ b/source/de/anomic/data/messageBoard.java @@ -90,8 +90,8 @@ public class messageBoard { return database.cacheNodeChunkSize(); } - public int[] dbCacheNodeFillStatus() { - return database.cacheNodeFillStatus(); + public int[] dbCacheNodeStatus() { + return database.cacheNodeStatus(); } public String[] dbCacheObjectStatus() { diff --git a/source/de/anomic/data/userDB.java b/source/de/anomic/data/userDB.java index e2c8caa6c..aeb5265b5 100644 --- a/source/de/anomic/data/userDB.java +++ b/source/de/anomic/data/userDB.java @@ -95,8 +95,8 @@ public final class userDB { return userTable.cacheNodeChunkSize(); } - public int[] dbCacheNodeFillStatus() { - return userTable.cacheNodeFillStatus(); + public int[] dbCacheNodeStatus() { + return userTable.cacheNodeStatus(); } void resetDatabase() { diff --git a/source/de/anomic/data/wikiBoard.java b/source/de/anomic/data/wikiBoard.java index 74ab30be9..48a02d610 100644 --- a/source/de/anomic/data/wikiBoard.java +++ b/source/de/anomic/data/wikiBoard.java @@ -110,10 +110,10 @@ public class wikiBoard { return i; } - public int[] dbCacheNodeFillStatus() { - int[] a = datbase.cacheNodeFillStatus(); - int[] b = bkpbase.cacheNodeFillStatus(); - return new int[]{a[0] + b[0], a[1] + b[1], a[2] + b[2], a[3] + b[3]}; + public int[] dbCacheNodeStatus() { + int[] a = datbase.cacheNodeStatus(); + int[] b = bkpbase.cacheNodeStatus(); + return kelondroRecords.cacheCombinedStatus(new int[][]{a, b}, 2); } public String[] dbCacheObjectStatus() { diff --git a/source/de/anomic/index/indexURL.java b/source/de/anomic/index/indexURL.java index 83f72d652..8292fb2d1 100644 --- a/source/de/anomic/index/indexURL.java +++ b/source/de/anomic/index/indexURL.java @@ -434,8 +434,8 @@ public class indexURL { return urlHashCache.cacheNodeChunkSize(); } - public int[] cacheNodeFillStatus() { - return urlHashCache.cacheNodeFillStatus(); + public int[] cacheNodeStatus() { + return urlHashCache.cacheNodeStatus(); } public String[] cacheObjectStatus() { diff --git a/source/de/anomic/kelondro/kelondroMap.java b/source/de/anomic/kelondro/kelondroMap.java index 38b508daf..44417f564 100644 --- a/source/de/anomic/kelondro/kelondroMap.java +++ b/source/de/anomic/kelondro/kelondroMap.java @@ -137,8 +137,8 @@ public class kelondroMap { return dyn.cacheNodeChunkSize(); } - public int[] cacheNodeFillStatus() { - return dyn.cacheNodeFillStatus(); + public int[] cacheNodeStatus() { + return dyn.cacheNodeStatus(); } public String[] cacheObjectStatus() { diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java index 2adec2b78..474699d00 100644 --- a/source/de/anomic/kelondro/kelondroRecords.java +++ b/source/de/anomic/kelondro/kelondroRecords.java @@ -149,11 +149,12 @@ public class kelondroRecords { private int TXTPROPW; // size of a single TXTPROPS element // caching buffer - protected HashMap[] XcacheHeaders; // the cache; holds overhead values and key element - protected int XcacheSize; // number of cache records - protected long XcacheStartup; // startup time; for cache aging - protected kelondroMScoreCluster cacheScore; // controls cache aging - + private HashMap[] cacheHeaders; // the cache; holds overhead values and key element + private int cacheSize; // number of cache records + private long cacheStartup; // startup time; for cache aging + private kelondroMScoreCluster cacheScore; // controls cache aging + private int readHit, readMiss, writeUnique, writeDouble, cacheFlush; + // optional logger protected Logger theLogger = null; @@ -423,20 +424,25 @@ public class kelondroRecords { private void initCache(long buffersize) { if (buffersize <= 0) { - this.XcacheSize = 0; - this.XcacheHeaders = null; + this.cacheSize = 0; + this.cacheHeaders = null; this.cacheScore = null; } else { if ((buffersize / cacheNodeChunkSize(false)) > size()) { - this.XcacheSize = (int) (buffersize / cacheNodeChunkSize(false)); + this.cacheSize = (int) (buffersize / cacheNodeChunkSize(false)); this.cacheScore = null; // no cache control because we have more cache slots than database entries } else { - this.XcacheSize = (int) (buffersize / cacheNodeChunkSize(true)); + this.cacheSize = (int) (buffersize / cacheNodeChunkSize(true)); this.cacheScore = new kelondroMScoreCluster(); // cache control of CP_HIGH caches } - this.XcacheHeaders = new HashMap[]{new HashMap(), new HashMap(), new HashMap()}; + this.cacheHeaders = new HashMap[]{new HashMap(), new HashMap(), new HashMap()}; } - this.XcacheStartup = System.currentTimeMillis(); + this.cacheStartup = System.currentTimeMillis(); + this.readHit = 0; + this.readMiss = 0; + this.writeUnique = 0; + this.writeDouble = 0; + this.cacheFlush = 0; } private static final long max = Runtime.getRuntime().maxMemory(); @@ -468,9 +474,32 @@ public class kelondroRecords { return i; } - public int[] cacheNodeFillStatus() { - if (XcacheHeaders == null) return new int[]{0,0,0,0}; - return new int[]{XcacheSize - (XcacheHeaders[CP_HIGH].size() + XcacheHeaders[CP_MEDIUM].size() + XcacheHeaders[CP_LOW].size()), XcacheHeaders[CP_HIGH].size(), XcacheHeaders[CP_MEDIUM].size(), XcacheHeaders[CP_LOW].size()}; + public int[] cacheNodeStatus() { + if (cacheHeaders == null) return new int[]{0,0,0,0,0,0,0,0,0}; + return new int[]{ + cacheSize, + cacheHeaders[CP_HIGH].size(), + cacheHeaders[CP_MEDIUM].size(), + cacheHeaders[CP_LOW].size(), + readHit, + readMiss, + writeUnique, + writeDouble, + cacheFlush + }; + } + + private static int[] cacheCombinedStatus(int[] a, int[] b) { + int[] c = new int[a.length]; + for (int i = a.length - 1; i >= 0; i--) c[i] = a[i] + b[i]; + return c; + } + + public static int[] cacheCombinedStatus(int[][] a, int l) { + if ((a == null) || (a.length == 0) || (l == 0)) return null; + if ((a.length >= 1) && (l == 1)) return a[0]; + if ((a.length >= 2) && (l == 2)) return cacheCombinedStatus(a[0], a[1]); + return cacheCombinedStatus(cacheCombinedStatus(a, l - 1), a[l - 1]); } protected Node newNode() throws IOException { @@ -486,23 +515,24 @@ public class kelondroRecords { } protected void deleteNode(Handle handle) throws IOException { - if (XcacheSize != 0) { - synchronized (XcacheHeaders) { + if (cacheSize != 0) { + synchronized (cacheHeaders) { if (cacheScore == null) { - XcacheHeaders[CP_LOW].remove(handle); - XcacheHeaders[CP_MEDIUM].remove(handle); - XcacheHeaders[CP_HIGH].remove(handle); - } else if (XcacheHeaders[CP_HIGH].get(handle) != null) { + cacheHeaders[CP_LOW].remove(handle); + cacheHeaders[CP_MEDIUM].remove(handle); + cacheHeaders[CP_HIGH].remove(handle); + } else if (cacheHeaders[CP_HIGH].get(handle) != null) { // remove handle from cache-control cacheScore.deleteScore(handle); - XcacheHeaders[CP_HIGH].remove(handle); - } else if (XcacheHeaders[CP_MEDIUM].get(handle) != null) { + cacheHeaders[CP_HIGH].remove(handle); + } else if (cacheHeaders[CP_MEDIUM].get(handle) != null) { // no cache control for medium-priority entries - XcacheHeaders[CP_MEDIUM].remove(handle); - } else if (XcacheHeaders[CP_LOW].get(handle) != null) { + cacheHeaders[CP_MEDIUM].remove(handle); + } else if (cacheHeaders[CP_LOW].get(handle) != null) { // no cache control for low-priority entries - XcacheHeaders[CP_LOW].remove(handle); + cacheHeaders[CP_LOW].remove(handle); } + cacheFlush++; } } dispose(handle); @@ -605,26 +635,27 @@ public class kelondroRecords { private void initContent() throws IOException { // create chunks; read them from file or cache this.tailChunk = null; - if (XcacheSize == 0) { + if (cacheSize == 0) { // read overhead and key //System.out.println("**NO CACHE for " + this.handle.index + "**"); this.headChunk = new byte[headchunksize]; entryFile.readFully(seekpos(this.handle), this.headChunk, 0, this.headChunk.length); this.headChanged = false; - } else synchronized(XcacheHeaders) { + } else synchronized(cacheHeaders) { byte[] cacheEntry = null; int cp = CP_HIGH; - cacheEntry = (byte[]) XcacheHeaders[CP_HIGH].get(this.handle); // first try + cacheEntry = (byte[]) cacheHeaders[CP_HIGH].get(this.handle); // first try if (cacheEntry == null) { - cacheEntry = (byte[]) XcacheHeaders[CP_MEDIUM].get(this.handle); // second try + cacheEntry = (byte[]) cacheHeaders[CP_MEDIUM].get(this.handle); // second try cp = CP_MEDIUM; } if (cacheEntry == null) { - cacheEntry = (byte[]) XcacheHeaders[CP_LOW].get(this.handle); // third try + cacheEntry = (byte[]) cacheHeaders[CP_LOW].get(this.handle); // third try cp = CP_LOW; } if (cacheEntry == null) { // cache miss, we read overhead and key from file + readMiss++; //System.out.println("**CACHE miss for " + this.handle.index + "**"); this.headChunk = new byte[headchunksize]; //this.tailChunk = new byte[tailchunksize]; @@ -642,12 +673,13 @@ public class kelondroRecords { update2Cache(cp); } else { // cache hit, copy overhead and key from cache + readHit++; //System.out.println("**CACHE HIT for " + this.handle.index + "**"); this.headChunk = new byte[headchunksize]; System.arraycopy(cacheEntry, 0, this.headChunk, 0, headchunksize); // update cache scores to announce this cache hit if ((cacheScore != null) && (cp == CP_HIGH)) { - cacheScore.setScore(this.handle, (int) ((System.currentTimeMillis() - XcacheStartup) / 1000)); + cacheScore.setScore(this.handle, (int) ((System.currentTimeMillis() - cacheStartup) / 1000)); } this.headChanged = false; } @@ -816,10 +848,10 @@ public class kelondroRecords { } private void update2Cache(int forPriority) { - if (XcacheSize > 0) { - XcacheHeaders[CP_LOW].remove(this.handle); - XcacheHeaders[CP_MEDIUM].remove(this.handle); - XcacheHeaders[CP_HIGH].remove(this.handle); + if (cacheSize > 0) { + cacheHeaders[CP_LOW].remove(this.handle); + cacheHeaders[CP_MEDIUM].remove(this.handle); + cacheHeaders[CP_HIGH].remove(this.handle); } if (cacheSpace(forPriority)) updateNodeCache(forPriority); } @@ -830,34 +862,37 @@ public class kelondroRecords { // returns true if it is allowed to add another entry to the cache // returns false if the cache is considered to be full if (forPriority == CP_NONE) return false; - if (XcacheSize == 0) return false; // no caching - long cs = XcacheHeaders[CP_LOW].size() + XcacheHeaders[CP_MEDIUM].size() + XcacheHeaders[CP_HIGH].size(); + if (cacheSize == 0) return false; // no caching + long cs = cacheHeaders[CP_LOW].size() + cacheHeaders[CP_MEDIUM].size() + cacheHeaders[CP_HIGH].size(); if (cs == 0) return true; // nothing there to flush - if ((cs < XcacheSize) && (availableMemory() >= memBlock)) return true; // no need to flush cache space + if ((cs < cacheSize) && (availableMemory() >= memBlock)) return true; // no need to flush cache space Handle delkey; // delete one entry. distinguish between different priority cases: if (forPriority == CP_LOW) { // remove only from low-priority cache - if (XcacheHeaders[CP_LOW].size() != 0) { + if (cacheHeaders[CP_LOW].size() != 0) { // just delete any of the low-priority entries - delkey = (Handle) XcacheHeaders[CP_LOW].keySet().iterator().next(); - XcacheHeaders[CP_LOW].remove(delkey); + delkey = (Handle) cacheHeaders[CP_LOW].keySet().iterator().next(); + cacheHeaders[CP_LOW].remove(delkey); + cacheFlush++; return true; } else { // we cannot delete any entry, therefore there is no space for another entry return false; } } else if (forPriority == CP_MEDIUM) { - if (XcacheHeaders[CP_LOW].size() != 0) { + if (cacheHeaders[CP_LOW].size() != 0) { // just delete any of the low-priority entries - delkey = (Handle) XcacheHeaders[CP_LOW].keySet().iterator().next(); - XcacheHeaders[CP_LOW].remove(delkey); + delkey = (Handle) cacheHeaders[CP_LOW].keySet().iterator().next(); + cacheHeaders[CP_LOW].remove(delkey); + cacheFlush++; return true; - } else if (XcacheHeaders[CP_MEDIUM].size() != 0) { + } else if (cacheHeaders[CP_MEDIUM].size() != 0) { // just delete any of the medium-priority entries - delkey = (Handle) XcacheHeaders[CP_MEDIUM].keySet().iterator().next(); - XcacheHeaders[CP_MEDIUM].remove(delkey); + delkey = (Handle) cacheHeaders[CP_MEDIUM].keySet().iterator().next(); + cacheHeaders[CP_MEDIUM].remove(delkey); + cacheFlush++; return true; } else { // we cannot delete any entry, therefore there is no space for another entry @@ -865,15 +900,17 @@ public class kelondroRecords { } } else { // request for a high-priority entry - if (XcacheHeaders[CP_LOW].size() != 0) { + if (cacheHeaders[CP_LOW].size() != 0) { // just delete any of the low-priority entries - delkey = (Handle) XcacheHeaders[CP_LOW].keySet().iterator().next(); - XcacheHeaders[CP_LOW].remove(delkey); + delkey = (Handle) cacheHeaders[CP_LOW].keySet().iterator().next(); + cacheHeaders[CP_LOW].remove(delkey); + cacheFlush++; return true; - } else if (XcacheHeaders[CP_MEDIUM].size() != 0) { + } else if (cacheHeaders[CP_MEDIUM].size() != 0) { // just delete any of the medium-priority entries - delkey = (Handle) XcacheHeaders[CP_MEDIUM].keySet().iterator().next(); - XcacheHeaders[CP_MEDIUM].remove(delkey); + delkey = (Handle) cacheHeaders[CP_MEDIUM].keySet().iterator().next(); + cacheHeaders[CP_MEDIUM].remove(delkey); + cacheFlush++; return true; } else if (cacheScore == null) { // no cache-control of high-priority cache @@ -884,16 +921,17 @@ public class kelondroRecords { // use the cache-control to find the right object delkey = (Handle) cacheScore.getMinObject(); cacheScore.deleteScore(delkey); - XcacheHeaders[CP_HIGH].remove(delkey); + cacheHeaders[CP_HIGH].remove(delkey); + cacheFlush++; return true; } catch (NoSuchElementException e) { // this is a strange error and could be caused by internal java problems // we simply clear the cache - String error = "cachScore error: " + e.getMessage() + "; cachesize=" + XcacheSize + ", cache.size()=[" + XcacheHeaders[0].size() + "," + XcacheHeaders[1].size() + "," + XcacheHeaders[2].size() + "], cacheScore.size()=" + cacheScore.size(); + String error = "cachScore error: " + e.getMessage() + "; cachesize=" + cacheSize + ", cache.size()=[" + cacheHeaders[0].size() + "," + cacheHeaders[1].size() + "," + cacheHeaders[2].size() + "], cacheScore.size()=" + cacheScore.size(); cacheScore = new kelondroMScoreCluster(); - XcacheHeaders[CP_LOW] = new HashMap(); - XcacheHeaders[CP_MEDIUM] = new HashMap(); - XcacheHeaders[CP_HIGH] = new HashMap(); + cacheHeaders[CP_LOW] = new HashMap(); + cacheHeaders[CP_MEDIUM] = new HashMap(); + cacheHeaders[CP_HIGH] = new HashMap(); throw new kelondroException(filename, error); } @@ -904,25 +942,26 @@ public class kelondroRecords { if (this.handle == null) return; // wrong access if (this.headChunk == null) return; // nothing there to cache if (priority == CP_NONE) return; // it is not wanted that this shall be cached - if (XcacheSize == 0) return; // we do not use the cache - int cs = XcacheHeaders[CP_LOW].size() + XcacheHeaders[CP_MEDIUM].size() + XcacheHeaders[CP_HIGH].size(); - if (cs >= XcacheSize) return; // no cache update if cache is full + if (cacheSize == 0) return; // we do not use the cache + int cs = cacheHeaders[CP_LOW].size() + cacheHeaders[CP_MEDIUM].size() + cacheHeaders[CP_HIGH].size(); + if (cs >= cacheSize) return; // no cache update if cache is full - synchronized (XcacheHeaders) { + synchronized (cacheHeaders) { // generate cache entry byte[] cacheEntry = new byte[headchunksize]; System.arraycopy(headChunk, 0, cacheEntry, 0, headchunksize); Handle cacheHandle = new Handle(this.handle.index); // store the cache entry - //XcacheHeaders.remove(cacheHandle); - if (priority != CP_LOW) XcacheHeaders[CP_LOW].remove(cacheHandle); - if (priority != CP_MEDIUM) XcacheHeaders[CP_MEDIUM].remove(cacheHandle); - if (priority != CP_HIGH) XcacheHeaders[CP_HIGH].remove(cacheHandle); - XcacheHeaders[priority].put(cacheHandle, cacheEntry); + boolean upd = false; + if (priority != CP_LOW) upd = upd || (cacheHeaders[CP_LOW].remove(cacheHandle) != null); + if (priority != CP_MEDIUM) upd = upd || (cacheHeaders[CP_MEDIUM].remove(cacheHandle) != null); + if (priority != CP_HIGH) upd = upd || (cacheHeaders[CP_HIGH].remove(cacheHandle) != null); + cacheHeaders[priority].put(cacheHandle, cacheEntry); if ((cacheScore != null) && (priority == CP_HIGH)) { - cacheScore.setScore(cacheHandle, (int) ((System.currentTimeMillis() - XcacheStartup) / 1000)); + cacheScore.setScore(cacheHandle, (int) ((System.currentTimeMillis() - cacheStartup) / 1000)); } + if (upd) writeDouble++; else writeUnique++; // delete the cache entry buffer cacheEntry = null; @@ -935,7 +974,7 @@ public class kelondroRecords { } protected void printCache() { - if (XcacheSize == 0) { + if (cacheSize == 0) { System.out.println("### file report: " + size() + " entries"); for (int i = 0; i < size() + 3; i++) { // print from file to compare @@ -948,9 +987,9 @@ public class kelondroRecords { System.out.println(); } } else { - System.out.println("### cache report: [" + XcacheHeaders[0].size() + "," + XcacheHeaders[0].size() + "," + XcacheHeaders[0].size() + "] entries"); + System.out.println("### cache report: [" + cacheHeaders[0].size() + "," + cacheHeaders[0].size() + "," + cacheHeaders[0].size() + "] entries"); for (int cp = 0; cp < 3; cp++) { - Iterator i = XcacheHeaders[cp].entrySet().iterator(); + Iterator i = cacheHeaders[cp].entrySet().iterator(); Map.Entry entry; while (i.hasNext()) { entry = (Map.Entry) i.next(); diff --git a/source/de/anomic/kelondro/kelondroStack.java b/source/de/anomic/kelondro/kelondroStack.java index 0e28257f8..482522ee1 100644 --- a/source/de/anomic/kelondro/kelondroStack.java +++ b/source/de/anomic/kelondro/kelondroStack.java @@ -99,7 +99,7 @@ public final class kelondroStack extends kelondroRecords { public static kelondroStack reset(kelondroStack stack) { // memorize settings to this file File f = new File(stack.filename); - long bz = stack.XcacheSize * stack.cacheNodeChunkSize(true); + long bz = stack.cacheNodeStatus()[0] * stack.cacheNodeChunkSize(true); int[] cols = stack.COLWIDTHS; // close and delete the file diff --git a/source/de/anomic/plasma/plasmaCrawlProfile.java b/source/de/anomic/plasma/plasmaCrawlProfile.java index 74c33429a..c5f0050a1 100644 --- a/source/de/anomic/plasma/plasmaCrawlProfile.java +++ b/source/de/anomic/plasma/plasmaCrawlProfile.java @@ -82,8 +82,8 @@ public class plasmaCrawlProfile { return profileTable.cacheNodeChunkSize(); } - public int[] dbCacheNodeFillStatus() { - return profileTable.cacheNodeFillStatus(); + public int[] dbCacheNodeStatus() { + return profileTable.cacheNodeStatus(); } public String[] dbCacheObjectStatus() { diff --git a/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java b/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java index 7d5c4bd90..75e25a505 100644 --- a/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java +++ b/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java @@ -89,8 +89,8 @@ public class plasmaCrawlRobotsTxt { return robotsTable.cacheNodeChunkSize(); } - public int[] dbCacheNodeFillStatus() { - return robotsTable.cacheNodeFillStatus(); + public int[] dbCacheNodeStatus() { + return robotsTable.cacheNodeStatus(); } public String[] dbCacheObjectStatus() { diff --git a/source/de/anomic/plasma/plasmaHTCache.java b/source/de/anomic/plasma/plasmaHTCache.java index 1d5387cc2..da77e5430 100644 --- a/source/de/anomic/plasma/plasmaHTCache.java +++ b/source/de/anomic/plasma/plasmaHTCache.java @@ -207,8 +207,8 @@ public final class plasmaHTCache { return this.responseHeaderDB.cacheNodeChunkSize(); } - public int[] dbCacheFillStatus() { - return this.responseHeaderDB.cacheNodeFillStatus(); + public int[] dbCacheStatus() { + return this.responseHeaderDB.cacheNodeStatus(); } public String[] dbCacheObjectStatus() { diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index a3ed0b87d..e4cfb51d8 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -130,8 +130,8 @@ public final class plasmaWordIndex { return assortmentCluster.cacheChunkSizeAvg(); } - public int[] assortmentsCacheFillStatusCml() { - return assortmentCluster.cacheFillStatusCml(); + public int[] assortmentsCacheNodeStatus() { + return assortmentCluster.cacheNodeStatus(); } public String[] assortmentsCacheObjectStatus() { diff --git a/source/de/anomic/plasma/plasmaWordIndexAssortment.java b/source/de/anomic/plasma/plasmaWordIndexAssortment.java index 1be8b0357..6201051af 100644 --- a/source/de/anomic/plasma/plasmaWordIndexAssortment.java +++ b/source/de/anomic/plasma/plasmaWordIndexAssortment.java @@ -266,8 +266,8 @@ public final class plasmaWordIndexAssortment { return assortments.cacheNodeChunkSize(); } - public int[] cacheNodeFillStatus() { - return assortments.cacheNodeFillStatus(); + public int[] cacheNodeStatus() { + return assortments.cacheNodeStatus(); } public String[] dbCacheObjectStatus() { diff --git a/source/de/anomic/plasma/plasmaWordIndexAssortmentCluster.java b/source/de/anomic/plasma/plasmaWordIndexAssortmentCluster.java index 5ca622dc9..19c04811e 100644 --- a/source/de/anomic/plasma/plasmaWordIndexAssortmentCluster.java +++ b/source/de/anomic/plasma/plasmaWordIndexAssortmentCluster.java @@ -278,18 +278,15 @@ public final class plasmaWordIndexAssortmentCluster { return a; } - public int[] cacheFillStatusCml() { - int[] a, cml = new int[]{0, 0, 0, 0}; - for (int i = 0; i < clusterCount; i++) { - a = assortments[i].cacheNodeFillStatus(); - for (int j = 0; j < 4; j++) cml[j] += a[j]; - } - return cml; + public int[] cacheNodeStatus() { + int[][] a = new int[assortments.length][]; + for (int i = assortments.length - 1; i >= 0; i--) a[i] = assortments[i].cacheNodeStatus(); + return kelondroRecords.cacheCombinedStatus(a, assortments.length); } public String[] cacheObjectStatus() { String[][] a = new String[assortments.length][]; - for (int i = 0; i < assortments.length; i++) a[i] = assortments[i].dbCacheObjectStatus(); + for (int i = assortments.length - 1; i >= 0; i--) a[i] = assortments[i].dbCacheObjectStatus(); return kelondroObjectCache.combinedStatus(a, a.length); } diff --git a/source/de/anomic/yacy/yacyNewsDB.java b/source/de/anomic/yacy/yacyNewsDB.java index 86fe75786..f5741779a 100644 --- a/source/de/anomic/yacy/yacyNewsDB.java +++ b/source/de/anomic/yacy/yacyNewsDB.java @@ -100,8 +100,8 @@ public class yacyNewsDB { return news.cacheNodeChunkSize(); } - public int[] dbCacheNodeFillStatus() { - return news.cacheNodeFillStatus(); + public int[] dbCacheNodeStatus() { + return news.cacheNodeStatus(); } public String[] dbCacheObjectStatus() { diff --git a/source/de/anomic/yacy/yacyNewsPool.java b/source/de/anomic/yacy/yacyNewsPool.java index c161c0105..795e6e8a9 100644 --- a/source/de/anomic/yacy/yacyNewsPool.java +++ b/source/de/anomic/yacy/yacyNewsPool.java @@ -110,8 +110,8 @@ public class yacyNewsPool { return newsDB.dbCacheNodeChunkSize(); } - public int[] dbCacheNodeFillStatus() { - return newsDB.dbCacheNodeFillStatus(); + public int[] dbCacheNodeStatus() { + return newsDB.dbCacheNodeStatus(); } public String[] dbCacheObjectStatus() { diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index e4abb25d0..e2316bdbf 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -181,11 +181,11 @@ public final class yacySeedDB { return i; } - public int[] dbCacheNodeFillStatus() { - int[] ac = seedActiveDB.cacheNodeFillStatus(); - int[] pa = seedPassiveDB.cacheNodeFillStatus(); - int[] po = seedPotentialDB.cacheNodeFillStatus(); - return new int[]{ac[0] + pa[0] + po[0], ac[1] + pa[1] + po[1], ac[2] + pa[2] + po[2], ac[3] + pa[3] + po[3]}; + public int[] dbCacheNodeStatus() { + int[] ac = seedActiveDB.cacheNodeStatus(); + int[] pa = seedPassiveDB.cacheNodeStatus(); + int[] po = seedPotentialDB.cacheNodeStatus(); + return kelondroRecords.cacheCombinedStatus(new int[][]{ac, pa, po}, 3); } public String[] dbCacheObjectStatus() {