added cache-performance analysis for node caches

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2140 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent a520ab2e8c
commit 4d8f8ba384

@ -81,7 +81,7 @@
<tr class="TableHeader" valign="bottom">
<td class="small" rowspan="2">Database<br>&nbsp;</td>
<td class="small" rowspan="2">DB&nbsp;Size<br>&nbsp;</td>
<td class="small" colspan="3">Node Cache</td>
<td class="small" colspan="4">Node Cache</td>
<td class="small" colspan="4">Object Cache</td>
<td class="small" colspan="5">Memory Occupation (KBytes)</td>
<td class="small" rowspan="2">Description<br>&nbsp;</td>
@ -91,10 +91,11 @@
<td class="small">Chunk Size<br>high/med/low<br>(bytes)</td>
<td class="small">Empty<br>(avail.)<br>Slots</td>
<td class="small">Used: High, Medium, Low Prio</td>
<td class="small">Node-Cache<br>Hit:Miss<br>Uniq:Doub<br>Flush<br></td>
<td class="small">&nbsp;<br>Size<br>Max</td>
<td class="small">&nbsp;<br>Size<br>Current</td>
<td class="small">Cache<br>Hit:Miss<br>Unique:Double<br>Flush<br></td>
<td class="small">Miss<br>Hit:Miss<br>Unique:Double<br>Flush<br></td>
<td class="small">Hit-Cache<br>Hit:Miss<br>Uniq:Doub<br>Flush<br></td>
<td class="small">Miss-Cache<br>Hit:Miss<br>Uniq:Doub<br>Flush<br></td>
<td class="small">Used Now</td>
<td class="small">Assigned Max</td>
<td class="small">Default Max</td>
@ -108,6 +109,7 @@
<td class="small" align="center">#[chunkRWI]#</td>
<td class="small" align="right">#[slempRWI]#</td>
<td class="small" align="right">#[slhigRWI]#<br>#[slmedRWI]#<br>#[sllowRWI]#</td>
<td class="small" align="right">#[slhittmissRWI]#<br>#[sluniqdoubRWI]#<br>#[slflushRWI]#</td>
<td class="small" align="right">#[ochunkmaxRWI]#</td>
<td class="small" align="right">#[ochunkcurRWI]#</td>
<td class="small" align="right">#[ohittmissRWI]#<br>#[ouniqdoubRWI]#<br>#[oflushRWI]#</td>
@ -128,6 +130,7 @@ cache will speed up crawls with a depth > 3.</td>
<td class="small" align="center">#[chunkHTTP]#</td>
<td class="small" align="right">#[slempHTTP]#</td>
<td class="small" align="right">#[slhigHTTP]#<br>#[slmedHTTP]#<br>#[sllowHTTP]#</td>
<td class="small" align="right">#[slhittmissHTTP]#<br>#[sluniqdoubHTTP]#<br>#[slflushHTTP]#</td>
<td class="small" align="right">#[ochunkmaxHTTP]#</td>
<td class="small" align="right">#[ochunkcurHTTP]#</td>
<td class="small" align="right">#[ohittmissHTTP]#<br>#[ouniqdoubHTTP]#<br>#[oflushHTTP]#</td>
@ -148,6 +151,7 @@ Increasing this cache will be most important for a fast proxy mode.</td>
<td class="small" align="center">#[chunkLURL]#</td>
<td class="small" align="right">#[slempLURL]#</td>
<td class="small" align="right">#[slhigLURL]#<br>#[slmedLURL]#<br>#[sllowLURL]#</td>
<td class="small" align="right">#[slhittmissLURL]#<br>#[sluniqdoubLURL]#<br>#[slflushLURL]#</td>
<td class="small" align="right">#[ochunkmaxLURL]#</td>
<td class="small" align="right">#[ochunkcurLURL]#</td>
<td class="small" align="right">#[ohittmissLURL]#<br>#[ouniqdoubLURL]#<br>#[oflushLURL]#</td>
@ -167,6 +171,7 @@ This cache is very important for a fast search process. Increasing the cache siz
<td class="small" align="center">#[chunkNURL]#</td>
<td class="small" align="right">#[slempNURL]#</td>
<td class="small" align="right">#[slhigNURL]#<br>#[slmedNURL]#<br>#[sllowNURL]#</td>
<td class="small" align="right">#[slhittmissNURL]#<br>#[sluniqdoubNURL]#<br>#[slflushNURL]#</td>
<td class="small" align="right">#[ochunkmaxNURL]#</td>
<td class="small" align="right">#[ochunkcurNURL]#</td>
<td class="small" align="right">#[ohittmissNURL]#<br>#[ouniqdoubNURL]#<br>#[oflushNURL]#</td>
@ -186,6 +191,7 @@ Increasing the cache size will result in faster double-check during URL recognit
<td class="small" align="center">#[chunkEURL]#</td>
<td class="small" align="right">#[slempEURL]#</td>
<td class="small" align="right">#[slhigEURL]#<br>#[slmedEURL]#<br>#[sllowEURL]#</td>
<td class="small" align="right">#[slhittmissEURL]#<br>#[sluniqdoubEURL]#<br>#[slflushEURL]#</td>
<td class="small" align="right">#[ochunkmaxEURL]#</td>
<td class="small" align="right">#[ochunkcurEURL]#</td>
<td class="small" align="right">#[ohittmissEURL]#<br>#[ouniqdoubEURL]#<br>#[oflushEURL]#</td>
@ -205,6 +211,7 @@ Increasing the cache size will most probably speed up crawling slightly, but not
<td class="small" align="center">#[chunkDHT]#</td>
<td class="small" align="right">#[slempDHT]#</td>
<td class="small" align="right">#[slhigDHT]#<br>#[slmedDHT]#<br>#[sllowDHT]#</td>
<td class="small" align="right">#[slhittmissDHT]#<br>#[sluniqdoubDHT]#<br>#[slflushDHT]#</td>
<td class="small" align="right">#[ochunkmaxDHT]#</td>
<td class="small" align="right">#[ochunkcurDHT]#</td>
<td class="small" align="right">#[ohittmissDHT]#<br>#[ouniqdoubDHT]#<br>#[oflushDHT]#</td>
@ -225,6 +232,7 @@ Increasing this cache may speed up many functions, but we need to test this to s
<td class="small" align="center">#[chunkMessage]#</td>
<td class="small" align="right">#[slempMessage]#</td>
<td class="small" align="right">#[slhigMessage]#<br>#[slmedMessage]#<br>#[sllowMessage]#</td>
<td class="small" align="right">#[slhittmissMessage]#<br>#[sluniqdoubMessage]#<br>#[slflushMessage]#</td>
<td class="small" align="right">#[ochunkmaxMessage]#</td>
<td class="small" align="right">#[ochunkcurMessage]#</td>
<td class="small" align="right">#[ohittmissMessage]#<br>#[ouniqdoubMessage]#<br>#[oflushMessage]#</td>
@ -243,6 +251,7 @@ Increasing this cache may speed up many functions, but we need to test this to s
<td class="small" align="center">#[chunkWiki]#</td>
<td class="small" align="right">#[slempWiki]#</td>
<td class="small" align="right">#[slhigWiki]#<br>#[slmedWiki]#<br>#[sllowWiki]#</td>
<td class="small" align="right">#[slhittmissWiki]#<br>#[sluniqdoubWiki]#<br>#[slflushWiki]#</td>
<td class="small" align="right">#[ochunkmaxWiki]#</td>
<td class="small" align="right">#[ochunkcurWiki]#</td>
<td class="small" align="right">#[ohittmissWiki]#<br>#[ouniqdoubWiki]#<br>#[oflushWiki]#</td>
@ -263,6 +272,7 @@ Increasing this cache may speed up access to the wiki pages.</td>
<td class="small" align="center">#[chunkBlog]#</td>
<td class="small" align="right">#[slempBlog]#</td>
<td class="small" align="right">#[slhigBlog]#<br>#[slmedBlog]#<br>#[sllowBlog]#</td>
<td class="small" align="right">#[slhittmissBlog]#<br>#[sluniqdoubBlog]#<br>#[slflushBlog]#</td>
<td class="small" align="right">#[ochunkmaxBlog]#</td>
<td class="small" align="right">#[ochunkcurBlog]#</td>
<td class="small" align="right">#[ohittmissBlog]#<br>#[ouniqdoubBlog]#<br>#[oflushBlog]#</td>
@ -282,6 +292,7 @@ Increasing this cache may speed up access to the Blog.</td>
<td class="small" align="center">#[chunkNews]#</td>
<td class="small" align="right">#[slempNews]#</td>
<td class="small" align="right">#[slhigNews]#<br>#[slmedNews]#<br>#[sllowNews]#</td>
<td class="small" align="right">#[slhittmissNews]#<br>#[sluniqdoubNews]#<br>#[slflushNews]#</td>
<td class="small" align="right">#[ochunkmaxNews]#</td>
<td class="small" align="right">#[ochunkcurNews]#</td>
<td class="small" align="right">#[ohittmissNews]#<br>#[ouniqdoubNews]#<br>#[oflushNews]#</td>
@ -301,6 +312,7 @@ Increasing this cache may speed up the peer-ping.</td>
<td class="small" align="center">#[chunkRobots]#</td>
<td class="small" align="right">#[slempRobots]#</td>
<td class="small" align="right">#[slhigRobots]#<br>#[slmedRobots]#<br>#[sllowRobots]#</td>
<td class="small" align="right">#[slhittmissRobots]#<br>#[sluniqdoubRobots]#<br>#[slflushRobots]#</td>
<td class="small" align="right">#[ochunkmaxRobots]#</td>
<td class="small" align="right">#[ochunkcurRobots]#</td>
<td class="small" align="right">#[ohittmissRobots]#<br>#[ouniqdoubRobots]#<br>#[oflushRobots]#</td>
@ -320,6 +332,7 @@ Increasing this cache may speed up validation if crawling of the URL is allowed.
<td class="small" align="center">#[chunkProfiles]#</td>
<td class="small" align="right">#[slempProfiles]#</td>
<td class="small" align="right">#[slhigProfiles]#<br>#[slmedProfiles]#<br>#[sllowProfiles]#</td>
<td class="small" align="right">#[slhittmissProfiles]#<br>#[sluniqdoubProfiles]#<br>#[slflushProfiles]#</td>
<td class="small" align="right">#[ochunkmaxProfiles]#</td>
<td class="small" align="right">#[ochunkcurProfiles]#</td>
<td class="small" align="right">#[ohittmissProfiles]#<br>#[ouniqdoubProfiles]#<br>#[oflushProfiles]#</td>
@ -334,7 +347,7 @@ Increasing this cache may speed up crawling, but not much space is needed, so th
</tr>
<tr class="TableCellSummary">
<td class="small" align="left" colspan="9">Totals</td>
<td class="small" align="left" colspan="10">Totals</td>
<td class="small" align="right">#[usedTotal]# MB</td>
<td class="small" align="right">#[currTotal]# MB</td>
<td class="small" align="right">#[dfltTotal]# MB</td>
@ -344,7 +357,7 @@ Increasing this cache may speed up crawling, but not much space is needed, so th
</tr>
<tr class="TableCellLight">
<td class="small" align="right" colspan="10">Re-Configuration:</td>
<td class="small" align="right" colspan="11">Re-Configuration:</td>
<td class="small" align="center"><input type="submit" name="setCustom" value="Set"><br>these<br>custom<br>values<br>&nbsp;</td>
<td class="small" align="center"><input type="submit" name="setDefault" value="Set"><br>all<br>default<br>values<br>&nbsp;</td>
<td class="small" align="center"><input type="submit" name="setGood" value="Set"><br>all<br>recom-<br>mended values</td>

@ -48,9 +48,7 @@ import java.util.Map;
import java.io.File;
import de.anomic.http.httpHeader;
import de.anomic.index.indexURL;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverMemory;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.server.serverFileUtils;
@ -166,73 +164,73 @@ public class PerformanceMemory_p {
req = sb.wordIndex.size();
chk = sb.wordIndex.assortmentsCacheChunkSizeAvg();
slt = sb.wordIndex.assortmentsCacheFillStatusCml();
slt = sb.wordIndex.assortmentsCacheNodeStatus();
ost = sb.wordIndex.assortmentsCacheObjectStatus();
putprop(prop, env, "RWI", set);
req = sb.cacheManager.dbSize();
chk = sb.cacheManager.dbCacheChunkSize();
slt = sb.cacheManager.dbCacheFillStatus();
slt = sb.cacheManager.dbCacheStatus();
ost = sb.cacheManager.dbCacheObjectStatus();
putprop(prop, env, "HTTP", set);
req = sb.urlPool.loadedURL.size();
chk = sb.urlPool.loadedURL.cacheNodeChunkSize();
slt = sb.urlPool.loadedURL.cacheNodeFillStatus();
slt = sb.urlPool.loadedURL.cacheNodeStatus();
ost = sb.urlPool.loadedURL.cacheObjectStatus();
putprop(prop, env, "LURL", set);
req = sb.urlPool.noticeURL.size();
chk = sb.urlPool.noticeURL.cacheNodeChunkSize();
slt = sb.urlPool.noticeURL.cacheNodeFillStatus();
slt = sb.urlPool.noticeURL.cacheNodeStatus();
ost = sb.urlPool.noticeURL.cacheObjectStatus();
putprop(prop, env, "NURL", set);
req = sb.urlPool.errorURL.size();
chk = sb.urlPool.errorURL.cacheNodeChunkSize();
slt = sb.urlPool.errorURL.cacheNodeFillStatus();
slt = sb.urlPool.errorURL.cacheNodeStatus();
ost = sb.urlPool.errorURL.cacheObjectStatus();
putprop(prop, env, "EURL", set);
req = yacyCore.seedDB.sizeConnected() + yacyCore.seedDB.sizeDisconnected() + yacyCore.seedDB.sizePotential();
chk = yacyCore.seedDB.dbCacheNodeChunkSize();
slt = yacyCore.seedDB.dbCacheNodeFillStatus();
slt = yacyCore.seedDB.dbCacheNodeStatus();
ost = yacyCore.seedDB.dbCacheObjectStatus();
putprop(prop, env, "DHT", set);
req = sb.messageDB.size();
chk = sb.messageDB.dbCacheNodeChunkSize();
slt = sb.messageDB.dbCacheNodeFillStatus();
slt = sb.messageDB.dbCacheNodeStatus();
ost = sb.messageDB.dbCacheObjectStatus();
putprop(prop, env, "Message", set);
req = sb.wikiDB.sizeOfTwo();
chk = sb.wikiDB.dbCacheNodeChunkSize();
slt = sb.wikiDB.dbCacheNodeFillStatus();
slt = sb.wikiDB.dbCacheNodeStatus();
ost = sb.wikiDB.dbCacheObjectStatus();
putprop(prop, env, "Wiki", set);
req = sb.blogDB.size();
chk = sb.blogDB.dbCacheNodeChunkSize();
slt = sb.blogDB.dbCacheNodeFillStatus();
slt = sb.blogDB.dbCacheNodeStatus();
ost = sb.blogDB.dbCacheObjectStatus();
putprop(prop, env, "Blog", set);
req = yacyCore.newsPool.dbSize();
chk = yacyCore.newsPool.dbCacheNodeChunkSize();
slt = yacyCore.newsPool.dbCacheNodeFillStatus();
slt = yacyCore.newsPool.dbCacheNodeStatus();
ost = yacyCore.newsPool.dbCacheObjectStatus();
putprop(prop, env, "News", set);
req = plasmaSwitchboard.robots.size();
chk = plasmaSwitchboard.robots.dbCacheNodeChunkSize();
slt = plasmaSwitchboard.robots.dbCacheNodeFillStatus();
slt = plasmaSwitchboard.robots.dbCacheNodeStatus();
ost = plasmaSwitchboard.robots.dbCacheObjectStatus();
putprop(prop, env, "Robots", set);
req = sb.profiles.size();
chk = sb.profiles.dbCacheNodeChunkSize();
slt = sb.profiles.dbCacheNodeFillStatus();
slt = sb.profiles.dbCacheNodeStatus();
ost = sb.profiles.dbCacheObjectStatus();
putprop(prop, env, "Profiles", set);
@ -296,10 +294,13 @@ public class PerformanceMemory_p {
if (set.equals("setBest")) env.setConfig("ramCache" + db, bst);
prop.put("chunk" + db, chk[2] + "/" + chk[1] + "/" + chk[0]);
prop.put("slreq" + db, req);
prop.put("slemp" + db, slt[0]);
prop.put("slemp" + db, slt[0] - slt[1] - slt[2] - slt[3]);
prop.put("slhig" + db, slt[1]);
prop.put("slmed" + db, slt[2]);
prop.put("sllow" + db, slt[3]);
prop.put("slhittmiss" + db, slt[4] + ":" + slt[5]);
prop.put("sluniqdoub" + db, slt[6] + ":" + slt[7]);
prop.put("slflush" + db, slt[8]);
prop.put("ochunkmax" + db, ost[0]);
prop.put("ochunkcur" + db, ost[1]);
prop.put("ohittmiss" + db, ost[5] + ":" + ost[6]);

@ -90,8 +90,8 @@ public class blogBoard {
return datbase.cacheNodeChunkSize();
}
public int[] dbCacheNodeFillStatus() {
return datbase.cacheNodeFillStatus();
public int[] dbCacheNodeStatus() {
return datbase.cacheNodeStatus();
}
public String[] dbCacheObjectStatus() {

@ -90,8 +90,8 @@ public class messageBoard {
return database.cacheNodeChunkSize();
}
public int[] dbCacheNodeFillStatus() {
return database.cacheNodeFillStatus();
public int[] dbCacheNodeStatus() {
return database.cacheNodeStatus();
}
public String[] dbCacheObjectStatus() {

@ -95,8 +95,8 @@ public final class userDB {
return userTable.cacheNodeChunkSize();
}
public int[] dbCacheNodeFillStatus() {
return userTable.cacheNodeFillStatus();
public int[] dbCacheNodeStatus() {
return userTable.cacheNodeStatus();
}
void resetDatabase() {

@ -110,10 +110,10 @@ public class wikiBoard {
return i;
}
public int[] dbCacheNodeFillStatus() {
int[] a = datbase.cacheNodeFillStatus();
int[] b = bkpbase.cacheNodeFillStatus();
return new int[]{a[0] + b[0], a[1] + b[1], a[2] + b[2], a[3] + b[3]};
public int[] dbCacheNodeStatus() {
int[] a = datbase.cacheNodeStatus();
int[] b = bkpbase.cacheNodeStatus();
return kelondroRecords.cacheCombinedStatus(new int[][]{a, b}, 2);
}
public String[] dbCacheObjectStatus() {

@ -434,8 +434,8 @@ public class indexURL {
return urlHashCache.cacheNodeChunkSize();
}
public int[] cacheNodeFillStatus() {
return urlHashCache.cacheNodeFillStatus();
public int[] cacheNodeStatus() {
return urlHashCache.cacheNodeStatus();
}
public String[] cacheObjectStatus() {

@ -137,8 +137,8 @@ public class kelondroMap {
return dyn.cacheNodeChunkSize();
}
public int[] cacheNodeFillStatus() {
return dyn.cacheNodeFillStatus();
public int[] cacheNodeStatus() {
return dyn.cacheNodeStatus();
}
public String[] cacheObjectStatus() {

@ -149,11 +149,12 @@ public class kelondroRecords {
private int TXTPROPW; // size of a single TXTPROPS element
// caching buffer
protected HashMap[] XcacheHeaders; // the cache; holds overhead values and key element
protected int XcacheSize; // number of cache records
protected long XcacheStartup; // startup time; for cache aging
protected kelondroMScoreCluster cacheScore; // controls cache aging
private HashMap[] cacheHeaders; // the cache; holds overhead values and key element
private int cacheSize; // number of cache records
private long cacheStartup; // startup time; for cache aging
private kelondroMScoreCluster cacheScore; // controls cache aging
private int readHit, readMiss, writeUnique, writeDouble, cacheFlush;
// optional logger
protected Logger theLogger = null;
@ -423,20 +424,25 @@ public class kelondroRecords {
private void initCache(long buffersize) {
if (buffersize <= 0) {
this.XcacheSize = 0;
this.XcacheHeaders = null;
this.cacheSize = 0;
this.cacheHeaders = null;
this.cacheScore = null;
} else {
if ((buffersize / cacheNodeChunkSize(false)) > size()) {
this.XcacheSize = (int) (buffersize / cacheNodeChunkSize(false));
this.cacheSize = (int) (buffersize / cacheNodeChunkSize(false));
this.cacheScore = null; // no cache control because we have more cache slots than database entries
} else {
this.XcacheSize = (int) (buffersize / cacheNodeChunkSize(true));
this.cacheSize = (int) (buffersize / cacheNodeChunkSize(true));
this.cacheScore = new kelondroMScoreCluster(); // cache control of CP_HIGH caches
}
this.XcacheHeaders = new HashMap[]{new HashMap(), new HashMap(), new HashMap()};
this.cacheHeaders = new HashMap[]{new HashMap(), new HashMap(), new HashMap()};
}
this.XcacheStartup = System.currentTimeMillis();
this.cacheStartup = System.currentTimeMillis();
this.readHit = 0;
this.readMiss = 0;
this.writeUnique = 0;
this.writeDouble = 0;
this.cacheFlush = 0;
}
private static final long max = Runtime.getRuntime().maxMemory();
@ -468,9 +474,32 @@ public class kelondroRecords {
return i;
}
public int[] cacheNodeFillStatus() {
if (XcacheHeaders == null) return new int[]{0,0,0,0};
return new int[]{XcacheSize - (XcacheHeaders[CP_HIGH].size() + XcacheHeaders[CP_MEDIUM].size() + XcacheHeaders[CP_LOW].size()), XcacheHeaders[CP_HIGH].size(), XcacheHeaders[CP_MEDIUM].size(), XcacheHeaders[CP_LOW].size()};
public int[] cacheNodeStatus() {
if (cacheHeaders == null) return new int[]{0,0,0,0,0,0,0,0,0};
return new int[]{
cacheSize,
cacheHeaders[CP_HIGH].size(),
cacheHeaders[CP_MEDIUM].size(),
cacheHeaders[CP_LOW].size(),
readHit,
readMiss,
writeUnique,
writeDouble,
cacheFlush
};
}
private static int[] cacheCombinedStatus(int[] a, int[] b) {
int[] c = new int[a.length];
for (int i = a.length - 1; i >= 0; i--) c[i] = a[i] + b[i];
return c;
}
public static int[] cacheCombinedStatus(int[][] a, int l) {
if ((a == null) || (a.length == 0) || (l == 0)) return null;
if ((a.length >= 1) && (l == 1)) return a[0];
if ((a.length >= 2) && (l == 2)) return cacheCombinedStatus(a[0], a[1]);
return cacheCombinedStatus(cacheCombinedStatus(a, l - 1), a[l - 1]);
}
protected Node newNode() throws IOException {
@ -486,23 +515,24 @@ public class kelondroRecords {
}
protected void deleteNode(Handle handle) throws IOException {
if (XcacheSize != 0) {
synchronized (XcacheHeaders) {
if (cacheSize != 0) {
synchronized (cacheHeaders) {
if (cacheScore == null) {
XcacheHeaders[CP_LOW].remove(handle);
XcacheHeaders[CP_MEDIUM].remove(handle);
XcacheHeaders[CP_HIGH].remove(handle);
} else if (XcacheHeaders[CP_HIGH].get(handle) != null) {
cacheHeaders[CP_LOW].remove(handle);
cacheHeaders[CP_MEDIUM].remove(handle);
cacheHeaders[CP_HIGH].remove(handle);
} else if (cacheHeaders[CP_HIGH].get(handle) != null) {
// remove handle from cache-control
cacheScore.deleteScore(handle);
XcacheHeaders[CP_HIGH].remove(handle);
} else if (XcacheHeaders[CP_MEDIUM].get(handle) != null) {
cacheHeaders[CP_HIGH].remove(handle);
} else if (cacheHeaders[CP_MEDIUM].get(handle) != null) {
// no cache control for medium-priority entries
XcacheHeaders[CP_MEDIUM].remove(handle);
} else if (XcacheHeaders[CP_LOW].get(handle) != null) {
cacheHeaders[CP_MEDIUM].remove(handle);
} else if (cacheHeaders[CP_LOW].get(handle) != null) {
// no cache control for low-priority entries
XcacheHeaders[CP_LOW].remove(handle);
cacheHeaders[CP_LOW].remove(handle);
}
cacheFlush++;
}
}
dispose(handle);
@ -605,26 +635,27 @@ public class kelondroRecords {
private void initContent() throws IOException {
// create chunks; read them from file or cache
this.tailChunk = null;
if (XcacheSize == 0) {
if (cacheSize == 0) {
// read overhead and key
//System.out.println("**NO CACHE for " + this.handle.index + "**");
this.headChunk = new byte[headchunksize];
entryFile.readFully(seekpos(this.handle), this.headChunk, 0, this.headChunk.length);
this.headChanged = false;
} else synchronized(XcacheHeaders) {
} else synchronized(cacheHeaders) {
byte[] cacheEntry = null;
int cp = CP_HIGH;
cacheEntry = (byte[]) XcacheHeaders[CP_HIGH].get(this.handle); // first try
cacheEntry = (byte[]) cacheHeaders[CP_HIGH].get(this.handle); // first try
if (cacheEntry == null) {
cacheEntry = (byte[]) XcacheHeaders[CP_MEDIUM].get(this.handle); // second try
cacheEntry = (byte[]) cacheHeaders[CP_MEDIUM].get(this.handle); // second try
cp = CP_MEDIUM;
}
if (cacheEntry == null) {
cacheEntry = (byte[]) XcacheHeaders[CP_LOW].get(this.handle); // third try
cacheEntry = (byte[]) cacheHeaders[CP_LOW].get(this.handle); // third try
cp = CP_LOW;
}
if (cacheEntry == null) {
// cache miss, we read overhead and key from file
readMiss++;
//System.out.println("**CACHE miss for " + this.handle.index + "**");
this.headChunk = new byte[headchunksize];
//this.tailChunk = new byte[tailchunksize];
@ -642,12 +673,13 @@ public class kelondroRecords {
update2Cache(cp);
} else {
// cache hit, copy overhead and key from cache
readHit++;
//System.out.println("**CACHE HIT for " + this.handle.index + "**");
this.headChunk = new byte[headchunksize];
System.arraycopy(cacheEntry, 0, this.headChunk, 0, headchunksize);
// update cache scores to announce this cache hit
if ((cacheScore != null) && (cp == CP_HIGH)) {
cacheScore.setScore(this.handle, (int) ((System.currentTimeMillis() - XcacheStartup) / 1000));
cacheScore.setScore(this.handle, (int) ((System.currentTimeMillis() - cacheStartup) / 1000));
}
this.headChanged = false;
}
@ -816,10 +848,10 @@ public class kelondroRecords {
}
private void update2Cache(int forPriority) {
if (XcacheSize > 0) {
XcacheHeaders[CP_LOW].remove(this.handle);
XcacheHeaders[CP_MEDIUM].remove(this.handle);
XcacheHeaders[CP_HIGH].remove(this.handle);
if (cacheSize > 0) {
cacheHeaders[CP_LOW].remove(this.handle);
cacheHeaders[CP_MEDIUM].remove(this.handle);
cacheHeaders[CP_HIGH].remove(this.handle);
}
if (cacheSpace(forPriority)) updateNodeCache(forPriority);
}
@ -830,34 +862,37 @@ public class kelondroRecords {
// returns true if it is allowed to add another entry to the cache
// returns false if the cache is considered to be full
if (forPriority == CP_NONE) return false;
if (XcacheSize == 0) return false; // no caching
long cs = XcacheHeaders[CP_LOW].size() + XcacheHeaders[CP_MEDIUM].size() + XcacheHeaders[CP_HIGH].size();
if (cacheSize == 0) return false; // no caching
long cs = cacheHeaders[CP_LOW].size() + cacheHeaders[CP_MEDIUM].size() + cacheHeaders[CP_HIGH].size();
if (cs == 0) return true; // nothing there to flush
if ((cs < XcacheSize) && (availableMemory() >= memBlock)) return true; // no need to flush cache space
if ((cs < cacheSize) && (availableMemory() >= memBlock)) return true; // no need to flush cache space
Handle delkey;
// delete one entry. distinguish between different priority cases:
if (forPriority == CP_LOW) {
// remove only from low-priority cache
if (XcacheHeaders[CP_LOW].size() != 0) {
if (cacheHeaders[CP_LOW].size() != 0) {
// just delete any of the low-priority entries
delkey = (Handle) XcacheHeaders[CP_LOW].keySet().iterator().next();
XcacheHeaders[CP_LOW].remove(delkey);
delkey = (Handle) cacheHeaders[CP_LOW].keySet().iterator().next();
cacheHeaders[CP_LOW].remove(delkey);
cacheFlush++;
return true;
} else {
// we cannot delete any entry, therefore there is no space for another entry
return false;
}
} else if (forPriority == CP_MEDIUM) {
if (XcacheHeaders[CP_LOW].size() != 0) {
if (cacheHeaders[CP_LOW].size() != 0) {
// just delete any of the low-priority entries
delkey = (Handle) XcacheHeaders[CP_LOW].keySet().iterator().next();
XcacheHeaders[CP_LOW].remove(delkey);
delkey = (Handle) cacheHeaders[CP_LOW].keySet().iterator().next();
cacheHeaders[CP_LOW].remove(delkey);
cacheFlush++;
return true;
} else if (XcacheHeaders[CP_MEDIUM].size() != 0) {
} else if (cacheHeaders[CP_MEDIUM].size() != 0) {
// just delete any of the medium-priority entries
delkey = (Handle) XcacheHeaders[CP_MEDIUM].keySet().iterator().next();
XcacheHeaders[CP_MEDIUM].remove(delkey);
delkey = (Handle) cacheHeaders[CP_MEDIUM].keySet().iterator().next();
cacheHeaders[CP_MEDIUM].remove(delkey);
cacheFlush++;
return true;
} else {
// we cannot delete any entry, therefore there is no space for another entry
@ -865,15 +900,17 @@ public class kelondroRecords {
}
} else {
// request for a high-priority entry
if (XcacheHeaders[CP_LOW].size() != 0) {
if (cacheHeaders[CP_LOW].size() != 0) {
// just delete any of the low-priority entries
delkey = (Handle) XcacheHeaders[CP_LOW].keySet().iterator().next();
XcacheHeaders[CP_LOW].remove(delkey);
delkey = (Handle) cacheHeaders[CP_LOW].keySet().iterator().next();
cacheHeaders[CP_LOW].remove(delkey);
cacheFlush++;
return true;
} else if (XcacheHeaders[CP_MEDIUM].size() != 0) {
} else if (cacheHeaders[CP_MEDIUM].size() != 0) {
// just delete any of the medium-priority entries
delkey = (Handle) XcacheHeaders[CP_MEDIUM].keySet().iterator().next();
XcacheHeaders[CP_MEDIUM].remove(delkey);
delkey = (Handle) cacheHeaders[CP_MEDIUM].keySet().iterator().next();
cacheHeaders[CP_MEDIUM].remove(delkey);
cacheFlush++;
return true;
} else if (cacheScore == null) {
// no cache-control of high-priority cache
@ -884,16 +921,17 @@ public class kelondroRecords {
// use the cache-control to find the right object
delkey = (Handle) cacheScore.getMinObject();
cacheScore.deleteScore(delkey);
XcacheHeaders[CP_HIGH].remove(delkey);
cacheHeaders[CP_HIGH].remove(delkey);
cacheFlush++;
return true;
} catch (NoSuchElementException e) {
// this is a strange error and could be caused by internal java problems
// we simply clear the cache
String error = "cachScore error: " + e.getMessage() + "; cachesize=" + XcacheSize + ", cache.size()=[" + XcacheHeaders[0].size() + "," + XcacheHeaders[1].size() + "," + XcacheHeaders[2].size() + "], cacheScore.size()=" + cacheScore.size();
String error = "cachScore error: " + e.getMessage() + "; cachesize=" + cacheSize + ", cache.size()=[" + cacheHeaders[0].size() + "," + cacheHeaders[1].size() + "," + cacheHeaders[2].size() + "], cacheScore.size()=" + cacheScore.size();
cacheScore = new kelondroMScoreCluster();
XcacheHeaders[CP_LOW] = new HashMap();
XcacheHeaders[CP_MEDIUM] = new HashMap();
XcacheHeaders[CP_HIGH] = new HashMap();
cacheHeaders[CP_LOW] = new HashMap();
cacheHeaders[CP_MEDIUM] = new HashMap();
cacheHeaders[CP_HIGH] = new HashMap();
throw new kelondroException(filename, error);
}
@ -904,25 +942,26 @@ public class kelondroRecords {
if (this.handle == null) return; // wrong access
if (this.headChunk == null) return; // nothing there to cache
if (priority == CP_NONE) return; // it is not wanted that this shall be cached
if (XcacheSize == 0) return; // we do not use the cache
int cs = XcacheHeaders[CP_LOW].size() + XcacheHeaders[CP_MEDIUM].size() + XcacheHeaders[CP_HIGH].size();
if (cs >= XcacheSize) return; // no cache update if cache is full
if (cacheSize == 0) return; // we do not use the cache
int cs = cacheHeaders[CP_LOW].size() + cacheHeaders[CP_MEDIUM].size() + cacheHeaders[CP_HIGH].size();
if (cs >= cacheSize) return; // no cache update if cache is full
synchronized (XcacheHeaders) {
synchronized (cacheHeaders) {
// generate cache entry
byte[] cacheEntry = new byte[headchunksize];
System.arraycopy(headChunk, 0, cacheEntry, 0, headchunksize);
Handle cacheHandle = new Handle(this.handle.index);
// store the cache entry
//XcacheHeaders.remove(cacheHandle);
if (priority != CP_LOW) XcacheHeaders[CP_LOW].remove(cacheHandle);
if (priority != CP_MEDIUM) XcacheHeaders[CP_MEDIUM].remove(cacheHandle);
if (priority != CP_HIGH) XcacheHeaders[CP_HIGH].remove(cacheHandle);
XcacheHeaders[priority].put(cacheHandle, cacheEntry);
boolean upd = false;
if (priority != CP_LOW) upd = upd || (cacheHeaders[CP_LOW].remove(cacheHandle) != null);
if (priority != CP_MEDIUM) upd = upd || (cacheHeaders[CP_MEDIUM].remove(cacheHandle) != null);
if (priority != CP_HIGH) upd = upd || (cacheHeaders[CP_HIGH].remove(cacheHandle) != null);
cacheHeaders[priority].put(cacheHandle, cacheEntry);
if ((cacheScore != null) && (priority == CP_HIGH)) {
cacheScore.setScore(cacheHandle, (int) ((System.currentTimeMillis() - XcacheStartup) / 1000));
cacheScore.setScore(cacheHandle, (int) ((System.currentTimeMillis() - cacheStartup) / 1000));
}
if (upd) writeDouble++; else writeUnique++;
// delete the cache entry buffer
cacheEntry = null;
@ -935,7 +974,7 @@ public class kelondroRecords {
}
protected void printCache() {
if (XcacheSize == 0) {
if (cacheSize == 0) {
System.out.println("### file report: " + size() + " entries");
for (int i = 0; i < size() + 3; i++) {
// print from file to compare
@ -948,9 +987,9 @@ public class kelondroRecords {
System.out.println();
}
} else {
System.out.println("### cache report: [" + XcacheHeaders[0].size() + "," + XcacheHeaders[0].size() + "," + XcacheHeaders[0].size() + "] entries");
System.out.println("### cache report: [" + cacheHeaders[0].size() + "," + cacheHeaders[0].size() + "," + cacheHeaders[0].size() + "] entries");
for (int cp = 0; cp < 3; cp++) {
Iterator i = XcacheHeaders[cp].entrySet().iterator();
Iterator i = cacheHeaders[cp].entrySet().iterator();
Map.Entry entry;
while (i.hasNext()) {
entry = (Map.Entry) i.next();

@ -99,7 +99,7 @@ public final class kelondroStack extends kelondroRecords {
public static kelondroStack reset(kelondroStack stack) {
// memorize settings to this file
File f = new File(stack.filename);
long bz = stack.XcacheSize * stack.cacheNodeChunkSize(true);
long bz = stack.cacheNodeStatus()[0] * stack.cacheNodeChunkSize(true);
int[] cols = stack.COLWIDTHS;
// close and delete the file

@ -82,8 +82,8 @@ public class plasmaCrawlProfile {
return profileTable.cacheNodeChunkSize();
}
public int[] dbCacheNodeFillStatus() {
return profileTable.cacheNodeFillStatus();
public int[] dbCacheNodeStatus() {
return profileTable.cacheNodeStatus();
}
public String[] dbCacheObjectStatus() {

@ -89,8 +89,8 @@ public class plasmaCrawlRobotsTxt {
return robotsTable.cacheNodeChunkSize();
}
public int[] dbCacheNodeFillStatus() {
return robotsTable.cacheNodeFillStatus();
public int[] dbCacheNodeStatus() {
return robotsTable.cacheNodeStatus();
}
public String[] dbCacheObjectStatus() {

@ -207,8 +207,8 @@ public final class plasmaHTCache {
return this.responseHeaderDB.cacheNodeChunkSize();
}
public int[] dbCacheFillStatus() {
return this.responseHeaderDB.cacheNodeFillStatus();
public int[] dbCacheStatus() {
return this.responseHeaderDB.cacheNodeStatus();
}
public String[] dbCacheObjectStatus() {

@ -130,8 +130,8 @@ public final class plasmaWordIndex {
return assortmentCluster.cacheChunkSizeAvg();
}
public int[] assortmentsCacheFillStatusCml() {
return assortmentCluster.cacheFillStatusCml();
public int[] assortmentsCacheNodeStatus() {
return assortmentCluster.cacheNodeStatus();
}
public String[] assortmentsCacheObjectStatus() {

@ -266,8 +266,8 @@ public final class plasmaWordIndexAssortment {
return assortments.cacheNodeChunkSize();
}
public int[] cacheNodeFillStatus() {
return assortments.cacheNodeFillStatus();
public int[] cacheNodeStatus() {
return assortments.cacheNodeStatus();
}
public String[] dbCacheObjectStatus() {

@ -278,18 +278,15 @@ public final class plasmaWordIndexAssortmentCluster {
return a;
}
public int[] cacheFillStatusCml() {
int[] a, cml = new int[]{0, 0, 0, 0};
for (int i = 0; i < clusterCount; i++) {
a = assortments[i].cacheNodeFillStatus();
for (int j = 0; j < 4; j++) cml[j] += a[j];
}
return cml;
public int[] cacheNodeStatus() {
int[][] a = new int[assortments.length][];
for (int i = assortments.length - 1; i >= 0; i--) a[i] = assortments[i].cacheNodeStatus();
return kelondroRecords.cacheCombinedStatus(a, assortments.length);
}
public String[] cacheObjectStatus() {
String[][] a = new String[assortments.length][];
for (int i = 0; i < assortments.length; i++) a[i] = assortments[i].dbCacheObjectStatus();
for (int i = assortments.length - 1; i >= 0; i--) a[i] = assortments[i].dbCacheObjectStatus();
return kelondroObjectCache.combinedStatus(a, a.length);
}

@ -100,8 +100,8 @@ public class yacyNewsDB {
return news.cacheNodeChunkSize();
}
public int[] dbCacheNodeFillStatus() {
return news.cacheNodeFillStatus();
public int[] dbCacheNodeStatus() {
return news.cacheNodeStatus();
}
public String[] dbCacheObjectStatus() {

@ -110,8 +110,8 @@ public class yacyNewsPool {
return newsDB.dbCacheNodeChunkSize();
}
public int[] dbCacheNodeFillStatus() {
return newsDB.dbCacheNodeFillStatus();
public int[] dbCacheNodeStatus() {
return newsDB.dbCacheNodeStatus();
}
public String[] dbCacheObjectStatus() {

@ -181,11 +181,11 @@ public final class yacySeedDB {
return i;
}
public int[] dbCacheNodeFillStatus() {
int[] ac = seedActiveDB.cacheNodeFillStatus();
int[] pa = seedPassiveDB.cacheNodeFillStatus();
int[] po = seedPotentialDB.cacheNodeFillStatus();
return new int[]{ac[0] + pa[0] + po[0], ac[1] + pa[1] + po[1], ac[2] + pa[2] + po[2], ac[3] + pa[3] + po[3]};
public int[] dbCacheNodeStatus() {
int[] ac = seedActiveDB.cacheNodeStatus();
int[] pa = seedPassiveDB.cacheNodeStatus();
int[] po = seedPotentialDB.cacheNodeStatus();
return kelondroRecords.cacheCombinedStatus(new int[][]{ac, pa, po}, 3);
}
public String[] dbCacheObjectStatus() {

Loading…
Cancel
Save