boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

removed high/med/low from kelondroRecords cache control.

Browse Source 
this was done because testing showed that cache-delete operations
slowed down record access most, even more that actual IO operations.
Cache-delete operations appeared when entries were shifted from low-priority
positions to high-priority positions. During a fill of x entries to a database,
x/2 delete situation happen which caused two or more delete operations.
removing the cache control means that these delete operations are not
necessary any more, but it is more difficult to decide which cache elements
shall be removed in case that the cache is full. There is not yet a stable
solution for this case, but the advantage of a faster cache is more important
that the flush problem.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2244 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent 6866bc2758
commit 66964dc015
23 changed files with 95 additions and 259 deletions
Show Stats Download Patch File Download Diff File

2
build.properties
Unescape View File

@ -3,7 +3,7 @@ javacSource=1.4
javacTarget=1.4 javacTarget=1.4
# Release Configuration # Release Configuration
releaseVersion=0.454 releaseVersion=0.455
releaseFile=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz releaseFile=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
#releaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz #releaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
releaseDir=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr} releaseDir=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}

67
htroot/PerformanceMemory_p.html
Unescape View File

@ -83,14 +83,14 @@
<td class="small" rowspan="2">DB&nbsp;Size<br>&nbsp;</td> <td class="small" rowspan="2">DB&nbsp;Size<br>&nbsp;</td>
<td class="small" colspan="4">Node Cache</td> <td class="small" colspan="4">Node Cache</td>
<td class="small" colspan="4">Object Cache</td> <td class="small" colspan="4">Object Cache</td>
<td class="small" colspan="5">Memory Occupation (KBytes)</td> <td class="small" colspan="4">Memory Occupation (KBytes)</td>
<td class="small" rowspan="2">Description<br>&nbsp;</td> <td class="small" rowspan="2">Description<br>&nbsp;</td>
</tr> </tr>
<tr class="TableHeader" valign="bottom"> <tr class="TableHeader" valign="bottom">
<td class="small">Chunk Size<br>high/med/low<br>(bytes)</td> <td class="small">Chunk<br>Size<br>(bytes)</td>
<td class="small">Empty<br>(avail.)<br>Slots</td> <td class="small">Empty<br>(avail.)<br>Slots</td>
<td class="small">Used: High, Medium, Low Prio</td> <td class="small">Used<br>(filled)<br>Slots</td>
<td class="small">Node-Cache<br>Hit:Miss<br>Uniq:Doub<br>Del:Flush<br></td> <td class="small">Node-Cache<br>Hit:Miss<br>Uniq:Doub<br>Del:Flush<br></td>
<td class="small">&nbsp;<br>Max<br>Size</td> <td class="small">&nbsp;<br>Max<br>Size</td>
<td class="small">&nbsp;<br>Hit-Size<br>Miss-Size</td> <td class="small">&nbsp;<br>Hit-Size<br>Miss-Size</td>
@ -99,16 +99,15 @@
<td class="small">Used Now</td> <td class="small">Used Now</td>
<td class="small">Assigned Max</td> <td class="small">Assigned Max</td>
<td class="small">Default Max</td> <td class="small">Default Max</td>
<td class="small">Good Max</td>
<td class="small">Best Max</td> <td class="small">Best Max</td>
</tr> </tr>
<tr class="TableCellDark"> <tr class="TableCellDark">
<td class="small" align="left">RWI Assortment Cluster</td> <td class="small" align="left">RWI Assortment Cluster</td>
<td class="small" align="right">#[slreqRWI]#</td> <td class="small" align="right">#[slreqRWI]#</td>
<td class="small" align="center">#[chunkRWI]#</td> <td class="small" align="right">#[chunkRWI]#</td>
<td class="small" align="right">#[slempRWI]#</td> <td class="small" align="right">#[slempRWI]#</td>
<td class="small" align="right">#[slhigRWI]#<br>#[slmedRWI]#<br>#[sllowRWI]#</td> <td class="small" align="right">#[slfilRWI]#</td>
<td class="small" align="right">#[slhittmissRWI]#<br>#[sluniqdoubRWI]#<br>#[slflushRWI]#</td> <td class="small" align="right">#[slhittmissRWI]#<br>#[sluniqdoubRWI]#<br>#[slflushRWI]#</td>
<td class="small" align="right">#[ochunkmaxRWI]#</td> <td class="small" align="right">#[ochunkmaxRWI]#</td>
<td class="small" align="right">#[ochunkcurRWI]#</td> <td class="small" align="right">#[ochunkcurRWI]#</td>
@ -117,7 +116,6 @@
<td class="small" align="right">#[usedRWI]#</td> <td class="small" align="right">#[usedRWI]#</td>
<td class="small" align="right"><input name="ramCacheRWI" type="text" align="right" size="6" maxlength="7" value="#[ramCacheRWI]#"></td> <td class="small" align="right"><input name="ramCacheRWI" type="text" align="right" size="6" maxlength="7" value="#[ramCacheRWI]#"></td>
<td class="small" align="right">#[dfltRWI]#</td> <td class="small" align="right">#[dfltRWI]#</td>
<td class="small" align="right">#[goodRWI]#</td>
<td class="small" align="right">#[bestRWI]#</td> <td class="small" align="right">#[bestRWI]#</td>
<td class="small" align="left">The Assortment Cluster stores most of the page indexes. <td class="small" align="left">The Assortment Cluster stores most of the page indexes.
Flushing speed of the temporary RWI cache depends on the size of this file cache. Increasing the space of this Flushing speed of the temporary RWI cache depends on the size of this file cache. Increasing the space of this
@ -127,9 +125,9 @@ cache will speed up crawls with a depth > 3.</td>
<tr class="TableCellDark"> <tr class="TableCellDark">
<td class="small" align="left">HTTP Response Header</td> <td class="small" align="left">HTTP Response Header</td>
<td class="small" align="right">#[slreqHTTP]#</td> <td class="small" align="right">#[slreqHTTP]#</td>
<td class="small" align="center">#[chunkHTTP]#</td> <td class="small" align="right">#[chunkHTTP]#</td>
<td class="small" align="right">#[slempHTTP]#</td> <td class="small" align="right">#[slempHTTP]#</td>
<td class="small" align="right">#[slhigHTTP]#<br>#[slmedHTTP]#<br>#[sllowHTTP]#</td> <td class="small" align="right">#[slfilHTTP]#</td>
<td class="small" align="right">#[slhittmissHTTP]#<br>#[sluniqdoubHTTP]#<br>#[slflushHTTP]#</td> <td class="small" align="right">#[slhittmissHTTP]#<br>#[sluniqdoubHTTP]#<br>#[slflushHTTP]#</td>
<td class="small" align="right">#[ochunkmaxHTTP]#</td> <td class="small" align="right">#[ochunkmaxHTTP]#</td>
<td class="small" align="right">#[ochunkcurHTTP]#</td> <td class="small" align="right">#[ochunkcurHTTP]#</td>
@ -138,7 +136,6 @@ cache will speed up crawls with a depth > 3.</td>
<td class="small" align="right">#[usedHTTP]#</td> <td class="small" align="right">#[usedHTTP]#</td>
<td class="small" align="right"><input name="ramCacheHTTP" type="text" align="right" size="6" maxlength="7" value="#[ramCacheHTTP]#"></td> <td class="small" align="right"><input name="ramCacheHTTP" type="text" align="right" size="6" maxlength="7" value="#[ramCacheHTTP]#"></td>
<td class="small" align="right">#[dfltHTTP]#</td> <td class="small" align="right">#[dfltHTTP]#</td>
<td class="small" align="right">#[goodHTTP]#</td>
<td class="small" align="right">#[bestHTTP]#</td> <td class="small" align="right">#[bestHTTP]#</td>
<td class="small" align="left">The Response Header database stores the HTTP heades that other servers send when YaCy retrieves web pages <td class="small" align="left">The Response Header database stores the HTTP heades that other servers send when YaCy retrieves web pages
during proxy mode, when performing crawls or if it fetches pages for snippet generation. during proxy mode, when performing crawls or if it fetches pages for snippet generation.
@ -148,9 +145,9 @@ Increasing this cache will be most important for a fast proxy mode.</td>
<tr class="TableCellDark"> <tr class="TableCellDark">
<td class="small" align="left">'loaded' URLs</td> <td class="small" align="left">'loaded' URLs</td>
<td class="small" align="right">#[slreqLURL]#</td> <td class="small" align="right">#[slreqLURL]#</td>
<td class="small" align="center">#[chunkLURL]#</td> <td class="small" align="right">#[chunkLURL]#</td>
<td class="small" align="right">#[slempLURL]#</td> <td class="small" align="right">#[slempLURL]#</td>
<td class="small" align="right">#[slhigLURL]#<br>#[slmedLURL]#<br>#[sllowLURL]#</td> <td class="small" align="right">#[slfilLURL]#</td>
<td class="small" align="right">#[slhittmissLURL]#<br>#[sluniqdoubLURL]#<br>#[slflushLURL]#</td> <td class="small" align="right">#[slhittmissLURL]#<br>#[sluniqdoubLURL]#<br>#[slflushLURL]#</td>
<td class="small" align="right">#[ochunkmaxLURL]#</td> <td class="small" align="right">#[ochunkmaxLURL]#</td>
<td class="small" align="right">#[ochunkcurLURL]#</td> <td class="small" align="right">#[ochunkcurLURL]#</td>
@ -159,7 +156,6 @@ Increasing this cache will be most important for a fast proxy mode.</td>
<td class="small" align="right">#[usedLURL]#</td> <td class="small" align="right">#[usedLURL]#</td>
<td class="small" align="right"><input name="ramCacheLURL" type="text" align="right" size="6" maxlength="7" value="#[ramCacheLURL]#"></td> <td class="small" align="right"><input name="ramCacheLURL" type="text" align="right" size="6" maxlength="7" value="#[ramCacheLURL]#"></td>
<td class="small" align="right">#[dfltLURL]#</td> <td class="small" align="right">#[dfltLURL]#</td>
<td class="small" align="right">#[goodLURL]#</td>
<td class="small" align="right">#[bestLURL]#</td> <td class="small" align="right">#[bestLURL]#</td>
<td class="small" align="left">This is the database that holds the hash/url - relation and properties regarding the url like load date and server date. <td class="small" align="left">This is the database that holds the hash/url - relation and properties regarding the url like load date and server date.
This cache is very important for a fast search process. Increasing the cache size will result in more search results and less IO during DHT transfer.</td> This cache is very important for a fast search process. Increasing the cache size will result in more search results and less IO during DHT transfer.</td>
@ -168,9 +164,9 @@ This cache is very important for a fast search process. Increasing the cache siz
<tr class="TableCellDark"> <tr class="TableCellDark">
<td class="small" align="left">'noticed' URLs</td> <td class="small" align="left">'noticed' URLs</td>
<td class="small" align="right">#[slreqNURL]#</td> <td class="small" align="right">#[slreqNURL]#</td>
<td class="small" align="center">#[chunkNURL]#</td> <td class="small" align="right">#[chunkNURL]#</td>
<td class="small" align="right">#[slempNURL]#</td> <td class="small" align="right">#[slempNURL]#</td>
<td class="small" align="right">#[slhigNURL]#<br>#[slmedNURL]#<br>#[sllowNURL]#</td> <td class="small" align="right">#[slfilNURL]#</td>
<td class="small" align="right">#[slhittmissNURL]#<br>#[sluniqdoubNURL]#<br>#[slflushNURL]#</td> <td class="small" align="right">#[slhittmissNURL]#<br>#[sluniqdoubNURL]#<br>#[slflushNURL]#</td>
<td class="small" align="right">#[ochunkmaxNURL]#</td> <td class="small" align="right">#[ochunkmaxNURL]#</td>
<td class="small" align="right">#[ochunkcurNURL]#</td> <td class="small" align="right">#[ochunkcurNURL]#</td>
@ -179,7 +175,6 @@ This cache is very important for a fast search process. Increasing the cache siz
<td class="small" align="right">#[usedNURL]#</td> <td class="small" align="right">#[usedNURL]#</td>
<td class="small" align="right"><input name="ramCacheNURL" type="text" align="right" size="6" maxlength="7" value="#[ramCacheNURL]#"></td> <td class="small" align="right"><input name="ramCacheNURL" type="text" align="right" size="6" maxlength="7" value="#[ramCacheNURL]#"></td>
<td class="small" align="right">#[dfltNURL]#</td> <td class="small" align="right">#[dfltNURL]#</td>
<td class="small" align="right">#[goodNURL]#</td>
<td class="small" align="right">#[bestNURL]#</td> <td class="small" align="right">#[bestNURL]#</td>
<td class="small" align="left">A noticed URL is one that was discovered during crawling but was not loaded yet. <td class="small" align="left">A noticed URL is one that was discovered during crawling but was not loaded yet.
Increasing the cache size will result in faster double-check during URL recognition when doing crawls.</td> Increasing the cache size will result in faster double-check during URL recognition when doing crawls.</td>
@ -188,9 +183,9 @@ Increasing the cache size will result in faster double-check during URL recognit
<tr class="TableCellDark"> <tr class="TableCellDark">
<td class="small" align="left">'error' URLs</td> <td class="small" align="left">'error' URLs</td>
<td class="small" align="right">#[slreqEURL]#</td> <td class="small" align="right">#[slreqEURL]#</td>
<td class="small" align="center">#[chunkEURL]#</td> <td class="small" align="right">#[chunkEURL]#</td>
<td class="small" align="right">#[slempEURL]#</td> <td class="small" align="right">#[slempEURL]#</td>
<td class="small" align="right">#[slhigEURL]#<br>#[slmedEURL]#<br>#[sllowEURL]#</td> <td class="small" align="right">#[slfilEURL]#</td>
<td class="small" align="right">#[slhittmissEURL]#<br>#[sluniqdoubEURL]#<br>#[slflushEURL]#</td> <td class="small" align="right">#[slhittmissEURL]#<br>#[sluniqdoubEURL]#<br>#[slflushEURL]#</td>
<td class="small" align="right">#[ochunkmaxEURL]#</td> <td class="small" align="right">#[ochunkmaxEURL]#</td>
<td class="small" align="right">#[ochunkcurEURL]#</td> <td class="small" align="right">#[ochunkcurEURL]#</td>
@ -199,7 +194,6 @@ Increasing the cache size will result in faster double-check during URL recognit
<td class="small" align="right">#[usedEURL]#</td> <td class="small" align="right">#[usedEURL]#</td>
<td class="small" align="right"><input name="ramCacheEURL" type="text" align="right" size="6" maxlength="7" value="#[ramCacheEURL]#"></td> <td class="small" align="right"><input name="ramCacheEURL" type="text" align="right" size="6" maxlength="7" value="#[ramCacheEURL]#"></td>
<td class="small" align="right">#[dfltEURL]#</td> <td class="small" align="right">#[dfltEURL]#</td>
<td class="small" align="right">#[goodEURL]#</td>
<td class="small" align="right">#[bestEURL]#</td> <td class="small" align="right">#[bestEURL]#</td>
<td class="small" align="left">URLs that cannot be loaded are stored in this database. It is also used for double-checked during crawling. <td class="small" align="left">URLs that cannot be loaded are stored in this database. It is also used for double-checked during crawling.
Increasing the cache size will most probably speed up crawling slightly, but not significantly.</td> Increasing the cache size will most probably speed up crawling slightly, but not significantly.</td>
@ -208,9 +202,9 @@ Increasing the cache size will most probably speed up crawling slightly, but not
<tr class="TableCellDark"> <tr class="TableCellDark">
<td class="small" align="left">DHT Control</td> <td class="small" align="left">DHT Control</td>
<td class="small" align="right">#[slreqDHT]#</td> <td class="small" align="right">#[slreqDHT]#</td>
<td class="small" align="center">#[chunkDHT]#</td> <td class="small" align="right">#[chunkDHT]#</td>
<td class="small" align="right">#[slempDHT]#</td> <td class="small" align="right">#[slempDHT]#</td>
<td class="small" align="right">#[slhigDHT]#<br>#[slmedDHT]#<br>#[sllowDHT]#</td> <td class="small" align="right">#[slfilDHT]#</td>
<td class="small" align="right">#[slhittmissDHT]#<br>#[sluniqdoubDHT]#<br>#[slflushDHT]#</td> <td class="small" align="right">#[slhittmissDHT]#<br>#[sluniqdoubDHT]#<br>#[slflushDHT]#</td>
<td class="small" align="right">#[ochunkmaxDHT]#</td> <td class="small" align="right">#[ochunkmaxDHT]#</td>
<td class="small" align="right">#[ochunkcurDHT]#</td> <td class="small" align="right">#[ochunkcurDHT]#</td>
@ -219,7 +213,6 @@ Increasing the cache size will most probably speed up crawling slightly, but not
<td class="small" align="right">#[usedDHT]#</td> <td class="small" align="right">#[usedDHT]#</td>
<td class="small" align="right"><input name="ramCacheDHT" type="text" align="right" size="6" maxlength="7" value="#[ramCacheDHT]#"></td> <td class="small" align="right"><input name="ramCacheDHT" type="text" align="right" size="6" maxlength="7" value="#[ramCacheDHT]#"></td>
<td class="small" align="right">#[dfltDHT]#</td> <td class="small" align="right">#[dfltDHT]#</td>
<td class="small" align="right">#[goodDHT]#</td>
<td class="small" align="right">#[bestDHT]#</td> <td class="small" align="right">#[bestDHT]#</td>
<td class="small" align="left">This is simply the cache for the seed-dbs (active, passive, potential). <td class="small" align="left">This is simply the cache for the seed-dbs (active, passive, potential).
This cache is divided into three equal parts. This cache is divided into three equal parts.
@ -229,9 +222,9 @@ Increasing this cache may speed up many functions, but we need to test this to s
<tr class="TableCellDark"> <tr class="TableCellDark">
<td class="small" align="left">Messages</td> <td class="small" align="left">Messages</td>
<td class="small" align="right">#[slreqMessage]#</td> <td class="small" align="right">#[slreqMessage]#</td>
<td class="small" align="center">#[chunkMessage]#</td> <td class="small" align="right">#[chunkMessage]#</td>
<td class="small" align="right">#[slempMessage]#</td> <td class="small" align="right">#[slempMessage]#</td>
<td class="small" align="right">#[slhigMessage]#<br>#[slmedMessage]#<br>#[sllowMessage]#</td> <td class="small" align="right">#[slfilMessage]#</td>
<td class="small" align="right">#[slhittmissMessage]#<br>#[sluniqdoubMessage]#<br>#[slflushMessage]#</td> <td class="small" align="right">#[slhittmissMessage]#<br>#[sluniqdoubMessage]#<br>#[slflushMessage]#</td>
<td class="small" align="right">#[ochunkmaxMessage]#</td> <td class="small" align="right">#[ochunkmaxMessage]#</td>
<td class="small" align="right">#[ochunkcurMessage]#</td> <td class="small" align="right">#[ochunkcurMessage]#</td>
@ -240,7 +233,6 @@ Increasing this cache may speed up many functions, but we need to test this to s
<td class="small" align="right">#[usedMessage]#</td> <td class="small" align="right">#[usedMessage]#</td>
<td class="small" align="right"><input name="ramCacheMessage" type="text" align="right" size="6" maxlength="7" value="#[ramCacheMessage]#"></td> <td class="small" align="right"><input name="ramCacheMessage" type="text" align="right" size="6" maxlength="7" value="#[ramCacheMessage]#"></td>
<td class="small" align="right">#[dfltMessage]#</td> <td class="small" align="right">#[dfltMessage]#</td>
<td class="small" align="right">#[goodMessage]#</td>
<td class="small" align="right">#[bestMessage]#</td> <td class="small" align="right">#[bestMessage]#</td>
<td class="small" align="left">The Message cache for peer-to-peer messages. Less important.</td> <td class="small" align="left">The Message cache for peer-to-peer messages. Less important.</td>
</tr> </tr>
@ -248,9 +240,9 @@ Increasing this cache may speed up many functions, but we need to test this to s
<tr class="TableCellDark"> <tr class="TableCellDark">
<td class="small" align="left">Wiki</td> <td class="small" align="left">Wiki</td>
<td class="small" align="right">#[slreqWiki]#</td> <td class="small" align="right">#[slreqWiki]#</td>
<td class="small" align="center">#[chunkWiki]#</td> <td class="small" align="right">#[chunkWiki]#</td>
<td class="small" align="right">#[slempWiki]#</td> <td class="small" align="right">#[slempWiki]#</td>
<td class="small" align="right">#[slhigWiki]#<br>#[slmedWiki]#<br>#[sllowWiki]#</td> <td class="small" align="right">#[slfilWiki]#</td>
<td class="small" align="right">#[slhittmissWiki]#<br>#[sluniqdoubWiki]#<br>#[slflushWiki]#</td> <td class="small" align="right">#[slhittmissWiki]#<br>#[sluniqdoubWiki]#<br>#[slflushWiki]#</td>
<td class="small" align="right">#[ochunkmaxWiki]#</td> <td class="small" align="right">#[ochunkmaxWiki]#</td>
<td class="small" align="right">#[ochunkcurWiki]#</td> <td class="small" align="right">#[ochunkcurWiki]#</td>
@ -259,7 +251,6 @@ Increasing this cache may speed up many functions, but we need to test this to s
<td class="small" align="right">#[usedWiki]#</td> <td class="small" align="right">#[usedWiki]#</td>
<td class="small" align="right"><input name="ramCacheWiki" type="text" align="right" size="6" maxlength="7" value="#[ramCacheWiki]#"></td> <td class="small" align="right"><input name="ramCacheWiki" type="text" align="right" size="6" maxlength="7" value="#[ramCacheWiki]#"></td>
<td class="small" align="right">#[dfltWiki]#</td> <td class="small" align="right">#[dfltWiki]#</td>
<td class="small" align="right">#[goodWiki]#</td>
<td class="small" align="right">#[bestWiki]#</td> <td class="small" align="right">#[bestWiki]#</td>
<td class="small" align="left">The YaCy-Wiki uses a database to store its pages. <td class="small" align="left">The YaCy-Wiki uses a database to store its pages.
This cache is divided in two parts, one for the wiki database and one for its backup. This cache is divided in two parts, one for the wiki database and one for its backup.
@ -269,9 +260,9 @@ Increasing this cache may speed up access to the wiki pages.</td>
<tr class="TableCellDark"> <tr class="TableCellDark">
<td class="small" align="left">Blog</td> <td class="small" align="left">Blog</td>
<td class="small" align="right">#[slreqBlog]#</td> <td class="small" align="right">#[slreqBlog]#</td>
<td class="small" align="center">#[chunkBlog]#</td> <td class="small" align="right">#[chunkBlog]#</td>
<td class="small" align="right">#[slempBlog]#</td> <td class="small" align="right">#[slempBlog]#</td>
<td class="small" align="right">#[slhigBlog]#<br>#[slmedBlog]#<br>#[sllowBlog]#</td> <td class="small" align="right">#[slfilBlog]#</td>
<td class="small" align="right">#[slhittmissBlog]#<br>#[sluniqdoubBlog]#<br>#[slflushBlog]#</td> <td class="small" align="right">#[slhittmissBlog]#<br>#[sluniqdoubBlog]#<br>#[slflushBlog]#</td>
<td class="small" align="right">#[ochunkmaxBlog]#</td> <td class="small" align="right">#[ochunkmaxBlog]#</td>
<td class="small" align="right">#[ochunkcurBlog]#</td> <td class="small" align="right">#[ochunkcurBlog]#</td>
@ -280,7 +271,6 @@ Increasing this cache may speed up access to the wiki pages.</td>
<td class="small" align="right">#[usedBlog]#</td> <td class="small" align="right">#[usedBlog]#</td>
<td class="small" align="right"><input name="ramCacheBlog" type="text" align="right" size="6" maxlength="7" value="#[ramCacheBlog]#"></td> <td class="small" align="right"><input name="ramCacheBlog" type="text" align="right" size="6" maxlength="7" value="#[ramCacheBlog]#"></td>
<td class="small" align="right">#[dfltBlog]#</td> <td class="small" align="right">#[dfltBlog]#</td>
<td class="small" align="right">#[goodBlog]#</td>
<td class="small" align="right">#[bestBlog]#</td> <td class="small" align="right">#[bestBlog]#</td>
<td class="small" align="left">The YaCy-Blog uses a database to store its entries. <td class="small" align="left">The YaCy-Blog uses a database to store its entries.
Increasing this cache may speed up access to the Blog.</td> Increasing this cache may speed up access to the Blog.</td>
@ -289,9 +279,9 @@ Increasing this cache may speed up access to the Blog.</td>
<tr class="TableCellDark"> <tr class="TableCellDark">
<td class="small" align="left">News</td> <td class="small" align="left">News</td>
<td class="small" align="right">#[slreqNews]#</td> <td class="small" align="right">#[slreqNews]#</td>
<td class="small" align="center">#[chunkNews]#</td> <td class="small" align="right">#[chunkNews]#</td>
<td class="small" align="right">#[slempNews]#</td> <td class="small" align="right">#[slempNews]#</td>
<td class="small" align="right">#[slhigNews]#<br>#[slmedNews]#<br>#[sllowNews]#</td> <td class="small" align="right">#[slfilNews]#</td>
<td class="small" align="right">#[slhittmissNews]#<br>#[sluniqdoubNews]#<br>#[slflushNews]#</td> <td class="small" align="right">#[slhittmissNews]#<br>#[sluniqdoubNews]#<br>#[slflushNews]#</td>
<td class="small" align="right">#[ochunkmaxNews]#</td> <td class="small" align="right">#[ochunkmaxNews]#</td>
<td class="small" align="right">#[ochunkcurNews]#</td> <td class="small" align="right">#[ochunkcurNews]#</td>
@ -300,7 +290,6 @@ Increasing this cache may speed up access to the Blog.</td>
<td class="small" align="right">#[usedNews]#</td> <td class="small" align="right">#[usedNews]#</td>
<td class="small" align="right"><input name="ramCacheNews" type="text" align="right" size="6" maxlength="7" value="#[ramCacheNews]#"></td> <td class="small" align="right"><input name="ramCacheNews" type="text" align="right" size="6" maxlength="7" value="#[ramCacheNews]#"></td>
<td class="small" align="right">#[dfltNews]#</td> <td class="small" align="right">#[dfltNews]#</td>
<td class="small" align="right">#[goodNews]#</td>
<td class="small" align="right">#[bestNews]#</td> <td class="small" align="right">#[bestNews]#</td>
<td class="small" align="left">The News-DB stores property-lists for news that are included in seeds. <td class="small" align="left">The News-DB stores property-lists for news that are included in seeds.
Increasing this cache may speed up the peer-ping.</td> Increasing this cache may speed up the peer-ping.</td>
@ -309,9 +298,9 @@ Increasing this cache may speed up the peer-ping.</td>
<tr class="TableCellDark"> <tr class="TableCellDark">
<td class="small" align="left">robots.txt DB</td> <td class="small" align="left">robots.txt DB</td>
<td class="small" align="right">#[slreqRobots]#</td> <td class="small" align="right">#[slreqRobots]#</td>
<td class="small" align="center">#[chunkRobots]#</td> <td class="small" align="right">#[chunkRobots]#</td>
<td class="small" align="right">#[slempRobots]#</td> <td class="small" align="right">#[slempRobots]#</td>
<td class="small" align="right">#[slhigRobots]#<br>#[slmedRobots]#<br>#[sllowRobots]#</td> <td class="small" align="right">#[slfilRobots]#</td>
<td class="small" align="right">#[slhittmissRobots]#<br>#[sluniqdoubRobots]#<br>#[slflushRobots]#</td> <td class="small" align="right">#[slhittmissRobots]#<br>#[sluniqdoubRobots]#<br>#[slflushRobots]#</td>
<td class="small" align="right">#[ochunkmaxRobots]#</td> <td class="small" align="right">#[ochunkmaxRobots]#</td>
<td class="small" align="right">#[ochunkcurRobots]#</td> <td class="small" align="right">#[ochunkcurRobots]#</td>
@ -320,7 +309,6 @@ Increasing this cache may speed up the peer-ping.</td>
<td class="small" align="right">#[usedRobots]#</td> <td class="small" align="right">#[usedRobots]#</td>
<td class="small" align="right"><input name="ramCacheRobots" type="text" align="right" size="6" maxlength="7" value="#[ramCacheRobots]#"></td> <td class="small" align="right"><input name="ramCacheRobots" type="text" align="right" size="6" maxlength="7" value="#[ramCacheRobots]#"></td>
<td class="small" align="right">#[dfltRobots]#</td> <td class="small" align="right">#[dfltRobots]#</td>
<td class="small" align="right">#[goodRobots]#</td>
<td class="small" align="right">#[bestRobots]#</td> <td class="small" align="right">#[bestRobots]#</td>
<td class="small" align="left">The robots.txt DB stores downloaded records from robots.txt files. <td class="small" align="left">The robots.txt DB stores downloaded records from robots.txt files.
Increasing this cache may speed up validation if crawling of the URL is allowed.</td> Increasing this cache may speed up validation if crawling of the URL is allowed.</td>
@ -329,9 +317,9 @@ Increasing this cache may speed up validation if crawling of the URL is allowed.
<tr class="TableCellDark"> <tr class="TableCellDark">
<td class="small" align="left">Crawl Profiles</td> <td class="small" align="left">Crawl Profiles</td>
<td class="small" align="right">#[slreqProfiles]#</td> <td class="small" align="right">#[slreqProfiles]#</td>
<td class="small" align="center">#[chunkProfiles]#</td> <td class="small" align="right">#[chunkProfiles]#</td>
<td class="small" align="right">#[slempProfiles]#</td> <td class="small" align="right">#[slempProfiles]#</td>
<td class="small" align="right">#[slhigProfiles]#<br>#[slmedProfiles]#<br>#[sllowProfiles]#</td> <td class="small" align="right">#[slfilProfiles]#</td>
<td class="small" align="right">#[slhittmissProfiles]#<br>#[sluniqdoubProfiles]#<br>#[slflushProfiles]#</td> <td class="small" align="right">#[slhittmissProfiles]#<br>#[sluniqdoubProfiles]#<br>#[slflushProfiles]#</td>
<td class="small" align="right">#[ochunkmaxProfiles]#</td> <td class="small" align="right">#[ochunkmaxProfiles]#</td>
<td class="small" align="right">#[ochunkcurProfiles]#</td> <td class="small" align="right">#[ochunkcurProfiles]#</td>
@ -340,7 +328,6 @@ Increasing this cache may speed up validation if crawling of the URL is allowed.
<td class="small" align="right">#[usedProfiles]#</td> <td class="small" align="right">#[usedProfiles]#</td>
<td class="small" align="right"><input name="ramCacheProfiles" type="text" align="right" size="6" maxlength="7" value="#[ramCacheProfiles]#"></td> <td class="small" align="right"><input name="ramCacheProfiles" type="text" align="right" size="6" maxlength="7" value="#[ramCacheProfiles]#"></td>
<td class="small" align="right">#[dfltProfiles]#</td> <td class="small" align="right">#[dfltProfiles]#</td>
<td class="small" align="right">#[goodProfiles]#</td>
<td class="small" align="right">#[bestProfiles]#</td> <td class="small" align="right">#[bestProfiles]#</td>
<td class="small" align="left">The profile database stores properties for each crawl that is started on the local peer. <td class="small" align="left">The profile database stores properties for each crawl that is started on the local peer.
Increasing this cache may speed up crawling, but not much space is needed, so the effect may be low.</td> Increasing this cache may speed up crawling, but not much space is needed, so the effect may be low.</td>

25
htroot/PerformanceMemory_p.java
Unescape View File

@ -62,11 +62,12 @@ public class PerformanceMemory_p {
private static final long MB = 1024 * KB; private static final long MB = 1024 * KB;
private static Map defaultSettings = null; private static Map defaultSettings = null;
private static int[] slt,chk; private static int chk;
private static int[] slt;
private static String[] ost; private static String[] ost;
private static long req, usd, bst, god; private static long req, usd, bst;
private static long usedTotal, currTotal, dfltTotal, goodTotal, bestTotal; private static long usedTotal, currTotal, dfltTotal, bestTotal;
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
// return variable that accumulates replacements // return variable that accumulates replacements
@ -160,7 +161,6 @@ public class PerformanceMemory_p {
usedTotal = 0; usedTotal = 0;
currTotal = 0; currTotal = 0;
dfltTotal = 0; dfltTotal = 0;
goodTotal = 0;
bestTotal = 0; bestTotal = 0;
req = sb.wordIndex.size(); req = sb.wordIndex.size();
@ -238,7 +238,6 @@ public class PerformanceMemory_p {
prop.put("usedTotal", usedTotal / MB); prop.put("usedTotal", usedTotal / MB);
prop.put("currTotal", currTotal / MB); prop.put("currTotal", currTotal / MB);
prop.put("dfltTotal", dfltTotal / MB); prop.put("dfltTotal", dfltTotal / MB);
prop.put("goodTotal", goodTotal / MB);
prop.put("bestTotal", bestTotal / MB); prop.put("bestTotal", bestTotal / MB);
// parse initialization memory settings // parse initialization memory settings
@ -294,17 +293,13 @@ public class PerformanceMemory_p {
} }
private static void putprop(serverObjects prop, serverSwitch env, String db, String set) { private static void putprop(serverObjects prop, serverSwitch env, String db, String set) {
usd = ((long) chk[0]) * ((long) slt[3]) + ((long) chk[1]) * ((long) slt[2]) + ((long) chk[2]) * ((long) slt[1]); usd = ((long) chk) * ((long) slt[1]);
bst = (((((long) chk[2]) * ((long) req)) >> 10) + 1) << 10; bst = (((((long) chk) * ((long) req)) >> 10) + 1) << 10;
god = (((((long) bst) / ((long) (1+slt[1]+slt[2]+slt[3])) * ((long) slt[1])) >> 10) + 1) << 10;
if (set.equals("setGood")) env.setConfig("ramCache" + db, god);
if (set.equals("setBest")) env.setConfig("ramCache" + db, bst); if (set.equals("setBest")) env.setConfig("ramCache" + db, bst);
prop.put("chunk" + db, chk[2] + "/" + chk[1] + "/" + chk[0]); prop.put("chunk" + db, chk);
prop.put("slreq" + db, req); prop.put("slreq" + db, req);
prop.put("slemp" + db, slt[0] - slt[1] - slt[2] - slt[3]); prop.put("slemp" + db, slt[0] - slt[1]);
prop.put("slhig" + db, slt[1]); prop.put("slfil" + db, slt[1]);
prop.put("slmed" + db, slt[2]);
prop.put("sllow" + db, slt[3]);
prop.put("slhittmiss" + db, slt[4] + ":" + slt[5]); prop.put("slhittmiss" + db, slt[4] + ":" + slt[5]);
prop.put("sluniqdoub" + db, slt[6] + ":" + slt[7]); prop.put("sluniqdoub" + db, slt[6] + ":" + slt[7]);
prop.put("slflush" + db, slt[8] + ":" + slt[9]); prop.put("slflush" + db, slt[8] + ":" + slt[9]);
@ -317,14 +312,12 @@ public class PerformanceMemory_p {
prop.put("nuniqdoub" + db, ost[14] + ":" + ost[15]); prop.put("nuniqdoub" + db, ost[14] + ":" + ost[15]);
prop.put("nflush" + db, ost[16] + ":" + ost[17]); prop.put("nflush" + db, ost[16] + ":" + ost[17]);
prop.put("used" + db, usd / KB); prop.put("used" + db, usd / KB);
prop.put("good" + db, god / KB);
prop.put("best" + db, bst / KB); prop.put("best" + db, bst / KB);
prop.put("dflt" + db, Long.parseLong((String) defaultSettings.get("ramCache" + db)) / KB); prop.put("dflt" + db, Long.parseLong((String) defaultSettings.get("ramCache" + db)) / KB);
prop.put("ramCache" + db, Long.parseLong(env.getConfig("ramCache" + db, "0")) / KB); prop.put("ramCache" + db, Long.parseLong(env.getConfig("ramCache" + db, "0")) / KB);
usedTotal += usd; usedTotal += usd;
currTotal += Long.parseLong(env.getConfig("ramCache" + db, "0")); currTotal += Long.parseLong(env.getConfig("ramCache" + db, "0"));
dfltTotal += Long.parseLong((String) defaultSettings.get("ramCache" + db)); dfltTotal += Long.parseLong((String) defaultSettings.get("ramCache" + db));
goodTotal += god;
bestTotal += bst; bestTotal += bst;
} }
} }

2
source/de/anomic/data/blogBoard.java
Unescape View File

@ -97,7 +97,7 @@ public class blogBoard {
return datbase.size(); return datbase.size();
} }
public int[] dbCacheNodeChunkSize() { public int dbCacheNodeChunkSize() {
return datbase.cacheNodeChunkSize(); return datbase.cacheNodeChunkSize();
} }

2
source/de/anomic/data/messageBoard.java
Unescape View File

@ -86,7 +86,7 @@ public class messageBoard {
return database.size(); return database.size();
} }
public int[] dbCacheNodeChunkSize() { public int dbCacheNodeChunkSize() {
return database.cacheNodeChunkSize(); return database.cacheNodeChunkSize();
} }

2
source/de/anomic/data/userDB.java
Unescape View File

@ -92,7 +92,7 @@ public final class userDB {
} }
} }
public int[] dbCacheNodeChunkSize() { public int dbCacheNodeChunkSize() {
return userTable.cacheNodeChunkSize(); return userTable.cacheNodeChunkSize();
} }

12
source/de/anomic/data/wikiBoard.java
Unescape View File

@ -100,14 +100,10 @@ public class wikiBoard {
return datbase.size(); return datbase.size();
} }
public int[] dbCacheNodeChunkSize() { public int dbCacheNodeChunkSize() {
int[] db = datbase.cacheNodeChunkSize(); int db = datbase.cacheNodeChunkSize();
int[] bk = bkpbase.cacheNodeChunkSize(); int bk = bkpbase.cacheNodeChunkSize();
int[] i = new int[3]; return (db + bk) / 2;
i[kelondroRecords.CP_LOW] = (db[kelondroRecords.CP_LOW] + bk[kelondroRecords.CP_LOW]) / 2;
i[kelondroRecords.CP_MEDIUM] = (db[kelondroRecords.CP_MEDIUM] + bk[kelondroRecords.CP_MEDIUM]) / 2;
i[kelondroRecords.CP_HIGH] = (db[kelondroRecords.CP_HIGH] + bk[kelondroRecords.CP_HIGH]) / 2;
return i;
} }
public int[] dbCacheNodeStatus() { public int[] dbCacheNodeStatus() {

2
source/de/anomic/index/indexURL.java
Unescape View File

@ -430,7 +430,7 @@ public class indexURL {
if (urlHashCache != null) urlHashCache.close(); if (urlHashCache != null) urlHashCache.close();
} }
public int[] cacheNodeChunkSize() { public int cacheNodeChunkSize() {
return urlHashCache.cacheNodeChunkSize(); return urlHashCache.cacheNodeChunkSize();
} }

2
source/de/anomic/kelondro/kelondroIntBytesMap.java
Unescape View File

@ -24,7 +24,7 @@
package de.anomic.kelondro; package de.anomic.kelondro;
public class kelondroIntBytesMap extends kelondroRowSet { public class kelondroIntBytesMap extends kelondroRowBufferedSet {
public kelondroIntBytesMap(int payloadSize, int initSize) { public kelondroIntBytesMap(int payloadSize, int initSize) {
super(new kelondroRow(new int[]{4, payloadSize}), initSize); super(new kelondroRow(new int[]{4, payloadSize}), initSize);

2
source/de/anomic/kelondro/kelondroMap.java
Unescape View File

@ -133,7 +133,7 @@ public class kelondroMap {
return dyn.row().width(0); return dyn.row().width(0);
} }
public int[] cacheNodeChunkSize() { public int cacheNodeChunkSize() {
return dyn.cacheNodeChunkSize(); return dyn.cacheNodeChunkSize();
} }

186
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -71,7 +71,6 @@ package de.anomic.kelondro;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.HashSet; import java.util.HashSet;
import java.util.NoSuchElementException;
import java.util.Random; import java.util.Random;
import java.util.StringTokenizer; import java.util.StringTokenizer;
import java.util.Iterator; import java.util.Iterator;
@ -86,7 +85,6 @@ public class kelondroRecords {
// memory calculation // memory calculation
private static final int element_in_cache = 4; // for kelondroCollectionObjectMap: 4; for HashMap: 52 private static final int element_in_cache = 4; // for kelondroCollectionObjectMap: 4; for HashMap: 52
private static final int cache_control_entry = 96;
// caching flags // caching flags
public static final int CP_NONE = -1; // cache priority none; entry shall not be cached public static final int CP_NONE = -1; // cache priority none; entry shall not be cached
@ -141,10 +139,8 @@ public class kelondroRecords {
private int TXTPROPW; // size of a single TXTPROPS element private int TXTPROPW; // size of a single TXTPROPS element
// caching buffer // caching buffer
private kelondroIntBytesMap[] cacheHeaders; // the cache; holds overhead values and key element private kelondroIntBytesMap cacheHeaders; // the cache; holds overhead values and key element
private int cacheSize; // number of cache records private int cacheSize; // number of cache records
private long cacheStartup; // startup time; for cache aging
private kelondroMScoreCluster cacheScore; // controls cache aging
private int readHit, readMiss, writeUnique, writeDouble, cacheDelete, cacheFlush; private int readHit, readMiss, writeUnique, writeDouble, cacheDelete, cacheFlush;
// optional logger // optional logger
@ -417,25 +413,11 @@ public class kelondroRecords {
if (buffersize <= 0) { if (buffersize <= 0) {
this.cacheSize = 0; this.cacheSize = 0;
this.cacheHeaders = null; this.cacheHeaders = null;
this.cacheScore = null;
} else { } else {
if ((buffersize / cacheNodeChunkSize(false)) > size()) { this.cacheSize = (int) (buffersize / cacheNodeChunkSize());
this.cacheSize = (int) (buffersize / cacheNodeChunkSize(false)); this.cacheHeaders = new kelondroIntBytesMap(this.headchunksize, this.cacheSize / 4);
this.cacheScore = null; // no cache control because we have more cache slots than database entries this.cacheHeaders.setOrdering(kelondroNaturalOrder.naturalOrder, 0);
} else {
this.cacheSize = (int) (buffersize / cacheNodeChunkSize(true));
this.cacheScore = new kelondroMScoreCluster(); // cache control of CP_HIGH caches
}
this.cacheHeaders = new kelondroIntBytesMap[]{
new kelondroIntBytesMap(this.headchunksize, this.cacheSize / 4),
new kelondroIntBytesMap(this.headchunksize, 0),
new kelondroIntBytesMap(this.headchunksize, this.cacheSize / 4)
};
this.cacheHeaders[0].setOrdering(kelondroNaturalOrder.naturalOrder, 0);
this.cacheHeaders[1].setOrdering(kelondroNaturalOrder.naturalOrder, 0);
this.cacheHeaders[2].setOrdering(kelondroNaturalOrder.naturalOrder, 0);
} }
this.cacheStartup = System.currentTimeMillis();
this.readHit = 0; this.readHit = 0;
this.readMiss = 0; this.readMiss = 0;
this.writeUnique = 0; this.writeUnique = 0;
@ -457,29 +439,17 @@ public class kelondroRecords {
return new File(filename); return new File(filename);
} }
protected final int cacheNodeChunkSize(boolean cacheControl) { public final int cacheNodeChunkSize() {
return this.headchunksize + element_in_cache + ((cacheControl) ? cache_control_entry : 0); return this.headchunksize + element_in_cache;
}
public int[] cacheNodeChunkSize() {
// returns three integers:
// #0: chunk size of CP_LOW - priority entries
// #1: chunk size of CP_MEDIUM - priority entries
// #2: chunk size of CP_HIGH - priority entries
int[] i = new int[3];
i[CP_LOW] = cacheNodeChunkSize(false);
i[CP_MEDIUM] = cacheNodeChunkSize(false);
i[CP_HIGH] = cacheNodeChunkSize(this.cacheScore != null);
return i;
} }
public int[] cacheNodeStatus() { public int[] cacheNodeStatus() {
if (cacheHeaders == null) return new int[]{0,0,0,0,0,0,0,0,0,0}; if (cacheHeaders == null) return new int[]{0,0,0,0,0,0,0,0,0,0};
return new int[]{ return new int[]{
cacheSize, cacheSize,
cacheHeaders[CP_HIGH].size(), cacheHeaders.size(),
cacheHeaders[CP_MEDIUM].size(), 0, // not used
cacheHeaders[CP_LOW].size(), 0, // not used
readHit, readHit,
readMiss, readMiss,
writeUnique, writeUnique,
@ -492,9 +462,7 @@ public class kelondroRecords {
public String cacheNodeStatusString() { public String cacheNodeStatusString() {
return return
"cacheMaxSize=" + cacheSize + "cacheMaxSize=" + cacheSize +
", CP_HIGH=" + cacheHeaders[CP_HIGH].size() + ", cacheCurrSize=" + cacheHeaders.size() +
", CP_MEDIUM=" + cacheHeaders[CP_MEDIUM].size() +
", CP_LOW=" + cacheHeaders[CP_LOW].size() +
", readHit=" + readHit + ", readHit=" + readHit +
", readMiss=" + readMiss + ", readMiss=" + readMiss +
", writeUnique=" + writeUnique + ", writeUnique=" + writeUnique +
@ -531,20 +499,7 @@ public class kelondroRecords {
protected void deleteNode(Handle handle) throws IOException { protected void deleteNode(Handle handle) throws IOException {
if (cacheSize != 0) { if (cacheSize != 0) {
synchronized (cacheHeaders) { synchronized (cacheHeaders) {
if (cacheScore == null) { cacheHeaders.removeb(handle.index);
cacheHeaders[CP_LOW].removeb(handle.index);
cacheHeaders[CP_MEDIUM].removeb(handle.index);
cacheHeaders[CP_HIGH].removeb(handle.index);
} else if (cacheHeaders[CP_HIGH].getb(handle.index) != null) {
// remove handle from cache-control
cacheScore.deleteScore(handle);
cacheHeaders[CP_HIGH].removeb(handle.index);
} else {
// no cache control for medium-priority entries and
// no cache control for low-priority entries
cacheHeaders[CP_MEDIUM].removeb(handle.index);
cacheHeaders[CP_LOW].removeb(handle.index);
}
cacheDelete++; cacheDelete++;
} }
} }
@ -657,15 +612,7 @@ public class kelondroRecords {
} else synchronized(cacheHeaders) { } else synchronized(cacheHeaders) {
byte[] cacheEntry = null; byte[] cacheEntry = null;
int cp = CP_HIGH; int cp = CP_HIGH;
cacheEntry = cacheHeaders[CP_HIGH].getb(this.handle.index); // first try cacheEntry = cacheHeaders.getb(this.handle.index);
if (cacheEntry == null) {
cacheEntry = cacheHeaders[CP_MEDIUM].getb(this.handle.index); // second try
cp = CP_MEDIUM;
}
if (cacheEntry == null) {
cacheEntry = cacheHeaders[CP_LOW].getb(this.handle.index); // third try
cp = CP_LOW;
}
if (cacheEntry == null) { if (cacheEntry == null) {
// cache miss, we read overhead and key from file // cache miss, we read overhead and key from file
readMiss++; readMiss++;
@ -694,10 +641,6 @@ public class kelondroRecords {
//this.headChunk = new byte[headchunksize]; //this.headChunk = new byte[headchunksize];
//System.arraycopy(cacheEntry, 0, this.headChunk, 0, headchunksize); //System.arraycopy(cacheEntry, 0, this.headChunk, 0, headchunksize);
this.headChunk = cacheEntry; this.headChunk = cacheEntry;
// update cache scores to announce this cache hit
if ((cacheScore != null) && (cp == CP_HIGH)) {
cacheScore.setScore(this.handle, (int) ((System.currentTimeMillis() - cacheStartup) / 1000));
}
this.headChanged = false; this.headChanged = false;
} }
} }
@ -855,90 +798,27 @@ public class kelondroRecords {
} }
private void update2Cache(int forPriority) { private void update2Cache(int forPriority) {
if (cacheSize > 0) { if (cacheSpace()) updateNodeCache(forPriority);
cacheHeaders[CP_LOW].removeb(this.handle.index);
cacheHeaders[CP_MEDIUM].removeb(this.handle.index);
cacheHeaders[CP_HIGH].removeb(this.handle.index);
}
if (cacheSpace(forPriority)) updateNodeCache(forPriority);
} }
private boolean cacheSpace(int forPriority) { private boolean cacheSpace() {
// check for space in cache // check for space in cache
// should be only called within a synchronized(XcacheHeaders) environment // should be only called within a synchronized(XcacheHeaders) environment
// returns true if it is allowed to add another entry to the cache // returns true if it is allowed to add another entry to the cache
// returns false if the cache is considered to be full // returns false if the cache is considered to be full
if (forPriority == CP_NONE) return false;
if (cacheSize == 0) return false; // no caching if (cacheSize == 0) return false; // no caching
long cs = cacheHeaders[CP_LOW].size() + cacheHeaders[CP_MEDIUM].size() + cacheHeaders[CP_HIGH].size(); if (cacheHeaders.size() == 0) return true; // nothing there to flush
if (cs == 0) return true; // nothing there to flush if ((cacheHeaders.size() < cacheSize) && (availableMemory() >= memBlock)) return true; // no need to flush cache space
if ((cs < cacheSize) && (availableMemory() >= memBlock)) return true; // no need to flush cache space
// delete one entry. distinguish between different priority cases: // delete one entry. distinguish between different priority cases:
if (forPriority == CP_LOW) { if (cacheHeaders.size() != 0) {
// remove only from low-priority cache // just delete any of the entries
if (cacheHeaders[CP_LOW].size() != 0) { cacheHeaders.removeOne();
// just delete any of the low-priority entries
cacheHeaders[CP_LOW].removeOne();
cacheFlush++;
return true;
} else {
// we cannot delete any entry, therefore there is no space for another entry
return false;
}
} else if (forPriority == CP_MEDIUM) {
if (cacheHeaders[CP_LOW].size() != 0) {
// just delete any of the low-priority entries
cacheHeaders[CP_LOW].removeOne();
cacheFlush++; cacheFlush++;
return true; return true;
} else if (cacheHeaders[CP_MEDIUM].size() != 0) {
// just delete any of the medium-priority entries
cacheHeaders[CP_MEDIUM].removeOne();
cacheFlush++;
return true;
} else {
// we cannot delete any entry, therefore there is no space for another entry
return false;
}
} else { } else {
// request for a high-priority entry // we cannot delete any entry, therefore there is no space for another entry
if (cacheHeaders[CP_LOW].size() != 0) {
// just delete any of the low-priority entries
cacheHeaders[CP_LOW].removeOne();
cacheFlush++;
return true;
} else if (cacheHeaders[CP_MEDIUM].size() != 0) {
// just delete any of the medium-priority entries
cacheHeaders[CP_MEDIUM].removeOne();
cacheFlush++;
return true;
} else if (cacheScore == null) {
// no cache-control of high-priority cache
// the cache is considered as full
return false; return false;
} else try {
// delete one from the high-priority entries
// use the cache-control to find the right object
Handle delkey = (Handle) cacheScore.getMinObject();
cacheScore.deleteScore(delkey);
cacheHeaders[CP_HIGH].removeb(delkey.index);
cacheFlush++;
return true;
} catch (NoSuchElementException e) {
// this is a strange error and could be caused by internal java problems
// we simply clear the cache
String error = "cachScore error: " + e.getMessage() + "; cachesize=" + cacheSize + ", cache.size()=[" + cacheHeaders[0].size() + "," + cacheHeaders[1].size() + "," + cacheHeaders[2].size() + "], cacheScore.size()=" + cacheScore.size();
cacheScore = new kelondroMScoreCluster();
cacheHeaders[CP_LOW] = new kelondroIntBytesMap(headchunksize, cacheSize / 4);
cacheHeaders[CP_MEDIUM] = new kelondroIntBytesMap(headchunksize, 0);
cacheHeaders[CP_HIGH] = new kelondroIntBytesMap(headchunksize, cacheSize / 4);
cacheHeaders[0].setOrdering(kelondroNaturalOrder.naturalOrder, 0);
cacheHeaders[1].setOrdering(kelondroNaturalOrder.naturalOrder, 0);
cacheHeaders[2].setOrdering(kelondroNaturalOrder.naturalOrder, 0);
throw new kelondroException(filename, error);
}
} }
} }
@ -947,8 +827,7 @@ public class kelondroRecords {
if (this.headChunk == null) return; // nothing there to cache if (this.headChunk == null) return; // nothing there to cache
if (priority == CP_NONE) return; // it is not wanted that this shall be cached if (priority == CP_NONE) return; // it is not wanted that this shall be cached
if (cacheSize == 0) return; // we do not use the cache if (cacheSize == 0) return; // we do not use the cache
int cs = cacheHeaders[CP_LOW].size() + cacheHeaders[CP_MEDIUM].size() + cacheHeaders[CP_HIGH].size(); if (cacheHeaders.size() >= cacheSize) return; // no cache update if cache is full
if (cs >= cacheSize) return; // no cache update if cache is full
synchronized (cacheHeaders) { synchronized (cacheHeaders) {
// generate cache entry // generate cache entry
@ -958,13 +837,7 @@ public class kelondroRecords {
// store the cache entry // store the cache entry
boolean upd = false; boolean upd = false;
if (priority != CP_LOW) upd = upd || (cacheHeaders[CP_LOW].removeb(cacheHandle.index) != null); cacheHeaders.putb(cacheHandle.index, headChunk);
if (priority != CP_MEDIUM) upd = upd || (cacheHeaders[CP_MEDIUM].removeb(cacheHandle.index) != null);
if (priority != CP_HIGH) upd = upd || (cacheHeaders[CP_HIGH].removeb(cacheHandle.index) != null);
cacheHeaders[priority].putb(cacheHandle.index, headChunk);
if ((cacheScore != null) && (priority == CP_HIGH)) {
cacheScore.setScore(cacheHandle, (int) ((System.currentTimeMillis() - cacheStartup) / 1000));
}
if (upd) writeDouble++; else writeUnique++; if (upd) writeDouble++; else writeUnique++;
// delete the cache entry buffer // delete the cache entry buffer
@ -991,15 +864,15 @@ public class kelondroRecords {
System.out.println(); System.out.println();
} }
} else { } else {
System.out.println("### cache report: [" + cacheHeaders[0].size() + "," + cacheHeaders[0].size() + "," + cacheHeaders[0].size() + "] entries"); System.out.println("### cache report: " + cacheHeaders.size() + " entries");
for (int cp = 0; cp < 3; cp++) {
Iterator i = cacheHeaders[cp].elements(); Iterator i = cacheHeaders.elements();
byte[] entry; byte[] entry;
while (i.hasNext()) { while (i.hasNext()) {
entry = (byte[]) i.next(); entry = (byte[]) i.next();
// print from cache // print from cache
System.out.print("#C " + cp + " "); System.out.print("#C ");
printChunk((byte[]) entry); printChunk((byte[]) entry);
System.out.println(); System.out.println();
@ -1013,7 +886,6 @@ public class kelondroRecords {
*/ */
System.out.println(); System.out.println();
} }
}
} }
System.out.println("### end report"); System.out.println("### end report");
} }
@ -1374,11 +1246,7 @@ public class kelondroRecords {
public kelondroProfile[] profiles() { public kelondroProfile[] profiles() {
return new kelondroProfile[]{ return new kelondroProfile[]{
(cacheHeaders == null) ? new kelondroProfile() : (cacheHeaders == null) ? new kelondroProfile() :
kelondroProfile.consolidate(new kelondroProfile[]{ cacheHeaders.profile(),
cacheHeaders[0].profile(),
cacheHeaders[1].profile(),
cacheHeaders[2].profile()
}),
entryFile.profile() entryFile.profile()
}; };
} }

2
source/de/anomic/kelondro/kelondroStack.java
Unescape View File

@ -99,7 +99,7 @@ public final class kelondroStack extends kelondroRecords {
public static kelondroStack reset(kelondroStack stack) { public static kelondroStack reset(kelondroStack stack) {
// memorize settings to this file // memorize settings to this file
File f = new File(stack.filename); File f = new File(stack.filename);
long bz = stack.cacheNodeStatus()[0] * stack.cacheNodeChunkSize(true); long bz = stack.cacheNodeStatus()[0] * stack.cacheNodeChunkSize();
kelondroRow row = stack.row(); kelondroRow row = stack.row();
// close and delete the file // close and delete the file

2
source/de/anomic/plasma/plasmaCrawlLURL.java
Unescape View File

@ -576,7 +576,7 @@ public final class plasmaCrawlLURL extends indexURL {
kelondroBase64Order.enhancedCoder.encodeLong(wordCount, urlWordCountLength).getBytes(), kelondroBase64Order.enhancedCoder.encodeLong(wordCount, urlWordCountLength).getBytes(),
}; };
urlHashCache.put(urlHashCache.row().newEntry(entry)); urlHashCache.put(urlHashCache.row().newEntry(entry));
serverLog.logFine("PLASMA","STORED new LURL " + url.toString()); //serverLog.logFine("PLASMA","STORED new LURL " + url.toString());
this.stored = true; this.stored = true;
} catch (Exception e) { } catch (Exception e) {
serverLog.logSevere("PLASMA", "INTERNAL ERROR AT plasmaCrawlLURL:store:" + e.toString(), e); serverLog.logSevere("PLASMA", "INTERNAL ERROR AT plasmaCrawlLURL:store:" + e.toString(), e);

2
source/de/anomic/plasma/plasmaCrawlProfile.java
Unescape View File

@ -78,7 +78,7 @@ public class plasmaCrawlProfile {
domsCache = new HashMap(); domsCache = new HashMap();
} }
public int[] dbCacheNodeChunkSize() { public int dbCacheNodeChunkSize() {
return profileTable.cacheNodeChunkSize(); return profileTable.cacheNodeChunkSize();
} }

2
source/de/anomic/plasma/plasmaCrawlRobotsTxt.java
Unescape View File

@ -85,7 +85,7 @@ public class plasmaCrawlRobotsTxt {
} }
} }
public int[] dbCacheNodeChunkSize() { public int dbCacheNodeChunkSize() {
return robotsTable.cacheNodeChunkSize(); return robotsTable.cacheNodeChunkSize();
} }

2
source/de/anomic/plasma/plasmaHTCache.java
Unescape View File

@ -203,7 +203,7 @@ public final class plasmaHTCache {
return this.responseHeaderDB.size(); return this.responseHeaderDB.size();
} }
public int[] dbCacheChunkSize() { public int dbCacheChunkSize() {
return this.responseHeaderDB.cacheNodeChunkSize(); return this.responseHeaderDB.cacheNodeChunkSize();
} }

2
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -808,7 +808,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// work off unwritten files // work off unwritten files
if (entry.cacheArray == null) { if (entry.cacheArray == null) {
this.log.logFine("EXISTING FILE (" + entry.cacheFile.length() + " bytes) for " + entry.cacheFile); //this.log.logFine("EXISTING FILE (" + entry.cacheFile.length() + " bytes) for " + entry.cacheFile);
} else { } else {
String error = entry.shallStoreCacheForProxy(); String error = entry.shallStoreCacheForProxy();
if (error == null) { if (error == null) {

2
source/de/anomic/plasma/plasmaWordIndex.java
Unescape View File

@ -133,7 +133,7 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI {
return assortmentCluster.sizes(); return assortmentCluster.sizes();
} }
public int[] assortmentsCacheChunkSizeAvg() { public int assortmentsCacheChunkSizeAvg() {
return assortmentCluster.cacheChunkSizeAvg(); return assortmentCluster.cacheChunkSizeAvg();
} }

2
source/de/anomic/plasma/plasmaWordIndexAssortment.java
Unescape View File

@ -264,7 +264,7 @@ public final class plasmaWordIndexAssortment {
return assortments.size(); return assortments.size();
} }
public int[] cacheNodeChunkSize() { public int cacheNodeChunkSize() {
return assortments.cacheNodeChunkSize(); return assortments.cacheNodeChunkSize();
} }

15
source/de/anomic/plasma/plasmaWordIndexAssortmentCluster.java
Unescape View File

@ -290,19 +290,14 @@ public final class plasmaWordIndexAssortmentCluster extends indexAbstractRI impl
return sizes; return sizes;
} }
public int[] cacheChunkSizeAvg() { public int cacheChunkSizeAvg() {
int[] i = new int[]{0, 0, 0}; int i = 0;
int[] a = new int[3]; int a;
for (int j = 0; j < clusterCount; j++) { for (int j = 0; j < clusterCount; j++) {
a = assortments[j].cacheNodeChunkSize(); a = assortments[j].cacheNodeChunkSize();
i[kelondroRecords.CP_LOW] += a[kelondroRecords.CP_LOW]; i += a;
i[kelondroRecords.CP_MEDIUM] += a[kelondroRecords.CP_MEDIUM];
i[kelondroRecords.CP_HIGH] += a[kelondroRecords.CP_HIGH];
} }
a[kelondroRecords.CP_LOW] = i[kelondroRecords.CP_LOW] / clusterCount; return i / clusterCount;
a[kelondroRecords.CP_MEDIUM] = i[kelondroRecords.CP_MEDIUM] / clusterCount;
a[kelondroRecords.CP_HIGH] = i[kelondroRecords.CP_HIGH] / clusterCount;
return a;
} }
public int[] cacheNodeStatus() { public int[] cacheNodeStatus() {

2
source/de/anomic/yacy/yacyNewsDB.java
Unescape View File

@ -100,7 +100,7 @@ public class yacyNewsDB {
news = createDB(path, bufferkb); news = createDB(path, bufferkb);
} }
public int[] dbCacheNodeChunkSize() { public int dbCacheNodeChunkSize() {
return news.cacheNodeChunkSize(); return news.cacheNodeChunkSize();
} }

2
source/de/anomic/yacy/yacyNewsPool.java
Unescape View File

@ -106,7 +106,7 @@ public class yacyNewsPool {
return newsDB.size(); return newsDB.size();
} }
public int[] dbCacheNodeChunkSize() { public int dbCacheNodeChunkSize() {
return newsDB.dbCacheNodeChunkSize(); return newsDB.dbCacheNodeChunkSize();
} }

15
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -170,15 +170,12 @@ public final class yacySeedDB {
} catch (IOException e) {} } catch (IOException e) {}
} }
public int[] dbCacheNodeChunkSize() { public int dbCacheNodeChunkSize() {
int[] ac = seedActiveDB.cacheNodeChunkSize(); int ac = seedActiveDB.cacheNodeChunkSize();
int[] pa = seedPassiveDB.cacheNodeChunkSize(); int pa = seedPassiveDB.cacheNodeChunkSize();
int[] po = seedPotentialDB.cacheNodeChunkSize(); int po = seedPotentialDB.cacheNodeChunkSize();
int[] i = new int[3];
i[kelondroRecords.CP_LOW] = (ac[kelondroRecords.CP_LOW] + pa[kelondroRecords.CP_LOW] + po[kelondroRecords.CP_LOW]) / 3; return (ac+ pa + po) / 3;
i[kelondroRecords.CP_MEDIUM] = (ac[kelondroRecords.CP_MEDIUM] + pa[kelondroRecords.CP_MEDIUM] + po[kelondroRecords.CP_MEDIUM]) / 3;
i[kelondroRecords.CP_HIGH] = (ac[kelondroRecords.CP_HIGH] + pa[kelondroRecords.CP_HIGH] + po[kelondroRecords.CP_HIGH]) / 3;
return i;
} }
public int[] dbCacheNodeStatus() { public int[] dbCacheNodeStatus() {

Write Preview
Loading…
Cancel
Save