- corrections to PerformanceMemory display of object cache

- configuration of object cache size in kelondroTree initializer

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2075 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent fd7c17e624
commit 3e31820c3d

@ -80,22 +80,23 @@
<form action="PerformanceMemory_p.html" method="post" enctype="multipart/form-data"> <form action="PerformanceMemory_p.html" method="post" enctype="multipart/form-data">
<tr class="TableHeader" valign="bottom"> <tr class="TableHeader" valign="bottom">
<td class="small" rowspan="2">Database<br>&nbsp;</td> <td class="small" rowspan="2">Database<br>&nbsp;</td>
<td class="small" rowspan="2">Node<br>Chunk Size<br>high/med/low<br>(bytes)</td> <td class="small" rowspan="2">DB&nbsp;Size<br>&nbsp;</td>
<td class="small" colspan="5">#Slots</td> <td class="small" colspan="5">Node Cache</td>
<td class="small" rowspan="2">Object<br>Chunk Size<br>Max</td> <td class="small" colspan="4">Object Cache</td>
<td class="small" rowspan="2">Object<br>Chunk Size<br>Current</td>
<td class="small" rowspan="2">Hit/Miss<br>Ratio<br></td>
<td class="small" rowspan="2">Unique/Double<br>Ratio<br></td>
<td class="small" colspan="5">Memory Occupation (KBytes)</td> <td class="small" colspan="5">Memory Occupation (KBytes)</td>
<td class="small" rowspan="2">Description<br>&nbsp;</td> <td class="small" rowspan="2">Description<br>&nbsp;</td>
</tr> </tr>
<tr class="TableHeader" valign="bottom"> <tr class="TableHeader" valign="bottom">
<td class="small">Needed<br>(=&nbsp;DB&nbsp;Size)</td> <td class="small">Chunk Size<br>high/med/low<br>(bytes)</td>
<td class="small">Empty<br>&nbsp;</td> <td class="small">Empty<br>(avail.)<br>Slots</td>
<td class="small">Used<br>High Prio</td> <td class="small">Used<br>High Prio</td>
<td class="small">Used<br>Medium Prio</td> <td class="small">Used<br>Medium Prio</td>
<td class="small">Used<br>Low Prio</td> <td class="small">Used<br>Low Prio</td>
<td class="small">Chunk Size<br>Max</td>
<td class="small">Chunk Size<br>Current</td>
<td class="small">Hit/<br>Miss<br>Ratio<br></td>
<td class="small">Unique/<br>Double<br>Ratio<br></td>
<td class="small">Used Now</td> <td class="small">Used Now</td>
<td class="small">Assigned Max</td> <td class="small">Assigned Max</td>
<td class="small">Default Max</td> <td class="small">Default Max</td>
@ -105,8 +106,8 @@
<tr class="TableCellDark"> <tr class="TableCellDark">
<td class="small" align="left">RWI Assortment Cluster</td> <td class="small" align="left">RWI Assortment Cluster</td>
<td class="small" align="center">#[chunkRWI]#</td>
<td class="small" align="right">#[slreqRWI]#</td> <td class="small" align="right">#[slreqRWI]#</td>
<td class="small" align="center">#[chunkRWI]#</td>
<td class="small" align="right">#[slempRWI]#</td> <td class="small" align="right">#[slempRWI]#</td>
<td class="small" align="right">#[slhigRWI]#</td> <td class="small" align="right">#[slhigRWI]#</td>
<td class="small" align="right">#[slmedRWI]#</td> <td class="small" align="right">#[slmedRWI]#</td>
@ -127,8 +128,8 @@ cache will speed up crawls with a depth > 3.</td>
<tr class="TableCellDark"> <tr class="TableCellDark">
<td class="small" align="left">HTTP Response Header</td> <td class="small" align="left">HTTP Response Header</td>
<td class="small" align="center">#[chunkHTTP]#</td>
<td class="small" align="right">#[slreqHTTP]#</td> <td class="small" align="right">#[slreqHTTP]#</td>
<td class="small" align="center">#[chunkHTTP]#</td>
<td class="small" align="right">#[slempHTTP]#</td> <td class="small" align="right">#[slempHTTP]#</td>
<td class="small" align="right">#[slhigHTTP]#</td> <td class="small" align="right">#[slhigHTTP]#</td>
<td class="small" align="right">#[slmedHTTP]#</td> <td class="small" align="right">#[slmedHTTP]#</td>
@ -149,8 +150,8 @@ Increasing this cache will be most important for a fast proxy mode.</td>
<tr class="TableCellDark"> <tr class="TableCellDark">
<td class="small" align="left">'loaded' URLs</td> <td class="small" align="left">'loaded' URLs</td>
<td class="small" align="center">#[chunkLURL]#</td>
<td class="small" align="right">#[slreqLURL]#</td> <td class="small" align="right">#[slreqLURL]#</td>
<td class="small" align="center">#[chunkLURL]#</td>
<td class="small" align="right">#[slempLURL]#</td> <td class="small" align="right">#[slempLURL]#</td>
<td class="small" align="right">#[slhigLURL]#</td> <td class="small" align="right">#[slhigLURL]#</td>
<td class="small" align="right">#[slmedLURL]#</td> <td class="small" align="right">#[slmedLURL]#</td>
@ -170,8 +171,8 @@ This cache is very important for a fast search process. Increasing the cache siz
<tr class="TableCellDark"> <tr class="TableCellDark">
<td class="small" align="left">'noticed' URLs</td> <td class="small" align="left">'noticed' URLs</td>
<td class="small" align="center">#[chunkNURL]#</td>
<td class="small" align="right">#[slreqNURL]#</td> <td class="small" align="right">#[slreqNURL]#</td>
<td class="small" align="center">#[chunkNURL]#</td>
<td class="small" align="right">#[slempNURL]#</td> <td class="small" align="right">#[slempNURL]#</td>
<td class="small" align="right">#[slhigNURL]#</td> <td class="small" align="right">#[slhigNURL]#</td>
<td class="small" align="right">#[slmedNURL]#</td> <td class="small" align="right">#[slmedNURL]#</td>
@ -191,8 +192,8 @@ Increasing the cache size will result in faster double-check during URL recognit
<tr class="TableCellDark"> <tr class="TableCellDark">
<td class="small" align="left">'error' URLs</td> <td class="small" align="left">'error' URLs</td>
<td class="small" align="center">#[chunkEURL]#</td>
<td class="small" align="right">#[slreqEURL]#</td> <td class="small" align="right">#[slreqEURL]#</td>
<td class="small" align="center">#[chunkEURL]#</td>
<td class="small" align="right">#[slempEURL]#</td> <td class="small" align="right">#[slempEURL]#</td>
<td class="small" align="right">#[slhigEURL]#</td> <td class="small" align="right">#[slhigEURL]#</td>
<td class="small" align="right">#[slmedEURL]#</td> <td class="small" align="right">#[slmedEURL]#</td>
@ -212,8 +213,8 @@ Increasing the cache size will most probably speed up crawling slightly, but not
<tr class="TableCellDark"> <tr class="TableCellDark">
<td class="small" align="left">DHT Control</td> <td class="small" align="left">DHT Control</td>
<td class="small" align="center">#[chunkDHT]#</td>
<td class="small" align="right">#[slreqDHT]#</td> <td class="small" align="right">#[slreqDHT]#</td>
<td class="small" align="center">#[chunkDHT]#</td>
<td class="small" align="right">#[slempDHT]#</td> <td class="small" align="right">#[slempDHT]#</td>
<td class="small" align="right">#[slhigDHT]#</td> <td class="small" align="right">#[slhigDHT]#</td>
<td class="small" align="right">#[slmedDHT]#</td> <td class="small" align="right">#[slmedDHT]#</td>
@ -234,8 +235,8 @@ Increasing this cache may speed up many functions, but we need to test this to s
<tr class="TableCellDark"> <tr class="TableCellDark">
<td class="small" align="left">Messages</td> <td class="small" align="left">Messages</td>
<td class="small" align="center">#[chunkMessage]#</td>
<td class="small" align="right">#[slreqMessage]#</td> <td class="small" align="right">#[slreqMessage]#</td>
<td class="small" align="center">#[chunkMessage]#</td>
<td class="small" align="right">#[slempMessage]#</td> <td class="small" align="right">#[slempMessage]#</td>
<td class="small" align="right">#[slhigMessage]#</td> <td class="small" align="right">#[slhigMessage]#</td>
<td class="small" align="right">#[slmedMessage]#</td> <td class="small" align="right">#[slmedMessage]#</td>
@ -254,8 +255,8 @@ Increasing this cache may speed up many functions, but we need to test this to s
<tr class="TableCellDark"> <tr class="TableCellDark">
<td class="small" align="left">Wiki</td> <td class="small" align="left">Wiki</td>
<td class="small" align="center">#[chunkWiki]#</td>
<td class="small" align="right">#[slreqWiki]#</td> <td class="small" align="right">#[slreqWiki]#</td>
<td class="small" align="center">#[chunkWiki]#</td>
<td class="small" align="right">#[slempWiki]#</td> <td class="small" align="right">#[slempWiki]#</td>
<td class="small" align="right">#[slhigWiki]#</td> <td class="small" align="right">#[slhigWiki]#</td>
<td class="small" align="right">#[slmedWiki]#</td> <td class="small" align="right">#[slmedWiki]#</td>
@ -276,8 +277,8 @@ Increasing this cache may speed up access to the wiki pages.</td>
<tr class="TableCellDark"> <tr class="TableCellDark">
<td class="small" align="left">Blog</td> <td class="small" align="left">Blog</td>
<td class="small" align="center">#[chunkBlog]#</td>
<td class="small" align="right">#[slreqBlog]#</td> <td class="small" align="right">#[slreqBlog]#</td>
<td class="small" align="center">#[chunkBlog]#</td>
<td class="small" align="right">#[slempBlog]#</td> <td class="small" align="right">#[slempBlog]#</td>
<td class="small" align="right">#[slhigBlog]#</td> <td class="small" align="right">#[slhigBlog]#</td>
<td class="small" align="right">#[slmedBlog]#</td> <td class="small" align="right">#[slmedBlog]#</td>
@ -297,8 +298,8 @@ Increasing this cache may speed up access to the Blog.</td>
<tr class="TableCellDark"> <tr class="TableCellDark">
<td class="small" align="left">News</td> <td class="small" align="left">News</td>
<td class="small" align="center">#[chunkNews]#</td>
<td class="small" align="right">#[slreqNews]#</td> <td class="small" align="right">#[slreqNews]#</td>
<td class="small" align="center">#[chunkNews]#</td>
<td class="small" align="right">#[slempNews]#</td> <td class="small" align="right">#[slempNews]#</td>
<td class="small" align="right">#[slhigNews]#</td> <td class="small" align="right">#[slhigNews]#</td>
<td class="small" align="right">#[slmedNews]#</td> <td class="small" align="right">#[slmedNews]#</td>
@ -318,8 +319,8 @@ Increasing this cache may speed up the peer-ping.</td>
<tr class="TableCellDark"> <tr class="TableCellDark">
<td class="small" align="left">robots.txt DB</td> <td class="small" align="left">robots.txt DB</td>
<td class="small" align="center">#[chunkRobots]#</td>
<td class="small" align="right">#[slreqRobots]#</td> <td class="small" align="right">#[slreqRobots]#</td>
<td class="small" align="center">#[chunkRobots]#</td>
<td class="small" align="right">#[slempRobots]#</td> <td class="small" align="right">#[slempRobots]#</td>
<td class="small" align="right">#[slhigRobots]#</td> <td class="small" align="right">#[slhigRobots]#</td>
<td class="small" align="right">#[slmedRobots]#</td> <td class="small" align="right">#[slmedRobots]#</td>
@ -339,8 +340,8 @@ Increasing this cache may speed up validation if crawling of the URL is allowed.
<tr class="TableCellDark"> <tr class="TableCellDark">
<td class="small" align="left">Crawl Profiles</td> <td class="small" align="left">Crawl Profiles</td>
<td class="small" align="center">#[chunkProfiles]#</td>
<td class="small" align="right">#[slreqProfiles]#</td> <td class="small" align="right">#[slreqProfiles]#</td>
<td class="small" align="center">#[chunkProfiles]#</td>
<td class="small" align="right">#[slempProfiles]#</td> <td class="small" align="right">#[slempProfiles]#</td>
<td class="small" align="right">#[slhigProfiles]#</td> <td class="small" align="right">#[slhigProfiles]#</td>
<td class="small" align="right">#[slmedProfiles]#</td> <td class="small" align="right">#[slmedProfiles]#</td>
@ -359,7 +360,7 @@ Increasing this cache may speed up crawling, but not much space is needed, so th
</tr> </tr>
<tr class="TableCellSummary"> <tr class="TableCellSummary">
<td class="small" align="left" colspan="7">Totals</td> <td class="small" align="left" colspan="11">Totals</td>
<td class="small" align="right">#[usedTotal]# MB</td> <td class="small" align="right">#[usedTotal]# MB</td>
<td class="small" align="right">#[currTotal]# MB</td> <td class="small" align="right">#[currTotal]# MB</td>
<td class="small" align="right">#[dfltTotal]# MB</td> <td class="small" align="right">#[dfltTotal]# MB</td>
@ -369,7 +370,7 @@ Increasing this cache may speed up crawling, but not much space is needed, so th
</tr> </tr>
<tr class="TableCellLight"> <tr class="TableCellLight">
<td class="small" align="right" colspan="8">Re-Configuration:</td> <td class="small" align="right" colspan="12">Re-Configuration:</td>
<td class="small" align="center"><input type="submit" name="setCustom" value="Set"><br>these<br>custom<br>values<br>&nbsp;</td> <td class="small" align="center"><input type="submit" name="setCustom" value="Set"><br>these<br>custom<br>values<br>&nbsp;</td>
<td class="small" align="center"><input type="submit" name="setDefault" value="Set"><br>all<br>default<br>values<br>&nbsp;</td> <td class="small" align="center"><input type="submit" name="setDefault" value="Set"><br>all<br>default<br>values<br>&nbsp;</td>
<td class="small" align="center"><input type="submit" name="setGood" value="Set"><br>all<br>recom-<br>mended values</td> <td class="small" align="center"><input type="submit" name="setGood" value="Set"><br>all<br>recom-<br>mended values</td>

@ -167,9 +167,9 @@ public class dbtest {
if (dbe.equals("kelondroold")) { if (dbe.equals("kelondroold")) {
File tablefile = new File(tablename + ".kelondro.db"); File tablefile = new File(tablename + ".kelondro.db");
if (tablefile.exists()) { if (tablefile.exists()) {
table = new kelondroTree(tablefile, buffer); table = new kelondroTree(tablefile, buffer, kelondroTree.defaultObjectCachePercent);
} else { } else {
table = new kelondroTree(tablefile, buffer, new int[]{keylength, valuelength, valuelength}, true); table = new kelondroTree(tablefile, buffer, kelondroTree.defaultObjectCachePercent, new int[]{keylength, valuelength, valuelength}, true);
} }
} }

@ -75,7 +75,7 @@ public class kelondroDyn extends kelondroTree {
public kelondroDyn(File file, long buffersize /*bytes*/, int key, int nodesize, char fillChar, kelondroOrder objectOrder, boolean exitOnFail) { public kelondroDyn(File file, long buffersize /*bytes*/, int key, int nodesize, char fillChar, kelondroOrder objectOrder, boolean exitOnFail) {
// creates a new dynamic tree // creates a new dynamic tree
super(file, buffersize, new int[] {key + counterlen, nodesize}, objectOrder, 1, 8, exitOnFail); super(file, buffersize, kelondroTree.defaultObjectCachePercent, new int[] {key + counterlen, nodesize}, objectOrder, 1, 8, exitOnFail);
this.keylen = columnSize(0) - counterlen; this.keylen = columnSize(0) - counterlen;
this.reclen = columnSize(1); this.reclen = columnSize(1);
this.fillChar = fillChar; this.fillChar = fillChar;
@ -88,7 +88,7 @@ public class kelondroDyn extends kelondroTree {
public kelondroDyn(File file, long buffersize, char fillChar) throws IOException { public kelondroDyn(File file, long buffersize, char fillChar) throws IOException {
// this opens a file with an existing dynamic tree // this opens a file with an existing dynamic tree
super(file, buffersize); super(file, buffersize, kelondroTree.defaultObjectCachePercent);
this.keylen = columnSize(0) - counterlen; this.keylen = columnSize(0) - counterlen;
this.reclen = columnSize(1); this.reclen = columnSize(1);
this.fillChar = fillChar; this.fillChar = fillChar;

@ -135,7 +135,7 @@ public class kelondroDynTree {
kelondroRA ra = table.getRA(key); // works always, even with no-existing entry kelondroRA ra = table.getRA(key); // works always, even with no-existing entry
treeRAHandles.put(key, ra); treeRAHandles.put(key, ra);
try { try {
return new kelondroTree(ra, buffersize, columns, false); return new kelondroTree(ra, buffersize, kelondroTree.defaultObjectCachePercent, columns, false);
} catch (RuntimeException e) { } catch (RuntimeException e) {
throw new IOException(e.getMessage()); throw new IOException(e.getMessage());
} }
@ -145,7 +145,7 @@ public class kelondroDynTree {
if (table.existsDyn(key)) { if (table.existsDyn(key)) {
kelondroRA ra = table.getRA(key); kelondroRA ra = table.getRA(key);
treeRAHandles.put(key, ra); treeRAHandles.put(key, ra);
return new kelondroTree(ra, buffersize); return new kelondroTree(ra, buffersize, kelondroTree.defaultObjectCachePercent);
} else { } else {
return null; return null;
} }

@ -63,18 +63,18 @@ public class kelondroFScoreCluster {
public kelondroFScoreCluster(File refcountDBfile, File countrefDBfile, kelondroOrder objectOrder, boolean exitOnFail) { public kelondroFScoreCluster(File refcountDBfile, File countrefDBfile, kelondroOrder objectOrder, boolean exitOnFail) {
if ((refcountDBfile.exists()) && (countrefDBfile.exists())) try { if ((refcountDBfile.exists()) && (countrefDBfile.exists())) try {
refcountDB = new kelondroTree(refcountDBfile, 0x100000L); refcountDB = new kelondroTree(refcountDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent);
refcountDB.setText(0, kelondroBase64Order.enhancedCoder.encodeLong(0, countlength).getBytes()); // counter of all occurrences refcountDB.setText(0, kelondroBase64Order.enhancedCoder.encodeLong(0, countlength).getBytes()); // counter of all occurrences
countrefDB = new kelondroTree(countrefDBfile, 0x100000L); countrefDB = new kelondroTree(countrefDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent);
countrefDB.setText(0, kelondroBase64Order.enhancedCoder.encodeLong(0, countlength).getBytes()); countrefDB.setText(0, kelondroBase64Order.enhancedCoder.encodeLong(0, countlength).getBytes());
} catch (IOException e) { } catch (IOException e) {
refcountDBfile.delete(); refcountDBfile.delete();
countrefDBfile.delete(); countrefDBfile.delete();
refcountDB = new kelondroTree(refcountDBfile, 0x100000L, new int[] {wordlength, countlength}, objectOrder, 1, countlength, exitOnFail); refcountDB = new kelondroTree(refcountDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new int[] {wordlength, countlength}, objectOrder, 1, countlength, exitOnFail);
countrefDB = new kelondroTree(countrefDBfile, 0x100000L, new int[] {countlength + wordlength, 4}, objectOrder, 1, countlength, exitOnFail); countrefDB = new kelondroTree(countrefDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new int[] {countlength + wordlength, 4}, objectOrder, 1, countlength, exitOnFail);
} else if ((!(refcountDBfile.exists())) && (!(countrefDBfile.exists()))) { } else if ((!(refcountDBfile.exists())) && (!(countrefDBfile.exists()))) {
refcountDB = new kelondroTree(refcountDBfile, 0x100000L, new int[] {wordlength, countlength}, objectOrder, 1, countlength, exitOnFail); refcountDB = new kelondroTree(refcountDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new int[] {wordlength, countlength}, objectOrder, 1, countlength, exitOnFail);
countrefDB = new kelondroTree(countrefDBfile, 0x100000L, new int[] {countlength + wordlength, 4}, objectOrder, 1, countlength, exitOnFail); countrefDB = new kelondroTree(countrefDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new int[] {countlength + wordlength, 4}, objectOrder, 1, countlength, exitOnFail);
} else { } else {
if (exitOnFail) { if (exitOnFail) {
System.exit(-1); System.exit(-1);

@ -165,7 +165,7 @@ public class kelondroObjectCache {
} }
public void put(String key, Object value) { public void put(String key, Object value) {
if (key == null) return; if ((key == null) || (value == null)) return;
Object prev = null; Object prev = null;
synchronized(cache) { synchronized(cache) {
prev = cache.put(key, value); prev = cache.put(key, value);

@ -84,15 +84,15 @@ public class kelondroSplittedTree implements kelondroIndex {
f = dbFile(pathToFiles, filenameStub, forkfactor, columns.length, i); f = dbFile(pathToFiles, filenameStub, forkfactor, columns.length, i);
if (f.exists()) { if (f.exists()) {
try { try {
ktfs[i] = new kelondroTree(f, buffersize/forkfactor); ktfs[i] = new kelondroTree(f, buffersize/forkfactor, kelondroTree.defaultObjectCachePercent);
this.order = ktfs[i].order(); this.order = ktfs[i].order();
} catch (IOException e) { } catch (IOException e) {
ktfs[i] = new kelondroTree(f, buffersize/forkfactor, ktfs[i] = new kelondroTree(f, buffersize/forkfactor, kelondroTree.defaultObjectCachePercent,
columns, objectOrder, txtProps, txtPropsWidth, exitOnFail); columns, objectOrder, txtProps, txtPropsWidth, exitOnFail);
this.order = objectOrder; this.order = objectOrder;
} }
} else { } else {
ktfs[i] = new kelondroTree(f, buffersize/forkfactor, ktfs[i] = new kelondroTree(f, buffersize/forkfactor, kelondroTree.defaultObjectCachePercent,
columns, objectOrder, txtProps, txtPropsWidth, exitOnFail); columns, objectOrder, txtProps, txtPropsWidth, exitOnFail);
this.order = objectOrder; this.order = objectOrder;
} }
@ -104,7 +104,7 @@ public class kelondroSplittedTree implements kelondroIndex {
long buffersize, int forkfactor, int columns) throws IOException { long buffersize, int forkfactor, int columns) throws IOException {
ktfs = new kelondroTree[forkfactor]; ktfs = new kelondroTree[forkfactor];
for (int i = 0; i < forkfactor; i++) { for (int i = 0; i < forkfactor; i++) {
ktfs[i] = new kelondroTree(dbFile(pathToFiles, filenameStub, forkfactor, columns, i), buffersize/forkfactor); ktfs[i] = new kelondroTree(dbFile(pathToFiles, filenameStub, forkfactor, columns, i), buffersize/forkfactor, kelondroTree.defaultObjectCachePercent);
} }
ff = forkfactor; ff = forkfactor;
this.order = objectOrder; this.order = objectOrder;

@ -93,13 +93,13 @@ public class kelondroTables {
File tablefile = new File(tablesPath, "table." + tablename + ".tdb"); File tablefile = new File(tablesPath, "table." + tablename + ".tdb");
kelondroTree Tree; kelondroTree Tree;
if (tablefile.exists()) try { if (tablefile.exists()) try {
Tree = new kelondroTree(tablefile, buffersize); Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) { } catch (IOException e) {
tablefile.getParentFile().mkdirs(); tablefile.getParentFile().mkdirs();
Tree = new kelondroTree(tablefile, buffersize, columns, exitOnFail); Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent, columns, exitOnFail);
} else { } else {
tablefile.getParentFile().mkdirs(); tablefile.getParentFile().mkdirs();
Tree = new kelondroTree(tablefile, buffersize, columns, exitOnFail); Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent, columns, exitOnFail);
} }
tTables.put(tablename, Tree); tTables.put(tablename, Tree);
} }

@ -81,8 +81,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
private static int root = 0; // pointer for FHandles-array: pointer to root node private static int root = 0; // pointer for FHandles-array: pointer to root node
// calibration of cache // calibration of cache
private static int nodeCachePercent = 90; public static int defaultObjectCachePercent = 10;
private static int objectCachePercent = 10;
// class variables // class variables
private Search writeSearchObj = new Search(); private Search writeSearchObj = new Search();
@ -92,23 +91,20 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
private long lastIteratorCount = readAheadChunkSize; private long lastIteratorCount = readAheadChunkSize;
private kelondroObjectCache objectCache; private kelondroObjectCache objectCache;
public kelondroTree(File file, long buffersize, int key, int value, boolean exitOnFail) {
this(file, buffersize, new int[] { key, value }, new kelondroNaturalOrder(true), 1, 8, exitOnFail);
}
public kelondroTree(kelondroRA ra, long buffersize, int key, int value, boolean exitOnFail) { public kelondroTree(File file, long buffersize, int objectCachePercent, int key, int value, boolean exitOnFail) {
this(ra, buffersize, new int[] { key, value }, new kelondroNaturalOrder(true), 1, 8, exitOnFail); this(file, buffersize, objectCachePercent, new int[] { key, value }, new kelondroNaturalOrder(true), 1, 8, exitOnFail);
} }
public kelondroTree(File file, long buffersize, int[] columns, boolean exitOnFail) { public kelondroTree(File file, long buffersize, int objectCachePercent, int[] columns, boolean exitOnFail) {
// this creates a new tree file // this creates a new tree file
this(file, buffersize, columns, new kelondroNaturalOrder(true), columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail); this(file, buffersize, objectCachePercent, columns, new kelondroNaturalOrder(true), columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
} }
public kelondroTree(File file, long buffersize, int[] columns, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) { public kelondroTree(File file, long buffersize, int objectCachePercent, int[] columns, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) {
// this creates a new tree file // this creates a new tree file
super(file, super(file,
nodeCachePercent * buffersize / (nodeCachePercent + objectCachePercent), (100 - objectCachePercent) * buffersize / 100,
thisOHBytes, thisOHHandles, columns, thisOHBytes, thisOHHandles, columns,
thisFHandles, txtProps, txtPropsWidth, exitOnFail); thisFHandles, txtProps, txtPropsWidth, exitOnFail);
try { try {
@ -121,20 +117,18 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
this.objectOrder = objectOrder; this.objectOrder = objectOrder;
writeOrderType(); writeOrderType();
super.setLogger(log); super.setLogger(log);
long objectbuffersize = objectCachePercent * buffersize / (nodeCachePercent + objectCachePercent); initObjectCache(buffersize, objectCachePercent);
long nodecachesize = objectbuffersize / cacheObjectChunkSize();
this.objectCache = new kelondroObjectCache(this.filename, (int) nodecachesize, nodecachesize * 300 , 4*1024*1024);
} }
public kelondroTree(kelondroRA ra, long buffersize, int[] columns, boolean exitOnFail) { public kelondroTree(kelondroRA ra, long buffersize, int objectCachePercent, int[] columns, boolean exitOnFail) {
// this creates a new tree within a kelondroRA // this creates a new tree within a kelondroRA
this(ra, buffersize, columns, new kelondroNaturalOrder(true), columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail); this(ra, buffersize, objectCachePercent, columns, new kelondroNaturalOrder(true), columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
} }
public kelondroTree(kelondroRA ra, long buffersize, int[] columns, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) { public kelondroTree(kelondroRA ra, long buffersize, int objectCachePercent, int[] columns, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) {
// this creates a new tree within a kelondroRA // this creates a new tree within a kelondroRA
super(ra, super(ra,
nodeCachePercent * buffersize / (nodeCachePercent + objectCachePercent), (100 - objectCachePercent) * buffersize / 100,
thisOHBytes, thisOHHandles, columns, thisOHBytes, thisOHHandles, columns,
thisFHandles, txtProps, txtPropsWidth, exitOnFail); thisFHandles, txtProps, txtPropsWidth, exitOnFail);
try { try {
@ -147,29 +141,33 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
this.objectOrder = objectOrder; this.objectOrder = objectOrder;
writeOrderType(); writeOrderType();
super.setLogger(log); super.setLogger(log);
long objectbuffersize = objectCachePercent * buffersize / (nodeCachePercent + objectCachePercent); initObjectCache(buffersize, objectCachePercent);
long nodecachesize = objectbuffersize / cacheObjectChunkSize();
this.objectCache = new kelondroObjectCache(this.filename, (int) nodecachesize, nodecachesize * 300 , 4*1024*1024);
} }
public kelondroTree(File file, long buffersize) throws IOException { public kelondroTree(File file, long buffersize, int objectCachePercent) throws IOException {
// this opens a file with an existing tree file // this opens a file with an existing tree file
super(file, nodeCachePercent * buffersize / (nodeCachePercent + objectCachePercent)); super(file, (100 - objectCachePercent) * buffersize / 100);
readOrderType(); readOrderType();
super.setLogger(log); super.setLogger(log);
long objectbuffersize = objectCachePercent * buffersize / (nodeCachePercent + objectCachePercent); initObjectCache(buffersize, objectCachePercent);
long nodecachesize = objectbuffersize / cacheObjectChunkSize();
this.objectCache = new kelondroObjectCache(this.filename, (int) nodecachesize, nodecachesize * 300 , 4*1024*1024);
} }
public kelondroTree(kelondroRA ra, long buffersize) throws IOException { public kelondroTree(kelondroRA ra, long buffersize, int objectCachePercent) throws IOException {
// this opens a file with an existing tree in a kelondroRA // this opens a file with an existing tree in a kelondroRA
super(ra, nodeCachePercent * buffersize / (nodeCachePercent + objectCachePercent)); super(ra, (100 - objectCachePercent) * buffersize / 100);
readOrderType(); readOrderType();
super.setLogger(log); super.setLogger(log);
long objectbuffersize = objectCachePercent * buffersize / (nodeCachePercent + objectCachePercent); initObjectCache(buffersize, objectCachePercent);
}
private void initObjectCache(long buffersize, int objectCachePercent) {
if (objectCachePercent > 0) {
long objectbuffersize = objectCachePercent * buffersize / 100;
long nodecachesize = objectbuffersize / cacheObjectChunkSize(); long nodecachesize = objectbuffersize / cacheObjectChunkSize();
this.objectCache = new kelondroObjectCache(this.filename, (int) nodecachesize, nodecachesize * 300 , 4*1024*1024); this.objectCache = new kelondroObjectCache(this.filename, (int) nodecachesize, nodecachesize * 300 , 4*1024*1024);
} else {
this.objectCache = null;
}
} }
public final int cacheObjectChunkSize() { public final int cacheObjectChunkSize() {
@ -177,7 +175,8 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
} }
public String[] cacheObjectStatus() { public String[] cacheObjectStatus() {
return this.objectCache.status(); if (this.objectCache == null) return null;
else return this.objectCache.status();
} }
private void writeOrderType() { private void writeOrderType() {
@ -224,22 +223,20 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
// Returns the value to which this map maps the specified key. // Returns the value to which this map maps the specified key.
public byte[][] get(byte[] key) throws IOException { public byte[][] get(byte[] key) throws IOException {
// System.out.println("kelondroTree.get " + new String(key) + " in " + filename); // System.out.println("kelondroTree.get " + new String(key) + " in " + filename);
byte[][] result = (byte[][]) objectCache.get(key); byte[][] result = (objectCache == null) ? null : (byte[][]) objectCache.get(key);
if (result != null) { if (result != null) {
//System.out.println("cache hit in objectCache, db:" + super.filename); //System.out.println("cache hit in objectCache, db:" + super.filename);
return result; return result;
} }
// writeLock.stay(2000, 1000);
synchronized (writeSearchObj) { synchronized (writeSearchObj) {
writeSearchObj.process(key); writeSearchObj.process(key);
if (writeSearchObj.found()) { if (writeSearchObj.found()) {
result = writeSearchObj.getMatcher().getValues(); result = writeSearchObj.getMatcher().getValues();
objectCache.put(key, result); if (objectCache != null) objectCache.put(key, result);
} else { } else {
result = null; result = null;
} }
} }
// writeLock.release();
return result; return result;
} }
@ -414,7 +411,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
if (newrow.length != columns()) throw new IllegalArgumentException("put: wrong row length " + newrow.length + "; must be " + columns()); if (newrow.length != columns()) throw new IllegalArgumentException("put: wrong row length " + newrow.length + "; must be " + columns());
// first try to find the key element in the database // first try to find the key element in the database
synchronized(writeSearchObj) { synchronized(writeSearchObj) {
objectCache.put(newrow[0], newrow); if (objectCache != null) objectCache.put(newrow[0], newrow);
writeSearchObj.process(newrow[0]); writeSearchObj.process(newrow[0]);
if (writeSearchObj.found()) { if (writeSearchObj.found()) {
// a node with this key exist. simply overwrite the content and return old content // a node with this key exist. simply overwrite the content and return old content
@ -679,7 +676,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
// Removes the mapping for this key from this map if present (optional operation). // Removes the mapping for this key from this map if present (optional operation).
public byte[][] remove(byte[] key) throws IOException { public byte[][] remove(byte[] key) throws IOException {
synchronized(writeSearchObj) { synchronized(writeSearchObj) {
objectCache.remove(key); if (objectCache != null) objectCache.remove(key);
writeSearchObj.process(key); writeSearchObj.process(key);
if (writeSearchObj.found()) { if (writeSearchObj.found()) {
Node result = writeSearchObj.getMatcher(); Node result = writeSearchObj.getMatcher();
@ -1337,7 +1334,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
// test script // test script
File testFile = new File("test.db"); File testFile = new File("test.db");
while (testFile.exists()) testFile.delete(); while (testFile.exists()) testFile.delete();
kelondroTree fm = new kelondroTree(testFile, 0x100000, 4, 4, true); kelondroTree fm = new kelondroTree(testFile, 0x100000, 10, 4, 4, true);
byte[] dummy = "".getBytes(); byte[] dummy = "".getBytes();
fm.put("abc0".getBytes(), dummy); fm.put("bcd0".getBytes(), dummy); fm.put("abc0".getBytes(), dummy); fm.put("bcd0".getBytes(), dummy);
fm.put("def0".getBytes(), dummy); fm.put("bab0".getBytes(), dummy); fm.put("def0".getBytes(), dummy); fm.put("bab0".getBytes(), dummy);
@ -1359,7 +1356,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
ret = null; ret = null;
} }
} else if (args.length == 2) { } else if (args.length == 2) {
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000); kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10);
if (args[0].equals("-v")) { if (args[0].equals("-v")) {
fm.print(); fm.print();
ret = null; ret = null;
@ -1367,11 +1364,11 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
fm.close(); fm.close();
} else if (args.length == 3) { } else if (args.length == 3) {
if (args[0].equals("-d")) { if (args[0].equals("-d")) {
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000); kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10);
fm.remove(args[2].getBytes()); fm.remove(args[2].getBytes());
fm.close(); fm.close();
} else if (args[0].equals("-i")) { } else if (args[0].equals("-i")) {
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000); kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10);
int i = fm.imp(new File(args[1]),";"); int i = fm.imp(new File(args[1]),";");
fm.close(); fm.close();
ret = (i + " records imported").getBytes(); ret = (i + " records imported").getBytes();
@ -1394,12 +1391,12 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
if (f != null) try {f.close();}catch(Exception e){} if (f != null) try {f.close();}catch(Exception e){}
} }
} else if (args[0].equals("-g")) { } else if (args[0].equals("-g")) {
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000); kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10);
byte[][] ret2 = fm.get(args[2].getBytes()); byte[][] ret2 = fm.get(args[2].getBytes());
ret = ((ret2 == null) ? null : ret2[1]); ret = ((ret2 == null) ? null : ret2[1]);
fm.close(); fm.close();
} else if (args[0].equals("-n")) { } else if (args[0].equals("-n")) {
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000); kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10);
//byte[][] keys = fm.getSequentialKeys(args[2].getBytes(), 500, true); //byte[][] keys = fm.getSequentialKeys(args[2].getBytes(), 500, true);
Iterator rowIt = fm.rows(true, false, (args[2].length() == 0) ? null : args[2].getBytes()); Iterator rowIt = fm.rows(true, false, (args[2].length() == 0) ? null : args[2].getBytes());
Vector v = new Vector(); Vector v = new Vector();
@ -1415,10 +1412,10 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
int[] lens = new int[2]; int[] lens = new int[2];
lens[0] = Integer.parseInt(args[1]); lens[0] = Integer.parseInt(args[1]);
lens[1] = Integer.parseInt(args[2]); lens[1] = Integer.parseInt(args[2]);
kelondroTree fm = new kelondroTree(f, 0x100000, lens, true); kelondroTree fm = new kelondroTree(f, 0x100000, 10, lens, true);
fm.close(); fm.close();
} else if (args[0].equals("-u")) { } else if (args[0].equals("-u")) {
kelondroTree fm = new kelondroTree(new File(args[3]), 0x100000); kelondroTree fm = new kelondroTree(new File(args[3]), 0x100000, 10);
ret = fm.put(args[1].getBytes(), args[2].getBytes()); ret = fm.put(args[1].getBytes(), args[2].getBytes());
fm.close(); fm.close();
} }
@ -1481,7 +1478,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
int steps = 0; int steps = 0;
while (true) { while (true) {
if (testFile.exists()) testFile.delete(); if (testFile.exists()) testFile.delete();
tt = new kelondroTree(testFile, 200, 4 ,4, true); tt = new kelondroTree(testFile, 200, 10, 4 ,4, true);
steps = 10 + ((int) System.currentTimeMillis() % 7) * (((int) System.currentTimeMillis() + 17) % 11); steps = 10 + ((int) System.currentTimeMillis() % 7) * (((int) System.currentTimeMillis() + 17) % 11);
t = s; t = s;
d = ""; d = "";
@ -1547,7 +1544,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
File f = new File("test.db"); File f = new File("test.db");
if (f.exists()) f.delete(); if (f.exists()) f.delete();
try { try {
kelondroTree tt = new kelondroTree(f, 0, 4, 4, true); kelondroTree tt = new kelondroTree(f, 0, 10, 4, 4, true);
byte[] b; byte[] b;
b = testWord('B'); tt.put(b, b); //tt.print(); b = testWord('B'); tt.put(b, b); //tt.print();
b = testWord('C'); tt.put(b, b); //tt.print(); b = testWord('C'); tt.put(b, b); //tt.print();
@ -1581,7 +1578,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
File f = new File("test.db"); File f = new File("test.db");
if (f.exists()) f.delete(); if (f.exists()) f.delete();
try { try {
kelondroTree tt = new kelondroTree(f, 0, 4, 4, true); kelondroTree tt = new kelondroTree(f, 0, 10, 4, 4, true);
byte[] b; byte[] b;
for (int i = 0; i < 100; i++) { for (int i = 0; i < 100; i++) {
b = ("T" + i).getBytes(); tt.put(b, b); b = ("T" + i).getBytes(); tt.put(b, b);
@ -1610,7 +1607,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
public static kelondroTree testTree(File f, String testentities) throws IOException { public static kelondroTree testTree(File f, String testentities) throws IOException {
if (f.exists()) f.delete(); if (f.exists()) f.delete();
kelondroTree tt = new kelondroTree(f, 0, 4, 4, true); kelondroTree tt = new kelondroTree(f, 0, 10, 4, 4, true);
byte[] b; byte[] b;
for (int i = 0; i < testentities.length(); i++) { for (int i = 0; i < testentities.length(); i++) {
b = testWord(testentities.charAt(i)); b = testWord(testentities.charAt(i));

@ -77,14 +77,14 @@ public class plasmaCrawlEURL extends plasmaURL {
}; };
if (cachePath.exists()) try { if (cachePath.exists()) try {
// open existing cache // open existing cache
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400); urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) { } catch (IOException e) {
cachePath.delete(); cachePath.delete();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, ce, true); urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true);
} else { } else {
// create new cache // create new cache
cachePath.getParentFile().mkdirs(); cachePath.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, ce, true); urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true);
} }
} }

@ -111,15 +111,15 @@ public final class plasmaCrawlLURL extends plasmaURL {
if (cachePath.exists()) { if (cachePath.exists()) {
// open existing cache // open existing cache
try { try {
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400); urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) { } catch (IOException e) {
cachePath.getParentFile().mkdirs(); cachePath.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, ce, true); urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true);
} }
} else { } else {
// create new cache // create new cache
cachePath.getParentFile().mkdirs(); cachePath.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, ce, true); urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true);
} }
// init result stacks // init result stacks

@ -163,14 +163,14 @@ public class plasmaCrawlNURL extends plasmaURL {
File cacheFile = new File(cacheStacksPath, "urlNotice1.db"); File cacheFile = new File(cacheStacksPath, "urlNotice1.db");
if (cacheFile.exists()) try { if (cacheFile.exists()) try {
// open existing cache // open existing cache
urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400); urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) { } catch (IOException e) {
cacheFile.delete(); cacheFile.delete();
urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, ce, true); urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true);
} else { } else {
// create new cache // create new cache
cacheFile.getParentFile().mkdirs(); cacheFile.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, ce, true); urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true);
} }
} }

@ -572,10 +572,10 @@ public final class plasmaCrawlStacker {
if (cacheFile.exists()) { if (cacheFile.exists()) {
// open existing cache // open existing cache
try { try {
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400); this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) { } catch (IOException e) {
cacheFile.delete(); cacheFile.delete();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, plasmaCrawlNURL.ce, true); this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.ce, true);
} }
try { try {
// loop through the list and fill the messageList with url hashs // loop through the list and fill the messageList with url hashs
@ -597,7 +597,7 @@ public final class plasmaCrawlStacker {
// deleting old db and creating a new db // deleting old db and creating a new db
try {this.urlEntryCache.close();}catch(Exception ex){} try {this.urlEntryCache.close();}catch(Exception ex){}
cacheFile.delete(); cacheFile.delete();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, plasmaCrawlNURL.ce, true); this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.ce, true);
} catch (IOException e) { } catch (IOException e) {
/* if we have an error, we start with a fresh database */ /* if we have an error, we start with a fresh database */
plasmaCrawlStacker.this.log.logSevere("Unable to initialize crawl stacker queue, IOException:" + e.getMessage() + ". Reseting DB.\n",e); plasmaCrawlStacker.this.log.logSevere("Unable to initialize crawl stacker queue, IOException:" + e.getMessage() + ". Reseting DB.\n",e);
@ -605,12 +605,12 @@ public final class plasmaCrawlStacker {
// deleting old db and creating a new db // deleting old db and creating a new db
try {this.urlEntryCache.close();}catch(Exception ex){} try {this.urlEntryCache.close();}catch(Exception ex){}
cacheFile.delete(); cacheFile.delete();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, plasmaCrawlNURL.ce, true); this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.ce, true);
} }
} else { } else {
// create new cache // create new cache
cacheFile.getParentFile().mkdirs(); cacheFile.getParentFile().mkdirs();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, plasmaCrawlNURL.ce, true); this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.ce, true);
} }
} }

@ -109,7 +109,7 @@ public final class plasmaWordIndexAssortment {
if (assortmentFile.exists()) { if (assortmentFile.exists()) {
// open existing assortment tree file // open existing assortment tree file
try { try {
assortments = new kelondroTree(assortmentFile, bufferSize); assortments = new kelondroTree(assortmentFile, bufferSize, kelondroTree.defaultObjectCachePercent);
if (log != null) log.logConfig("Opened Assortment Database, " + assortments.size() + " entries, width " + assortmentLength + ", " + bufferkb + "kb buffer"); if (log != null) log.logConfig("Opened Assortment Database, " + assortments.size() + " entries, width " + assortmentLength + ", " + bufferkb + "kb buffer");
return; return;
} catch (IOException e){ } catch (IOException e){
@ -120,7 +120,7 @@ public final class plasmaWordIndexAssortment {
assortmentFile.delete(); // make space for new one assortmentFile.delete(); // make space for new one
} }
// create new assortment tree file // create new assortment tree file
assortments = new kelondroTree(assortmentFile, bufferSize, bufferStructure(assortmentLength), true); assortments = new kelondroTree(assortmentFile, bufferSize, kelondroTree.defaultObjectCachePercent, bufferStructure(assortmentLength), true);
if (log != null) log.logConfig("Created new Assortment Database, width " + assortmentLength + ", " + bufferkb + "kb buffer"); if (log != null) log.logConfig("Created new Assortment Database, width " + assortmentLength + ", " + bufferkb + "kb buffer");
} }
@ -240,7 +240,7 @@ public final class plasmaWordIndexAssortment {
if (!(assortmentFile.delete())) throw new RuntimeException("cannot delete assortment database"); if (!(assortmentFile.delete())) throw new RuntimeException("cannot delete assortment database");
} }
if (assortmentFile.exists()) assortmentFile.delete(); if (assortmentFile.exists()) assortmentFile.delete();
assortments = new kelondroTree(assortmentFile, bufferSize, bufferStructure(assortmentLength), true); assortments = new kelondroTree(assortmentFile, bufferSize, kelondroTree.defaultObjectCachePercent, bufferStructure(assortmentLength), true);
} }
public Iterator hashes(String startWordHash, boolean up, boolean rot) throws IOException { public Iterator hashes(String startWordHash, boolean up, boolean rot) throws IOException {

@ -87,13 +87,13 @@ public final class plasmaWordIndexEntity {
if (cacheSize > 1048576) cacheSize = 1048576; if (cacheSize > 1048576) cacheSize = 1048576;
if (theLocation.exists()) try { if (theLocation.exists()) try {
// open existing index file // open existing index file
kt = new kelondroTree(theLocation, cacheSize); kt = new kelondroTree(theLocation, cacheSize, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) { } catch (IOException e) {
theLocation.delete(); theLocation.delete();
kt = new kelondroTree(theLocation, cacheSize, plasmaURL.urlHashLength, plasmaWordIndexEntry.attrSpace, false); kt = new kelondroTree(theLocation, cacheSize, kelondroTree.defaultObjectCachePercent, plasmaURL.urlHashLength, plasmaWordIndexEntry.attrSpace, false);
} else { } else {
// create new index file // create new index file
kt = new kelondroTree(theLocation, cacheSize, plasmaURL.urlHashLength, plasmaWordIndexEntry.attrSpace, false); kt = new kelondroTree(theLocation, cacheSize, kelondroTree.defaultObjectCachePercent, plasmaURL.urlHashLength, plasmaWordIndexEntry.attrSpace, false);
} }
return kt; // everyone who get this should close it when finished! return kt; // everyone who get this should close it when finished!
} }

@ -72,7 +72,7 @@ public class yacyNewsDB {
this.bufferkb = bufferkb; this.bufferkb = bufferkb;
if (path.exists()) try { if (path.exists()) try {
news = new kelondroTree(path, bufferkb * 0x400); news = new kelondroTree(path, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) { } catch (IOException e) {
news = createDB(path, bufferkb); news = createDB(path, bufferkb);
} else { } else {
@ -81,7 +81,7 @@ public class yacyNewsDB {
} }
private static kelondroTree createDB(File path, int bufferkb) { private static kelondroTree createDB(File path, int bufferkb) {
return new kelondroTree(path, bufferkb * 0x400, new int[] { return new kelondroTree(path, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new int[] {
yacyNewsRecord.idLength(), // id = created + originator yacyNewsRecord.idLength(), // id = created + originator
yacyNewsRecord.categoryStringLength, // category yacyNewsRecord.categoryStringLength, // category
yacyCore.universalDateShortPattern.length(), // received yacyCore.universalDateShortPattern.length(), // received

Loading…
Cancel
Save