- corrections to PerformanceMemory display of object cache

- configuration of object cache size in kelondroTree initializer

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2075 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent fd7c17e624
commit 3e31820c3d

@ -80,22 +80,23 @@
<form action="PerformanceMemory_p.html" method="post" enctype="multipart/form-data">
<tr class="TableHeader" valign="bottom">
<td class="small" rowspan="2">Database<br>&nbsp;</td>
<td class="small" rowspan="2">Node<br>Chunk Size<br>high/med/low<br>(bytes)</td>
<td class="small" colspan="5">#Slots</td>
<td class="small" rowspan="2">Object<br>Chunk Size<br>Max</td>
<td class="small" rowspan="2">Object<br>Chunk Size<br>Current</td>
<td class="small" rowspan="2">Hit/Miss<br>Ratio<br></td>
<td class="small" rowspan="2">Unique/Double<br>Ratio<br></td>
<td class="small" rowspan="2">DB&nbsp;Size<br>&nbsp;</td>
<td class="small" colspan="5">Node Cache</td>
<td class="small" colspan="4">Object Cache</td>
<td class="small" colspan="5">Memory Occupation (KBytes)</td>
<td class="small" rowspan="2">Description<br>&nbsp;</td>
</tr>
<tr class="TableHeader" valign="bottom">
<td class="small">Needed<br>(=&nbsp;DB&nbsp;Size)</td>
<td class="small">Empty<br>&nbsp;</td>
<td class="small">Chunk Size<br>high/med/low<br>(bytes)</td>
<td class="small">Empty<br>(avail.)<br>Slots</td>
<td class="small">Used<br>High Prio</td>
<td class="small">Used<br>Medium Prio</td>
<td class="small">Used<br>Low Prio</td>
<td class="small">Chunk Size<br>Max</td>
<td class="small">Chunk Size<br>Current</td>
<td class="small">Hit/<br>Miss<br>Ratio<br></td>
<td class="small">Unique/<br>Double<br>Ratio<br></td>
<td class="small">Used Now</td>
<td class="small">Assigned Max</td>
<td class="small">Default Max</td>
@ -105,8 +106,8 @@
<tr class="TableCellDark">
<td class="small" align="left">RWI Assortment Cluster</td>
<td class="small" align="center">#[chunkRWI]#</td>
<td class="small" align="right">#[slreqRWI]#</td>
<td class="small" align="center">#[chunkRWI]#</td>
<td class="small" align="right">#[slempRWI]#</td>
<td class="small" align="right">#[slhigRWI]#</td>
<td class="small" align="right">#[slmedRWI]#</td>
@ -127,8 +128,8 @@ cache will speed up crawls with a depth > 3.</td>
<tr class="TableCellDark">
<td class="small" align="left">HTTP Response Header</td>
<td class="small" align="center">#[chunkHTTP]#</td>
<td class="small" align="right">#[slreqHTTP]#</td>
<td class="small" align="center">#[chunkHTTP]#</td>
<td class="small" align="right">#[slempHTTP]#</td>
<td class="small" align="right">#[slhigHTTP]#</td>
<td class="small" align="right">#[slmedHTTP]#</td>
@ -149,8 +150,8 @@ Increasing this cache will be most important for a fast proxy mode.</td>
<tr class="TableCellDark">
<td class="small" align="left">'loaded' URLs</td>
<td class="small" align="center">#[chunkLURL]#</td>
<td class="small" align="right">#[slreqLURL]#</td>
<td class="small" align="center">#[chunkLURL]#</td>
<td class="small" align="right">#[slempLURL]#</td>
<td class="small" align="right">#[slhigLURL]#</td>
<td class="small" align="right">#[slmedLURL]#</td>
@ -170,8 +171,8 @@ This cache is very important for a fast search process. Increasing the cache siz
<tr class="TableCellDark">
<td class="small" align="left">'noticed' URLs</td>
<td class="small" align="center">#[chunkNURL]#</td>
<td class="small" align="right">#[slreqNURL]#</td>
<td class="small" align="center">#[chunkNURL]#</td>
<td class="small" align="right">#[slempNURL]#</td>
<td class="small" align="right">#[slhigNURL]#</td>
<td class="small" align="right">#[slmedNURL]#</td>
@ -191,8 +192,8 @@ Increasing the cache size will result in faster double-check during URL recognit
<tr class="TableCellDark">
<td class="small" align="left">'error' URLs</td>
<td class="small" align="center">#[chunkEURL]#</td>
<td class="small" align="right">#[slreqEURL]#</td>
<td class="small" align="center">#[chunkEURL]#</td>
<td class="small" align="right">#[slempEURL]#</td>
<td class="small" align="right">#[slhigEURL]#</td>
<td class="small" align="right">#[slmedEURL]#</td>
@ -212,8 +213,8 @@ Increasing the cache size will most probably speed up crawling slightly, but not
<tr class="TableCellDark">
<td class="small" align="left">DHT Control</td>
<td class="small" align="center">#[chunkDHT]#</td>
<td class="small" align="right">#[slreqDHT]#</td>
<td class="small" align="center">#[chunkDHT]#</td>
<td class="small" align="right">#[slempDHT]#</td>
<td class="small" align="right">#[slhigDHT]#</td>
<td class="small" align="right">#[slmedDHT]#</td>
@ -234,8 +235,8 @@ Increasing this cache may speed up many functions, but we need to test this to s
<tr class="TableCellDark">
<td class="small" align="left">Messages</td>
<td class="small" align="center">#[chunkMessage]#</td>
<td class="small" align="right">#[slreqMessage]#</td>
<td class="small" align="center">#[chunkMessage]#</td>
<td class="small" align="right">#[slempMessage]#</td>
<td class="small" align="right">#[slhigMessage]#</td>
<td class="small" align="right">#[slmedMessage]#</td>
@ -254,8 +255,8 @@ Increasing this cache may speed up many functions, but we need to test this to s
<tr class="TableCellDark">
<td class="small" align="left">Wiki</td>
<td class="small" align="center">#[chunkWiki]#</td>
<td class="small" align="right">#[slreqWiki]#</td>
<td class="small" align="center">#[chunkWiki]#</td>
<td class="small" align="right">#[slempWiki]#</td>
<td class="small" align="right">#[slhigWiki]#</td>
<td class="small" align="right">#[slmedWiki]#</td>
@ -276,8 +277,8 @@ Increasing this cache may speed up access to the wiki pages.</td>
<tr class="TableCellDark">
<td class="small" align="left">Blog</td>
<td class="small" align="center">#[chunkBlog]#</td>
<td class="small" align="right">#[slreqBlog]#</td>
<td class="small" align="center">#[chunkBlog]#</td>
<td class="small" align="right">#[slempBlog]#</td>
<td class="small" align="right">#[slhigBlog]#</td>
<td class="small" align="right">#[slmedBlog]#</td>
@ -297,8 +298,8 @@ Increasing this cache may speed up access to the Blog.</td>
<tr class="TableCellDark">
<td class="small" align="left">News</td>
<td class="small" align="center">#[chunkNews]#</td>
<td class="small" align="right">#[slreqNews]#</td>
<td class="small" align="center">#[chunkNews]#</td>
<td class="small" align="right">#[slempNews]#</td>
<td class="small" align="right">#[slhigNews]#</td>
<td class="small" align="right">#[slmedNews]#</td>
@ -318,8 +319,8 @@ Increasing this cache may speed up the peer-ping.</td>
<tr class="TableCellDark">
<td class="small" align="left">robots.txt DB</td>
<td class="small" align="center">#[chunkRobots]#</td>
<td class="small" align="right">#[slreqRobots]#</td>
<td class="small" align="center">#[chunkRobots]#</td>
<td class="small" align="right">#[slempRobots]#</td>
<td class="small" align="right">#[slhigRobots]#</td>
<td class="small" align="right">#[slmedRobots]#</td>
@ -339,8 +340,8 @@ Increasing this cache may speed up validation if crawling of the URL is allowed.
<tr class="TableCellDark">
<td class="small" align="left">Crawl Profiles</td>
<td class="small" align="center">#[chunkProfiles]#</td>
<td class="small" align="right">#[slreqProfiles]#</td>
<td class="small" align="center">#[chunkProfiles]#</td>
<td class="small" align="right">#[slempProfiles]#</td>
<td class="small" align="right">#[slhigProfiles]#</td>
<td class="small" align="right">#[slmedProfiles]#</td>
@ -359,7 +360,7 @@ Increasing this cache may speed up crawling, but not much space is needed, so th
</tr>
<tr class="TableCellSummary">
<td class="small" align="left" colspan="7">Totals</td>
<td class="small" align="left" colspan="11">Totals</td>
<td class="small" align="right">#[usedTotal]# MB</td>
<td class="small" align="right">#[currTotal]# MB</td>
<td class="small" align="right">#[dfltTotal]# MB</td>
@ -369,7 +370,7 @@ Increasing this cache may speed up crawling, but not much space is needed, so th
</tr>
<tr class="TableCellLight">
<td class="small" align="right" colspan="8">Re-Configuration:</td>
<td class="small" align="right" colspan="12">Re-Configuration:</td>
<td class="small" align="center"><input type="submit" name="setCustom" value="Set"><br>these<br>custom<br>values<br>&nbsp;</td>
<td class="small" align="center"><input type="submit" name="setDefault" value="Set"><br>all<br>default<br>values<br>&nbsp;</td>
<td class="small" align="center"><input type="submit" name="setGood" value="Set"><br>all<br>recom-<br>mended values</td>

@ -167,9 +167,9 @@ public class dbtest {
if (dbe.equals("kelondroold")) {
File tablefile = new File(tablename + ".kelondro.db");
if (tablefile.exists()) {
table = new kelondroTree(tablefile, buffer);
table = new kelondroTree(tablefile, buffer, kelondroTree.defaultObjectCachePercent);
} else {
table = new kelondroTree(tablefile, buffer, new int[]{keylength, valuelength, valuelength}, true);
table = new kelondroTree(tablefile, buffer, kelondroTree.defaultObjectCachePercent, new int[]{keylength, valuelength, valuelength}, true);
}
}

@ -75,7 +75,7 @@ public class kelondroDyn extends kelondroTree {
public kelondroDyn(File file, long buffersize /*bytes*/, int key, int nodesize, char fillChar, kelondroOrder objectOrder, boolean exitOnFail) {
// creates a new dynamic tree
super(file, buffersize, new int[] {key + counterlen, nodesize}, objectOrder, 1, 8, exitOnFail);
super(file, buffersize, kelondroTree.defaultObjectCachePercent, new int[] {key + counterlen, nodesize}, objectOrder, 1, 8, exitOnFail);
this.keylen = columnSize(0) - counterlen;
this.reclen = columnSize(1);
this.fillChar = fillChar;
@ -88,7 +88,7 @@ public class kelondroDyn extends kelondroTree {
public kelondroDyn(File file, long buffersize, char fillChar) throws IOException {
// this opens a file with an existing dynamic tree
super(file, buffersize);
super(file, buffersize, kelondroTree.defaultObjectCachePercent);
this.keylen = columnSize(0) - counterlen;
this.reclen = columnSize(1);
this.fillChar = fillChar;

@ -135,7 +135,7 @@ public class kelondroDynTree {
kelondroRA ra = table.getRA(key); // works always, even with no-existing entry
treeRAHandles.put(key, ra);
try {
return new kelondroTree(ra, buffersize, columns, false);
return new kelondroTree(ra, buffersize, kelondroTree.defaultObjectCachePercent, columns, false);
} catch (RuntimeException e) {
throw new IOException(e.getMessage());
}
@ -145,7 +145,7 @@ public class kelondroDynTree {
if (table.existsDyn(key)) {
kelondroRA ra = table.getRA(key);
treeRAHandles.put(key, ra);
return new kelondroTree(ra, buffersize);
return new kelondroTree(ra, buffersize, kelondroTree.defaultObjectCachePercent);
} else {
return null;
}

@ -63,18 +63,18 @@ public class kelondroFScoreCluster {
public kelondroFScoreCluster(File refcountDBfile, File countrefDBfile, kelondroOrder objectOrder, boolean exitOnFail) {
if ((refcountDBfile.exists()) && (countrefDBfile.exists())) try {
refcountDB = new kelondroTree(refcountDBfile, 0x100000L);
refcountDB = new kelondroTree(refcountDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent);
refcountDB.setText(0, kelondroBase64Order.enhancedCoder.encodeLong(0, countlength).getBytes()); // counter of all occurrences
countrefDB = new kelondroTree(countrefDBfile, 0x100000L);
countrefDB = new kelondroTree(countrefDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent);
countrefDB.setText(0, kelondroBase64Order.enhancedCoder.encodeLong(0, countlength).getBytes());
} catch (IOException e) {
refcountDBfile.delete();
countrefDBfile.delete();
refcountDB = new kelondroTree(refcountDBfile, 0x100000L, new int[] {wordlength, countlength}, objectOrder, 1, countlength, exitOnFail);
countrefDB = new kelondroTree(countrefDBfile, 0x100000L, new int[] {countlength + wordlength, 4}, objectOrder, 1, countlength, exitOnFail);
refcountDB = new kelondroTree(refcountDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new int[] {wordlength, countlength}, objectOrder, 1, countlength, exitOnFail);
countrefDB = new kelondroTree(countrefDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new int[] {countlength + wordlength, 4}, objectOrder, 1, countlength, exitOnFail);
} else if ((!(refcountDBfile.exists())) && (!(countrefDBfile.exists()))) {
refcountDB = new kelondroTree(refcountDBfile, 0x100000L, new int[] {wordlength, countlength}, objectOrder, 1, countlength, exitOnFail);
countrefDB = new kelondroTree(countrefDBfile, 0x100000L, new int[] {countlength + wordlength, 4}, objectOrder, 1, countlength, exitOnFail);
refcountDB = new kelondroTree(refcountDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new int[] {wordlength, countlength}, objectOrder, 1, countlength, exitOnFail);
countrefDB = new kelondroTree(countrefDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new int[] {countlength + wordlength, 4}, objectOrder, 1, countlength, exitOnFail);
} else {
if (exitOnFail) {
System.exit(-1);

@ -165,7 +165,7 @@ public class kelondroObjectCache {
}
public void put(String key, Object value) {
if (key == null) return;
if ((key == null) || (value == null)) return;
Object prev = null;
synchronized(cache) {
prev = cache.put(key, value);

@ -84,15 +84,15 @@ public class kelondroSplittedTree implements kelondroIndex {
f = dbFile(pathToFiles, filenameStub, forkfactor, columns.length, i);
if (f.exists()) {
try {
ktfs[i] = new kelondroTree(f, buffersize/forkfactor);
ktfs[i] = new kelondroTree(f, buffersize/forkfactor, kelondroTree.defaultObjectCachePercent);
this.order = ktfs[i].order();
} catch (IOException e) {
ktfs[i] = new kelondroTree(f, buffersize/forkfactor,
ktfs[i] = new kelondroTree(f, buffersize/forkfactor, kelondroTree.defaultObjectCachePercent,
columns, objectOrder, txtProps, txtPropsWidth, exitOnFail);
this.order = objectOrder;
}
} else {
ktfs[i] = new kelondroTree(f, buffersize/forkfactor,
ktfs[i] = new kelondroTree(f, buffersize/forkfactor, kelondroTree.defaultObjectCachePercent,
columns, objectOrder, txtProps, txtPropsWidth, exitOnFail);
this.order = objectOrder;
}
@ -104,7 +104,7 @@ public class kelondroSplittedTree implements kelondroIndex {
long buffersize, int forkfactor, int columns) throws IOException {
ktfs = new kelondroTree[forkfactor];
for (int i = 0; i < forkfactor; i++) {
ktfs[i] = new kelondroTree(dbFile(pathToFiles, filenameStub, forkfactor, columns, i), buffersize/forkfactor);
ktfs[i] = new kelondroTree(dbFile(pathToFiles, filenameStub, forkfactor, columns, i), buffersize/forkfactor, kelondroTree.defaultObjectCachePercent);
}
ff = forkfactor;
this.order = objectOrder;

@ -93,13 +93,13 @@ public class kelondroTables {
File tablefile = new File(tablesPath, "table." + tablename + ".tdb");
kelondroTree Tree;
if (tablefile.exists()) try {
Tree = new kelondroTree(tablefile, buffersize);
Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) {
tablefile.getParentFile().mkdirs();
Tree = new kelondroTree(tablefile, buffersize, columns, exitOnFail);
Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent, columns, exitOnFail);
} else {
tablefile.getParentFile().mkdirs();
Tree = new kelondroTree(tablefile, buffersize, columns, exitOnFail);
Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent, columns, exitOnFail);
}
tTables.put(tablename, Tree);
}

@ -81,8 +81,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
private static int root = 0; // pointer for FHandles-array: pointer to root node
// calibration of cache
private static int nodeCachePercent = 90;
private static int objectCachePercent = 10;
public static int defaultObjectCachePercent = 10;
// class variables
private Search writeSearchObj = new Search();
@ -92,23 +91,20 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
private long lastIteratorCount = readAheadChunkSize;
private kelondroObjectCache objectCache;
public kelondroTree(File file, long buffersize, int key, int value, boolean exitOnFail) {
this(file, buffersize, new int[] { key, value }, new kelondroNaturalOrder(true), 1, 8, exitOnFail);
}
public kelondroTree(kelondroRA ra, long buffersize, int key, int value, boolean exitOnFail) {
this(ra, buffersize, new int[] { key, value }, new kelondroNaturalOrder(true), 1, 8, exitOnFail);
public kelondroTree(File file, long buffersize, int objectCachePercent, int key, int value, boolean exitOnFail) {
this(file, buffersize, objectCachePercent, new int[] { key, value }, new kelondroNaturalOrder(true), 1, 8, exitOnFail);
}
public kelondroTree(File file, long buffersize, int[] columns, boolean exitOnFail) {
public kelondroTree(File file, long buffersize, int objectCachePercent, int[] columns, boolean exitOnFail) {
// this creates a new tree file
this(file, buffersize, columns, new kelondroNaturalOrder(true), columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
this(file, buffersize, objectCachePercent, columns, new kelondroNaturalOrder(true), columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
}
public kelondroTree(File file, long buffersize, int[] columns, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) {
public kelondroTree(File file, long buffersize, int objectCachePercent, int[] columns, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) {
// this creates a new tree file
super(file,
nodeCachePercent * buffersize / (nodeCachePercent + objectCachePercent),
(100 - objectCachePercent) * buffersize / 100,
thisOHBytes, thisOHHandles, columns,
thisFHandles, txtProps, txtPropsWidth, exitOnFail);
try {
@ -121,20 +117,18 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
this.objectOrder = objectOrder;
writeOrderType();
super.setLogger(log);
long objectbuffersize = objectCachePercent * buffersize / (nodeCachePercent + objectCachePercent);
long nodecachesize = objectbuffersize / cacheObjectChunkSize();
this.objectCache = new kelondroObjectCache(this.filename, (int) nodecachesize, nodecachesize * 300 , 4*1024*1024);
initObjectCache(buffersize, objectCachePercent);
}
public kelondroTree(kelondroRA ra, long buffersize, int[] columns, boolean exitOnFail) {
public kelondroTree(kelondroRA ra, long buffersize, int objectCachePercent, int[] columns, boolean exitOnFail) {
// this creates a new tree within a kelondroRA
this(ra, buffersize, columns, new kelondroNaturalOrder(true), columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
this(ra, buffersize, objectCachePercent, columns, new kelondroNaturalOrder(true), columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
}
public kelondroTree(kelondroRA ra, long buffersize, int[] columns, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) {
public kelondroTree(kelondroRA ra, long buffersize, int objectCachePercent, int[] columns, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) {
// this creates a new tree within a kelondroRA
super(ra,
nodeCachePercent * buffersize / (nodeCachePercent + objectCachePercent),
(100 - objectCachePercent) * buffersize / 100,
thisOHBytes, thisOHHandles, columns,
thisFHandles, txtProps, txtPropsWidth, exitOnFail);
try {
@ -147,29 +141,33 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
this.objectOrder = objectOrder;
writeOrderType();
super.setLogger(log);
long objectbuffersize = objectCachePercent * buffersize / (nodeCachePercent + objectCachePercent);
long nodecachesize = objectbuffersize / cacheObjectChunkSize();
this.objectCache = new kelondroObjectCache(this.filename, (int) nodecachesize, nodecachesize * 300 , 4*1024*1024);
initObjectCache(buffersize, objectCachePercent);
}
public kelondroTree(File file, long buffersize) throws IOException {
public kelondroTree(File file, long buffersize, int objectCachePercent) throws IOException {
// this opens a file with an existing tree file
super(file, nodeCachePercent * buffersize / (nodeCachePercent + objectCachePercent));
super(file, (100 - objectCachePercent) * buffersize / 100);
readOrderType();
super.setLogger(log);
long objectbuffersize = objectCachePercent * buffersize / (nodeCachePercent + objectCachePercent);
long nodecachesize = objectbuffersize / cacheObjectChunkSize();
this.objectCache = new kelondroObjectCache(this.filename, (int) nodecachesize, nodecachesize * 300 , 4*1024*1024);
initObjectCache(buffersize, objectCachePercent);
}
public kelondroTree(kelondroRA ra, long buffersize) throws IOException {
public kelondroTree(kelondroRA ra, long buffersize, int objectCachePercent) throws IOException {
// this opens a file with an existing tree in a kelondroRA
super(ra, nodeCachePercent * buffersize / (nodeCachePercent + objectCachePercent));
super(ra, (100 - objectCachePercent) * buffersize / 100);
readOrderType();
super.setLogger(log);
long objectbuffersize = objectCachePercent * buffersize / (nodeCachePercent + objectCachePercent);
long nodecachesize = objectbuffersize / cacheObjectChunkSize();
this.objectCache = new kelondroObjectCache(this.filename, (int) nodecachesize, nodecachesize * 300 , 4*1024*1024);
initObjectCache(buffersize, objectCachePercent);
}
private void initObjectCache(long buffersize, int objectCachePercent) {
if (objectCachePercent > 0) {
long objectbuffersize = objectCachePercent * buffersize / 100;
long nodecachesize = objectbuffersize / cacheObjectChunkSize();
this.objectCache = new kelondroObjectCache(this.filename, (int) nodecachesize, nodecachesize * 300 , 4*1024*1024);
} else {
this.objectCache = null;
}
}
public final int cacheObjectChunkSize() {
@ -177,7 +175,8 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
}
public String[] cacheObjectStatus() {
return this.objectCache.status();
if (this.objectCache == null) return null;
else return this.objectCache.status();
}
private void writeOrderType() {
@ -224,22 +223,20 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
// Returns the value to which this map maps the specified key.
public byte[][] get(byte[] key) throws IOException {
// System.out.println("kelondroTree.get " + new String(key) + " in " + filename);
byte[][] result = (byte[][]) objectCache.get(key);
byte[][] result = (objectCache == null) ? null : (byte[][]) objectCache.get(key);
if (result != null) {
//System.out.println("cache hit in objectCache, db:" + super.filename);
return result;
}
// writeLock.stay(2000, 1000);
synchronized (writeSearchObj) {
writeSearchObj.process(key);
if (writeSearchObj.found()) {
result = writeSearchObj.getMatcher().getValues();
objectCache.put(key, result);
if (objectCache != null) objectCache.put(key, result);
} else {
result = null;
}
}
// writeLock.release();
return result;
}
@ -414,7 +411,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
if (newrow.length != columns()) throw new IllegalArgumentException("put: wrong row length " + newrow.length + "; must be " + columns());
// first try to find the key element in the database
synchronized(writeSearchObj) {
objectCache.put(newrow[0], newrow);
if (objectCache != null) objectCache.put(newrow[0], newrow);
writeSearchObj.process(newrow[0]);
if (writeSearchObj.found()) {
// a node with this key exist. simply overwrite the content and return old content
@ -554,7 +551,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
}
private void assignChild(Node parentNode, Node childNode, int childType) throws IOException {
parentNode.setOHHandle(childType, childNode.handle());
parentNode.setOHHandle(childType, childNode.handle());
childNode.setOHHandle(parent, parentNode.handle());
commitNode(parentNode);
commitNode(childNode);
@ -598,71 +595,71 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
}
private static byte max0(byte b) {
if (b > 0) return b; else return 0;
if (b > 0) return b; else return 0;
}
private static byte min0(byte b) {
if (b < 0) return b; else return 0;
if (b < 0) return b; else return 0;
}
private void LL_RightRotation(Node parentNode, Node childNode) throws IOException {
// replace the parent node; the parent is afterwards unlinked
// replace the parent node; the parent is afterwards unlinked
Handle p2Handle = parentNode.getOHHandle(parent);
Node p2Node = (p2Handle == null) ? null : getNode(p2Handle, null, 0);
replace(parentNode, p2Node, childNode);
// set the left son of the parent to the right son of the childNode
Handle childOfChild = childNode.getOHHandle(rightchild);
if (childOfChild == null) {
parentNode.setOHHandle(leftchild, null);
} else {
assignChild(parentNode, getNode(childOfChild, childNode, rightchild), leftchild);
}
// set the left son of the parent to the right son of the childNode
Handle childOfChild = childNode.getOHHandle(rightchild);
if (childOfChild == null) {
parentNode.setOHHandle(leftchild, null);
} else {
assignChild(parentNode, getNode(childOfChild, childNode, rightchild), leftchild);
}
// link the old parent node as the right child of childNode
assignChild(childNode, parentNode, rightchild);
// link the old parent node as the right child of childNode
assignChild(childNode, parentNode, rightchild);
// - newBal(parent) = oldBal(parent) - 1 - max(oldBal(leftChild), 0)
// - newBal(leftChild) = oldBal(leftChild) - 1 + min(newBal(parent), 0)
byte parentBalance = parentNode.getOHByte(balance);
byte childBalance = childNode.getOHByte(balance);
byte oldBalParent = parentBalance;
byte oldBalChild = childBalance;
parentBalance = (byte) (oldBalParent - 1 - max0(oldBalChild));
childBalance = (byte) (oldBalChild - 1 + min0(parentBalance));
parentNode.setOHByte(balance, parentBalance);
// - newBal(parent) = oldBal(parent) - 1 - max(oldBal(leftChild), 0)
// - newBal(leftChild) = oldBal(leftChild) - 1 + min(newBal(parent), 0)
byte parentBalance = parentNode.getOHByte(balance);
byte childBalance = childNode.getOHByte(balance);
byte oldBalParent = parentBalance;
byte oldBalChild = childBalance;
parentBalance = (byte) (oldBalParent - 1 - max0(oldBalChild));
childBalance = (byte) (oldBalChild - 1 + min0(parentBalance));
parentNode.setOHByte(balance, parentBalance);
childNode.setOHByte(balance, childBalance);
commitNode(parentNode);
commitNode(childNode);
}
private void RR_LeftRotation(Node parentNode, Node childNode) throws IOException {
// replace the parent node; the parent is afterwards unlinked
// replace the parent node; the parent is afterwards unlinked
Handle p2Handle = parentNode.getOHHandle(parent);
Node p2Node = (p2Handle == null) ? null : getNode(p2Handle, null, 0);
replace(parentNode, p2Node, childNode);
replace(parentNode, p2Node, childNode);
// set the left son of the parent to the right son of the childNode
Handle childOfChild = childNode.getOHHandle(leftchild);
if (childOfChild == null) {
parentNode.setOHHandle(rightchild, null);
} else {
assignChild(parentNode, getNode(childOfChild, childNode, leftchild), rightchild);
}
// set the left son of the parent to the right son of the childNode
Handle childOfChild = childNode.getOHHandle(leftchild);
if (childOfChild == null) {
parentNode.setOHHandle(rightchild, null);
} else {
assignChild(parentNode, getNode(childOfChild, childNode, leftchild), rightchild);
}
// link the old parent node as the left child of childNode
assignChild(childNode, parentNode, leftchild);
// link the old parent node as the left child of childNode
assignChild(childNode, parentNode, leftchild);
// - newBal(parent) = oldBal(parent) + 1 - min(oldBal(rightChild), 0)
// - newBal(rightChild) = oldBal(rightChild) + 1 + max(newBal(parent), 0)
byte parentBalance = parentNode.getOHByte(balance);
byte childBalance = childNode.getOHByte(balance);
byte oldBalParent = parentBalance;
byte oldBalChild = childBalance;
parentBalance = (byte) (oldBalParent + 1 - min0(oldBalChild));
childBalance = (byte) (oldBalChild + 1 + max0(parentBalance));
parentNode.setOHByte(balance, parentBalance);
childNode.setOHByte(balance, childBalance);
// - newBal(parent) = oldBal(parent) + 1 - min(oldBal(rightChild), 0)
// - newBal(rightChild) = oldBal(rightChild) + 1 + max(newBal(parent), 0)
byte parentBalance = parentNode.getOHByte(balance);
byte childBalance = childNode.getOHByte(balance);
byte oldBalParent = parentBalance;
byte oldBalChild = childBalance;
parentBalance = (byte) (oldBalParent + 1 - min0(oldBalChild));
childBalance = (byte) (oldBalChild + 1 + max0(parentBalance));
parentNode.setOHByte(balance, parentBalance);
childNode.setOHByte(balance, childBalance);
commitNode(parentNode);
commitNode(childNode);
}
@ -679,7 +676,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
// Removes the mapping for this key from this map if present (optional operation).
public byte[][] remove(byte[] key) throws IOException {
synchronized(writeSearchObj) {
objectCache.remove(key);
if (objectCache != null) objectCache.remove(key);
writeSearchObj.process(key);
if (writeSearchObj.found()) {
Node result = writeSearchObj.getMatcher();
@ -1337,7 +1334,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
// test script
File testFile = new File("test.db");
while (testFile.exists()) testFile.delete();
kelondroTree fm = new kelondroTree(testFile, 0x100000, 4, 4, true);
kelondroTree fm = new kelondroTree(testFile, 0x100000, 10, 4, 4, true);
byte[] dummy = "".getBytes();
fm.put("abc0".getBytes(), dummy); fm.put("bcd0".getBytes(), dummy);
fm.put("def0".getBytes(), dummy); fm.put("bab0".getBytes(), dummy);
@ -1359,7 +1356,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
ret = null;
}
} else if (args.length == 2) {
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000);
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10);
if (args[0].equals("-v")) {
fm.print();
ret = null;
@ -1367,11 +1364,11 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
fm.close();
} else if (args.length == 3) {
if (args[0].equals("-d")) {
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000);
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10);
fm.remove(args[2].getBytes());
fm.close();
} else if (args[0].equals("-i")) {
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000);
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10);
int i = fm.imp(new File(args[1]),";");
fm.close();
ret = (i + " records imported").getBytes();
@ -1394,12 +1391,12 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
if (f != null) try {f.close();}catch(Exception e){}
}
} else if (args[0].equals("-g")) {
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000);
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10);
byte[][] ret2 = fm.get(args[2].getBytes());
ret = ((ret2 == null) ? null : ret2[1]);
fm.close();
} else if (args[0].equals("-n")) {
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000);
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10);
//byte[][] keys = fm.getSequentialKeys(args[2].getBytes(), 500, true);
Iterator rowIt = fm.rows(true, false, (args[2].length() == 0) ? null : args[2].getBytes());
Vector v = new Vector();
@ -1415,10 +1412,10 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
int[] lens = new int[2];
lens[0] = Integer.parseInt(args[1]);
lens[1] = Integer.parseInt(args[2]);
kelondroTree fm = new kelondroTree(f, 0x100000, lens, true);
kelondroTree fm = new kelondroTree(f, 0x100000, 10, lens, true);
fm.close();
} else if (args[0].equals("-u")) {
kelondroTree fm = new kelondroTree(new File(args[3]), 0x100000);
kelondroTree fm = new kelondroTree(new File(args[3]), 0x100000, 10);
ret = fm.put(args[1].getBytes(), args[2].getBytes());
fm.close();
}
@ -1481,7 +1478,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
int steps = 0;
while (true) {
if (testFile.exists()) testFile.delete();
tt = new kelondroTree(testFile, 200, 4 ,4, true);
tt = new kelondroTree(testFile, 200, 10, 4 ,4, true);
steps = 10 + ((int) System.currentTimeMillis() % 7) * (((int) System.currentTimeMillis() + 17) % 11);
t = s;
d = "";
@ -1547,7 +1544,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
File f = new File("test.db");
if (f.exists()) f.delete();
try {
kelondroTree tt = new kelondroTree(f, 0, 4, 4, true);
kelondroTree tt = new kelondroTree(f, 0, 10, 4, 4, true);
byte[] b;
b = testWord('B'); tt.put(b, b); //tt.print();
b = testWord('C'); tt.put(b, b); //tt.print();
@ -1581,7 +1578,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
File f = new File("test.db");
if (f.exists()) f.delete();
try {
kelondroTree tt = new kelondroTree(f, 0, 4, 4, true);
kelondroTree tt = new kelondroTree(f, 0, 10, 4, 4, true);
byte[] b;
for (int i = 0; i < 100; i++) {
b = ("T" + i).getBytes(); tt.put(b, b);
@ -1610,7 +1607,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
public static kelondroTree testTree(File f, String testentities) throws IOException {
if (f.exists()) f.delete();
kelondroTree tt = new kelondroTree(f, 0, 4, 4, true);
kelondroTree tt = new kelondroTree(f, 0, 10, 4, 4, true);
byte[] b;
for (int i = 0; i < testentities.length(); i++) {
b = testWord(testentities.charAt(i));

@ -77,14 +77,14 @@ public class plasmaCrawlEURL extends plasmaURL {
};
if (cachePath.exists()) try {
// open existing cache
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400);
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) {
cachePath.delete();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, ce, true);
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true);
} else {
// create new cache
cachePath.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, ce, true);
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true);
}
}

@ -111,15 +111,15 @@ public final class plasmaCrawlLURL extends plasmaURL {
if (cachePath.exists()) {
// open existing cache
try {
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400);
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) {
cachePath.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, ce, true);
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true);
}
} else {
// create new cache
cachePath.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, ce, true);
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true);
}
// init result stacks

@ -163,14 +163,14 @@ public class plasmaCrawlNURL extends plasmaURL {
File cacheFile = new File(cacheStacksPath, "urlNotice1.db");
if (cacheFile.exists()) try {
// open existing cache
urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400);
urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) {
cacheFile.delete();
urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, ce, true);
urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true);
} else {
// create new cache
cacheFile.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, ce, true);
urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true);
}
}

@ -572,10 +572,10 @@ public final class plasmaCrawlStacker {
if (cacheFile.exists()) {
// open existing cache
try {
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400);
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) {
cacheFile.delete();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, plasmaCrawlNURL.ce, true);
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.ce, true);
}
try {
// loop through the list and fill the messageList with url hashs
@ -597,7 +597,7 @@ public final class plasmaCrawlStacker {
// deleting old db and creating a new db
try {this.urlEntryCache.close();}catch(Exception ex){}
cacheFile.delete();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, plasmaCrawlNURL.ce, true);
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.ce, true);
} catch (IOException e) {
/* if we have an error, we start with a fresh database */
plasmaCrawlStacker.this.log.logSevere("Unable to initialize crawl stacker queue, IOException:" + e.getMessage() + ". Reseting DB.\n",e);
@ -605,12 +605,12 @@ public final class plasmaCrawlStacker {
// deleting old db and creating a new db
try {this.urlEntryCache.close();}catch(Exception ex){}
cacheFile.delete();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, plasmaCrawlNURL.ce, true);
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.ce, true);
}
} else {
// create new cache
cacheFile.getParentFile().mkdirs();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, plasmaCrawlNURL.ce, true);
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.ce, true);
}
}

@ -109,7 +109,7 @@ public final class plasmaWordIndexAssortment {
if (assortmentFile.exists()) {
// open existing assortment tree file
try {
assortments = new kelondroTree(assortmentFile, bufferSize);
assortments = new kelondroTree(assortmentFile, bufferSize, kelondroTree.defaultObjectCachePercent);
if (log != null) log.logConfig("Opened Assortment Database, " + assortments.size() + " entries, width " + assortmentLength + ", " + bufferkb + "kb buffer");
return;
} catch (IOException e){
@ -120,7 +120,7 @@ public final class plasmaWordIndexAssortment {
assortmentFile.delete(); // make space for new one
}
// create new assortment tree file
assortments = new kelondroTree(assortmentFile, bufferSize, bufferStructure(assortmentLength), true);
assortments = new kelondroTree(assortmentFile, bufferSize, kelondroTree.defaultObjectCachePercent, bufferStructure(assortmentLength), true);
if (log != null) log.logConfig("Created new Assortment Database, width " + assortmentLength + ", " + bufferkb + "kb buffer");
}
@ -240,7 +240,7 @@ public final class plasmaWordIndexAssortment {
if (!(assortmentFile.delete())) throw new RuntimeException("cannot delete assortment database");
}
if (assortmentFile.exists()) assortmentFile.delete();
assortments = new kelondroTree(assortmentFile, bufferSize, bufferStructure(assortmentLength), true);
assortments = new kelondroTree(assortmentFile, bufferSize, kelondroTree.defaultObjectCachePercent, bufferStructure(assortmentLength), true);
}
public Iterator hashes(String startWordHash, boolean up, boolean rot) throws IOException {

@ -87,13 +87,13 @@ public final class plasmaWordIndexEntity {
if (cacheSize > 1048576) cacheSize = 1048576;
if (theLocation.exists()) try {
// open existing index file
kt = new kelondroTree(theLocation, cacheSize);
kt = new kelondroTree(theLocation, cacheSize, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) {
theLocation.delete();
kt = new kelondroTree(theLocation, cacheSize, plasmaURL.urlHashLength, plasmaWordIndexEntry.attrSpace, false);
kt = new kelondroTree(theLocation, cacheSize, kelondroTree.defaultObjectCachePercent, plasmaURL.urlHashLength, plasmaWordIndexEntry.attrSpace, false);
} else {
// create new index file
kt = new kelondroTree(theLocation, cacheSize, plasmaURL.urlHashLength, plasmaWordIndexEntry.attrSpace, false);
kt = new kelondroTree(theLocation, cacheSize, kelondroTree.defaultObjectCachePercent, plasmaURL.urlHashLength, plasmaWordIndexEntry.attrSpace, false);
}
return kt; // everyone who get this should close it when finished!
}

@ -72,7 +72,7 @@ public class yacyNewsDB {
this.bufferkb = bufferkb;
if (path.exists()) try {
news = new kelondroTree(path, bufferkb * 0x400);
news = new kelondroTree(path, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) {
news = createDB(path, bufferkb);
} else {
@ -81,7 +81,7 @@ public class yacyNewsDB {
}
private static kelondroTree createDB(File path, int bufferkb) {
return new kelondroTree(path, bufferkb * 0x400, new int[] {
return new kelondroTree(path, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new int[] {
yacyNewsRecord.idLength(), // id = created + originator
yacyNewsRecord.categoryStringLength, // category
yacyCore.universalDateShortPattern.length(), // received

Loading…
Cancel
Save