diff --git a/htroot/PerformanceMemory_p.html b/htroot/PerformanceMemory_p.html index c219b8c8b..b3c6c627a 100644 --- a/htroot/PerformanceMemory_p.html +++ b/htroot/PerformanceMemory_p.html @@ -80,22 +80,23 @@
Database
  -Node
Chunk Size
high/med/low
(bytes) -#Slots -Object
Chunk Size
Max -Object
Chunk Size
Current -Hit/Miss
Ratio
-Unique/Double
Ratio
+DB Size
  +Node Cache +Object Cache Memory Occupation (KBytes) Description
  -Needed
(= DB Size) -Empty
  +Chunk Size
high/med/low
(bytes) +Empty
(avail.)
Slots Used
High Prio Used
Medium Prio Used
Low Prio +Chunk Size
Max +Chunk Size
Current +Hit/
Miss
Ratio
+Unique/
Double
Ratio
Used Now Assigned Max Default Max @@ -105,8 +106,8 @@ RWI Assortment Cluster -#[chunkRWI]# #[slreqRWI]# +#[chunkRWI]# #[slempRWI]# #[slhigRWI]# #[slmedRWI]# @@ -127,8 +128,8 @@ cache will speed up crawls with a depth > 3. HTTP Response Header -#[chunkHTTP]# #[slreqHTTP]# +#[chunkHTTP]# #[slempHTTP]# #[slhigHTTP]# #[slmedHTTP]# @@ -149,8 +150,8 @@ Increasing this cache will be most important for a fast proxy mode. 'loaded' URLs -#[chunkLURL]# #[slreqLURL]# +#[chunkLURL]# #[slempLURL]# #[slhigLURL]# #[slmedLURL]# @@ -170,8 +171,8 @@ This cache is very important for a fast search process. Increasing the cache siz 'noticed' URLs -#[chunkNURL]# #[slreqNURL]# +#[chunkNURL]# #[slempNURL]# #[slhigNURL]# #[slmedNURL]# @@ -191,8 +192,8 @@ Increasing the cache size will result in faster double-check during URL recognit 'error' URLs -#[chunkEURL]# #[slreqEURL]# +#[chunkEURL]# #[slempEURL]# #[slhigEURL]# #[slmedEURL]# @@ -212,8 +213,8 @@ Increasing the cache size will most probably speed up crawling slightly, but not DHT Control -#[chunkDHT]# #[slreqDHT]# +#[chunkDHT]# #[slempDHT]# #[slhigDHT]# #[slmedDHT]# @@ -234,8 +235,8 @@ Increasing this cache may speed up many functions, but we need to test this to s Messages -#[chunkMessage]# #[slreqMessage]# +#[chunkMessage]# #[slempMessage]# #[slhigMessage]# #[slmedMessage]# @@ -254,8 +255,8 @@ Increasing this cache may speed up many functions, but we need to test this to s Wiki -#[chunkWiki]# #[slreqWiki]# +#[chunkWiki]# #[slempWiki]# #[slhigWiki]# #[slmedWiki]# @@ -276,8 +277,8 @@ Increasing this cache may speed up access to the wiki pages. Blog -#[chunkBlog]# #[slreqBlog]# +#[chunkBlog]# #[slempBlog]# #[slhigBlog]# #[slmedBlog]# @@ -297,8 +298,8 @@ Increasing this cache may speed up access to the Blog. News -#[chunkNews]# #[slreqNews]# +#[chunkNews]# #[slempNews]# #[slhigNews]# #[slmedNews]# @@ -318,8 +319,8 @@ Increasing this cache may speed up the peer-ping. robots.txt DB -#[chunkRobots]# #[slreqRobots]# +#[chunkRobots]# #[slempRobots]# #[slhigRobots]# #[slmedRobots]# @@ -339,8 +340,8 @@ Increasing this cache may speed up validation if crawling of the URL is allowed. Crawl Profiles -#[chunkProfiles]# #[slreqProfiles]# +#[chunkProfiles]# #[slempProfiles]# #[slhigProfiles]# #[slmedProfiles]# @@ -359,7 +360,7 @@ Increasing this cache may speed up crawling, but not much space is needed, so th -Totals +Totals #[usedTotal]# MB #[currTotal]# MB #[dfltTotal]# MB @@ -369,7 +370,7 @@ Increasing this cache may speed up crawling, but not much space is needed, so th -Re-Configuration: +Re-Configuration:
these
custom
values
 
all
default
values
 
all
recom-
mended values diff --git a/source/dbtest.java b/source/dbtest.java index f1371efc1..0fbbd9a4e 100644 --- a/source/dbtest.java +++ b/source/dbtest.java @@ -167,9 +167,9 @@ public class dbtest { if (dbe.equals("kelondroold")) { File tablefile = new File(tablename + ".kelondro.db"); if (tablefile.exists()) { - table = new kelondroTree(tablefile, buffer); + table = new kelondroTree(tablefile, buffer, kelondroTree.defaultObjectCachePercent); } else { - table = new kelondroTree(tablefile, buffer, new int[]{keylength, valuelength, valuelength}, true); + table = new kelondroTree(tablefile, buffer, kelondroTree.defaultObjectCachePercent, new int[]{keylength, valuelength, valuelength}, true); } } diff --git a/source/de/anomic/kelondro/kelondroDyn.java b/source/de/anomic/kelondro/kelondroDyn.java index 9c1f21bf8..72a339158 100644 --- a/source/de/anomic/kelondro/kelondroDyn.java +++ b/source/de/anomic/kelondro/kelondroDyn.java @@ -75,7 +75,7 @@ public class kelondroDyn extends kelondroTree { public kelondroDyn(File file, long buffersize /*bytes*/, int key, int nodesize, char fillChar, kelondroOrder objectOrder, boolean exitOnFail) { // creates a new dynamic tree - super(file, buffersize, new int[] {key + counterlen, nodesize}, objectOrder, 1, 8, exitOnFail); + super(file, buffersize, kelondroTree.defaultObjectCachePercent, new int[] {key + counterlen, nodesize}, objectOrder, 1, 8, exitOnFail); this.keylen = columnSize(0) - counterlen; this.reclen = columnSize(1); this.fillChar = fillChar; @@ -88,7 +88,7 @@ public class kelondroDyn extends kelondroTree { public kelondroDyn(File file, long buffersize, char fillChar) throws IOException { // this opens a file with an existing dynamic tree - super(file, buffersize); + super(file, buffersize, kelondroTree.defaultObjectCachePercent); this.keylen = columnSize(0) - counterlen; this.reclen = columnSize(1); this.fillChar = fillChar; diff --git a/source/de/anomic/kelondro/kelondroDynTree.java b/source/de/anomic/kelondro/kelondroDynTree.java index 6c9628792..2802be463 100644 --- a/source/de/anomic/kelondro/kelondroDynTree.java +++ b/source/de/anomic/kelondro/kelondroDynTree.java @@ -135,7 +135,7 @@ public class kelondroDynTree { kelondroRA ra = table.getRA(key); // works always, even with no-existing entry treeRAHandles.put(key, ra); try { - return new kelondroTree(ra, buffersize, columns, false); + return new kelondroTree(ra, buffersize, kelondroTree.defaultObjectCachePercent, columns, false); } catch (RuntimeException e) { throw new IOException(e.getMessage()); } @@ -145,7 +145,7 @@ public class kelondroDynTree { if (table.existsDyn(key)) { kelondroRA ra = table.getRA(key); treeRAHandles.put(key, ra); - return new kelondroTree(ra, buffersize); + return new kelondroTree(ra, buffersize, kelondroTree.defaultObjectCachePercent); } else { return null; } diff --git a/source/de/anomic/kelondro/kelondroFScoreCluster.java b/source/de/anomic/kelondro/kelondroFScoreCluster.java index 65eb8e2d7..cbdf57758 100644 --- a/source/de/anomic/kelondro/kelondroFScoreCluster.java +++ b/source/de/anomic/kelondro/kelondroFScoreCluster.java @@ -63,18 +63,18 @@ public class kelondroFScoreCluster { public kelondroFScoreCluster(File refcountDBfile, File countrefDBfile, kelondroOrder objectOrder, boolean exitOnFail) { if ((refcountDBfile.exists()) && (countrefDBfile.exists())) try { - refcountDB = new kelondroTree(refcountDBfile, 0x100000L); + refcountDB = new kelondroTree(refcountDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent); refcountDB.setText(0, kelondroBase64Order.enhancedCoder.encodeLong(0, countlength).getBytes()); // counter of all occurrences - countrefDB = new kelondroTree(countrefDBfile, 0x100000L); + countrefDB = new kelondroTree(countrefDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent); countrefDB.setText(0, kelondroBase64Order.enhancedCoder.encodeLong(0, countlength).getBytes()); } catch (IOException e) { refcountDBfile.delete(); countrefDBfile.delete(); - refcountDB = new kelondroTree(refcountDBfile, 0x100000L, new int[] {wordlength, countlength}, objectOrder, 1, countlength, exitOnFail); - countrefDB = new kelondroTree(countrefDBfile, 0x100000L, new int[] {countlength + wordlength, 4}, objectOrder, 1, countlength, exitOnFail); + refcountDB = new kelondroTree(refcountDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new int[] {wordlength, countlength}, objectOrder, 1, countlength, exitOnFail); + countrefDB = new kelondroTree(countrefDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new int[] {countlength + wordlength, 4}, objectOrder, 1, countlength, exitOnFail); } else if ((!(refcountDBfile.exists())) && (!(countrefDBfile.exists()))) { - refcountDB = new kelondroTree(refcountDBfile, 0x100000L, new int[] {wordlength, countlength}, objectOrder, 1, countlength, exitOnFail); - countrefDB = new kelondroTree(countrefDBfile, 0x100000L, new int[] {countlength + wordlength, 4}, objectOrder, 1, countlength, exitOnFail); + refcountDB = new kelondroTree(refcountDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new int[] {wordlength, countlength}, objectOrder, 1, countlength, exitOnFail); + countrefDB = new kelondroTree(countrefDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new int[] {countlength + wordlength, 4}, objectOrder, 1, countlength, exitOnFail); } else { if (exitOnFail) { System.exit(-1); diff --git a/source/de/anomic/kelondro/kelondroObjectCache.java b/source/de/anomic/kelondro/kelondroObjectCache.java index 8163849c0..e177ddb0f 100644 --- a/source/de/anomic/kelondro/kelondroObjectCache.java +++ b/source/de/anomic/kelondro/kelondroObjectCache.java @@ -165,7 +165,7 @@ public class kelondroObjectCache { } public void put(String key, Object value) { - if (key == null) return; + if ((key == null) || (value == null)) return; Object prev = null; synchronized(cache) { prev = cache.put(key, value); diff --git a/source/de/anomic/kelondro/kelondroSplittedTree.java b/source/de/anomic/kelondro/kelondroSplittedTree.java index e030bdad4..2bd2b1ebf 100644 --- a/source/de/anomic/kelondro/kelondroSplittedTree.java +++ b/source/de/anomic/kelondro/kelondroSplittedTree.java @@ -84,15 +84,15 @@ public class kelondroSplittedTree implements kelondroIndex { f = dbFile(pathToFiles, filenameStub, forkfactor, columns.length, i); if (f.exists()) { try { - ktfs[i] = new kelondroTree(f, buffersize/forkfactor); + ktfs[i] = new kelondroTree(f, buffersize/forkfactor, kelondroTree.defaultObjectCachePercent); this.order = ktfs[i].order(); } catch (IOException e) { - ktfs[i] = new kelondroTree(f, buffersize/forkfactor, + ktfs[i] = new kelondroTree(f, buffersize/forkfactor, kelondroTree.defaultObjectCachePercent, columns, objectOrder, txtProps, txtPropsWidth, exitOnFail); this.order = objectOrder; } } else { - ktfs[i] = new kelondroTree(f, buffersize/forkfactor, + ktfs[i] = new kelondroTree(f, buffersize/forkfactor, kelondroTree.defaultObjectCachePercent, columns, objectOrder, txtProps, txtPropsWidth, exitOnFail); this.order = objectOrder; } @@ -104,7 +104,7 @@ public class kelondroSplittedTree implements kelondroIndex { long buffersize, int forkfactor, int columns) throws IOException { ktfs = new kelondroTree[forkfactor]; for (int i = 0; i < forkfactor; i++) { - ktfs[i] = new kelondroTree(dbFile(pathToFiles, filenameStub, forkfactor, columns, i), buffersize/forkfactor); + ktfs[i] = new kelondroTree(dbFile(pathToFiles, filenameStub, forkfactor, columns, i), buffersize/forkfactor, kelondroTree.defaultObjectCachePercent); } ff = forkfactor; this.order = objectOrder; diff --git a/source/de/anomic/kelondro/kelondroTables.java b/source/de/anomic/kelondro/kelondroTables.java index 13badd72c..3d9b05bcb 100644 --- a/source/de/anomic/kelondro/kelondroTables.java +++ b/source/de/anomic/kelondro/kelondroTables.java @@ -93,13 +93,13 @@ public class kelondroTables { File tablefile = new File(tablesPath, "table." + tablename + ".tdb"); kelondroTree Tree; if (tablefile.exists()) try { - Tree = new kelondroTree(tablefile, buffersize); + Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent); } catch (IOException e) { tablefile.getParentFile().mkdirs(); - Tree = new kelondroTree(tablefile, buffersize, columns, exitOnFail); + Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent, columns, exitOnFail); } else { tablefile.getParentFile().mkdirs(); - Tree = new kelondroTree(tablefile, buffersize, columns, exitOnFail); + Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent, columns, exitOnFail); } tTables.put(tablename, Tree); } diff --git a/source/de/anomic/kelondro/kelondroTree.java b/source/de/anomic/kelondro/kelondroTree.java index 0fba669ed..6a6af60e1 100644 --- a/source/de/anomic/kelondro/kelondroTree.java +++ b/source/de/anomic/kelondro/kelondroTree.java @@ -81,8 +81,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { private static int root = 0; // pointer for FHandles-array: pointer to root node // calibration of cache - private static int nodeCachePercent = 90; - private static int objectCachePercent = 10; + public static int defaultObjectCachePercent = 10; // class variables private Search writeSearchObj = new Search(); @@ -92,23 +91,20 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { private long lastIteratorCount = readAheadChunkSize; private kelondroObjectCache objectCache; - public kelondroTree(File file, long buffersize, int key, int value, boolean exitOnFail) { - this(file, buffersize, new int[] { key, value }, new kelondroNaturalOrder(true), 1, 8, exitOnFail); - } - - public kelondroTree(kelondroRA ra, long buffersize, int key, int value, boolean exitOnFail) { - this(ra, buffersize, new int[] { key, value }, new kelondroNaturalOrder(true), 1, 8, exitOnFail); + + public kelondroTree(File file, long buffersize, int objectCachePercent, int key, int value, boolean exitOnFail) { + this(file, buffersize, objectCachePercent, new int[] { key, value }, new kelondroNaturalOrder(true), 1, 8, exitOnFail); } - public kelondroTree(File file, long buffersize, int[] columns, boolean exitOnFail) { + public kelondroTree(File file, long buffersize, int objectCachePercent, int[] columns, boolean exitOnFail) { // this creates a new tree file - this(file, buffersize, columns, new kelondroNaturalOrder(true), columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail); + this(file, buffersize, objectCachePercent, columns, new kelondroNaturalOrder(true), columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail); } - public kelondroTree(File file, long buffersize, int[] columns, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) { + public kelondroTree(File file, long buffersize, int objectCachePercent, int[] columns, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) { // this creates a new tree file super(file, - nodeCachePercent * buffersize / (nodeCachePercent + objectCachePercent), + (100 - objectCachePercent) * buffersize / 100, thisOHBytes, thisOHHandles, columns, thisFHandles, txtProps, txtPropsWidth, exitOnFail); try { @@ -121,20 +117,18 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { this.objectOrder = objectOrder; writeOrderType(); super.setLogger(log); - long objectbuffersize = objectCachePercent * buffersize / (nodeCachePercent + objectCachePercent); - long nodecachesize = objectbuffersize / cacheObjectChunkSize(); - this.objectCache = new kelondroObjectCache(this.filename, (int) nodecachesize, nodecachesize * 300 , 4*1024*1024); + initObjectCache(buffersize, objectCachePercent); } - public kelondroTree(kelondroRA ra, long buffersize, int[] columns, boolean exitOnFail) { + public kelondroTree(kelondroRA ra, long buffersize, int objectCachePercent, int[] columns, boolean exitOnFail) { // this creates a new tree within a kelondroRA - this(ra, buffersize, columns, new kelondroNaturalOrder(true), columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail); + this(ra, buffersize, objectCachePercent, columns, new kelondroNaturalOrder(true), columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail); } - public kelondroTree(kelondroRA ra, long buffersize, int[] columns, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) { + public kelondroTree(kelondroRA ra, long buffersize, int objectCachePercent, int[] columns, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) { // this creates a new tree within a kelondroRA super(ra, - nodeCachePercent * buffersize / (nodeCachePercent + objectCachePercent), + (100 - objectCachePercent) * buffersize / 100, thisOHBytes, thisOHHandles, columns, thisFHandles, txtProps, txtPropsWidth, exitOnFail); try { @@ -147,29 +141,33 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { this.objectOrder = objectOrder; writeOrderType(); super.setLogger(log); - long objectbuffersize = objectCachePercent * buffersize / (nodeCachePercent + objectCachePercent); - long nodecachesize = objectbuffersize / cacheObjectChunkSize(); - this.objectCache = new kelondroObjectCache(this.filename, (int) nodecachesize, nodecachesize * 300 , 4*1024*1024); + initObjectCache(buffersize, objectCachePercent); } - - public kelondroTree(File file, long buffersize) throws IOException { + + public kelondroTree(File file, long buffersize, int objectCachePercent) throws IOException { // this opens a file with an existing tree file - super(file, nodeCachePercent * buffersize / (nodeCachePercent + objectCachePercent)); + super(file, (100 - objectCachePercent) * buffersize / 100); readOrderType(); super.setLogger(log); - long objectbuffersize = objectCachePercent * buffersize / (nodeCachePercent + objectCachePercent); - long nodecachesize = objectbuffersize / cacheObjectChunkSize(); - this.objectCache = new kelondroObjectCache(this.filename, (int) nodecachesize, nodecachesize * 300 , 4*1024*1024); + initObjectCache(buffersize, objectCachePercent); } - - public kelondroTree(kelondroRA ra, long buffersize) throws IOException { + + public kelondroTree(kelondroRA ra, long buffersize, int objectCachePercent) throws IOException { // this opens a file with an existing tree in a kelondroRA - super(ra, nodeCachePercent * buffersize / (nodeCachePercent + objectCachePercent)); + super(ra, (100 - objectCachePercent) * buffersize / 100); readOrderType(); super.setLogger(log); - long objectbuffersize = objectCachePercent * buffersize / (nodeCachePercent + objectCachePercent); - long nodecachesize = objectbuffersize / cacheObjectChunkSize(); - this.objectCache = new kelondroObjectCache(this.filename, (int) nodecachesize, nodecachesize * 300 , 4*1024*1024); + initObjectCache(buffersize, objectCachePercent); + } + + private void initObjectCache(long buffersize, int objectCachePercent) { + if (objectCachePercent > 0) { + long objectbuffersize = objectCachePercent * buffersize / 100; + long nodecachesize = objectbuffersize / cacheObjectChunkSize(); + this.objectCache = new kelondroObjectCache(this.filename, (int) nodecachesize, nodecachesize * 300 , 4*1024*1024); + } else { + this.objectCache = null; + } } public final int cacheObjectChunkSize() { @@ -177,7 +175,8 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { } public String[] cacheObjectStatus() { - return this.objectCache.status(); + if (this.objectCache == null) return null; + else return this.objectCache.status(); } private void writeOrderType() { @@ -224,22 +223,20 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { // Returns the value to which this map maps the specified key. public byte[][] get(byte[] key) throws IOException { // System.out.println("kelondroTree.get " + new String(key) + " in " + filename); - byte[][] result = (byte[][]) objectCache.get(key); + byte[][] result = (objectCache == null) ? null : (byte[][]) objectCache.get(key); if (result != null) { //System.out.println("cache hit in objectCache, db:" + super.filename); return result; } - // writeLock.stay(2000, 1000); synchronized (writeSearchObj) { writeSearchObj.process(key); if (writeSearchObj.found()) { result = writeSearchObj.getMatcher().getValues(); - objectCache.put(key, result); + if (objectCache != null) objectCache.put(key, result); } else { result = null; } } - // writeLock.release(); return result; } @@ -414,7 +411,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { if (newrow.length != columns()) throw new IllegalArgumentException("put: wrong row length " + newrow.length + "; must be " + columns()); // first try to find the key element in the database synchronized(writeSearchObj) { - objectCache.put(newrow[0], newrow); + if (objectCache != null) objectCache.put(newrow[0], newrow); writeSearchObj.process(newrow[0]); if (writeSearchObj.found()) { // a node with this key exist. simply overwrite the content and return old content @@ -554,7 +551,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { } private void assignChild(Node parentNode, Node childNode, int childType) throws IOException { - parentNode.setOHHandle(childType, childNode.handle()); + parentNode.setOHHandle(childType, childNode.handle()); childNode.setOHHandle(parent, parentNode.handle()); commitNode(parentNode); commitNode(childNode); @@ -598,71 +595,71 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { } private static byte max0(byte b) { - if (b > 0) return b; else return 0; + if (b > 0) return b; else return 0; } private static byte min0(byte b) { - if (b < 0) return b; else return 0; + if (b < 0) return b; else return 0; } private void LL_RightRotation(Node parentNode, Node childNode) throws IOException { - // replace the parent node; the parent is afterwards unlinked + // replace the parent node; the parent is afterwards unlinked Handle p2Handle = parentNode.getOHHandle(parent); Node p2Node = (p2Handle == null) ? null : getNode(p2Handle, null, 0); replace(parentNode, p2Node, childNode); - // set the left son of the parent to the right son of the childNode - Handle childOfChild = childNode.getOHHandle(rightchild); - if (childOfChild == null) { - parentNode.setOHHandle(leftchild, null); - } else { - assignChild(parentNode, getNode(childOfChild, childNode, rightchild), leftchild); - } + // set the left son of the parent to the right son of the childNode + Handle childOfChild = childNode.getOHHandle(rightchild); + if (childOfChild == null) { + parentNode.setOHHandle(leftchild, null); + } else { + assignChild(parentNode, getNode(childOfChild, childNode, rightchild), leftchild); + } - // link the old parent node as the right child of childNode - assignChild(childNode, parentNode, rightchild); + // link the old parent node as the right child of childNode + assignChild(childNode, parentNode, rightchild); - // - newBal(parent) = oldBal(parent) - 1 - max(oldBal(leftChild), 0) - // - newBal(leftChild) = oldBal(leftChild) - 1 + min(newBal(parent), 0) - byte parentBalance = parentNode.getOHByte(balance); - byte childBalance = childNode.getOHByte(balance); - byte oldBalParent = parentBalance; - byte oldBalChild = childBalance; - parentBalance = (byte) (oldBalParent - 1 - max0(oldBalChild)); - childBalance = (byte) (oldBalChild - 1 + min0(parentBalance)); - parentNode.setOHByte(balance, parentBalance); + // - newBal(parent) = oldBal(parent) - 1 - max(oldBal(leftChild), 0) + // - newBal(leftChild) = oldBal(leftChild) - 1 + min(newBal(parent), 0) + byte parentBalance = parentNode.getOHByte(balance); + byte childBalance = childNode.getOHByte(balance); + byte oldBalParent = parentBalance; + byte oldBalChild = childBalance; + parentBalance = (byte) (oldBalParent - 1 - max0(oldBalChild)); + childBalance = (byte) (oldBalChild - 1 + min0(parentBalance)); + parentNode.setOHByte(balance, parentBalance); childNode.setOHByte(balance, childBalance); commitNode(parentNode); commitNode(childNode); } private void RR_LeftRotation(Node parentNode, Node childNode) throws IOException { - // replace the parent node; the parent is afterwards unlinked + // replace the parent node; the parent is afterwards unlinked Handle p2Handle = parentNode.getOHHandle(parent); Node p2Node = (p2Handle == null) ? null : getNode(p2Handle, null, 0); - replace(parentNode, p2Node, childNode); + replace(parentNode, p2Node, childNode); - // set the left son of the parent to the right son of the childNode - Handle childOfChild = childNode.getOHHandle(leftchild); - if (childOfChild == null) { - parentNode.setOHHandle(rightchild, null); - } else { - assignChild(parentNode, getNode(childOfChild, childNode, leftchild), rightchild); - } + // set the left son of the parent to the right son of the childNode + Handle childOfChild = childNode.getOHHandle(leftchild); + if (childOfChild == null) { + parentNode.setOHHandle(rightchild, null); + } else { + assignChild(parentNode, getNode(childOfChild, childNode, leftchild), rightchild); + } - // link the old parent node as the left child of childNode - assignChild(childNode, parentNode, leftchild); + // link the old parent node as the left child of childNode + assignChild(childNode, parentNode, leftchild); - // - newBal(parent) = oldBal(parent) + 1 - min(oldBal(rightChild), 0) - // - newBal(rightChild) = oldBal(rightChild) + 1 + max(newBal(parent), 0) - byte parentBalance = parentNode.getOHByte(balance); - byte childBalance = childNode.getOHByte(balance); - byte oldBalParent = parentBalance; - byte oldBalChild = childBalance; - parentBalance = (byte) (oldBalParent + 1 - min0(oldBalChild)); - childBalance = (byte) (oldBalChild + 1 + max0(parentBalance)); - parentNode.setOHByte(balance, parentBalance); - childNode.setOHByte(balance, childBalance); + // - newBal(parent) = oldBal(parent) + 1 - min(oldBal(rightChild), 0) + // - newBal(rightChild) = oldBal(rightChild) + 1 + max(newBal(parent), 0) + byte parentBalance = parentNode.getOHByte(balance); + byte childBalance = childNode.getOHByte(balance); + byte oldBalParent = parentBalance; + byte oldBalChild = childBalance; + parentBalance = (byte) (oldBalParent + 1 - min0(oldBalChild)); + childBalance = (byte) (oldBalChild + 1 + max0(parentBalance)); + parentNode.setOHByte(balance, parentBalance); + childNode.setOHByte(balance, childBalance); commitNode(parentNode); commitNode(childNode); } @@ -679,7 +676,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { // Removes the mapping for this key from this map if present (optional operation). public byte[][] remove(byte[] key) throws IOException { synchronized(writeSearchObj) { - objectCache.remove(key); + if (objectCache != null) objectCache.remove(key); writeSearchObj.process(key); if (writeSearchObj.found()) { Node result = writeSearchObj.getMatcher(); @@ -1337,7 +1334,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { // test script File testFile = new File("test.db"); while (testFile.exists()) testFile.delete(); - kelondroTree fm = new kelondroTree(testFile, 0x100000, 4, 4, true); + kelondroTree fm = new kelondroTree(testFile, 0x100000, 10, 4, 4, true); byte[] dummy = "".getBytes(); fm.put("abc0".getBytes(), dummy); fm.put("bcd0".getBytes(), dummy); fm.put("def0".getBytes(), dummy); fm.put("bab0".getBytes(), dummy); @@ -1359,7 +1356,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { ret = null; } } else if (args.length == 2) { - kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000); + kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10); if (args[0].equals("-v")) { fm.print(); ret = null; @@ -1367,11 +1364,11 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { fm.close(); } else if (args.length == 3) { if (args[0].equals("-d")) { - kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000); + kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10); fm.remove(args[2].getBytes()); fm.close(); } else if (args[0].equals("-i")) { - kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000); + kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10); int i = fm.imp(new File(args[1]),";"); fm.close(); ret = (i + " records imported").getBytes(); @@ -1394,12 +1391,12 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { if (f != null) try {f.close();}catch(Exception e){} } } else if (args[0].equals("-g")) { - kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000); + kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10); byte[][] ret2 = fm.get(args[2].getBytes()); ret = ((ret2 == null) ? null : ret2[1]); fm.close(); } else if (args[0].equals("-n")) { - kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000); + kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10); //byte[][] keys = fm.getSequentialKeys(args[2].getBytes(), 500, true); Iterator rowIt = fm.rows(true, false, (args[2].length() == 0) ? null : args[2].getBytes()); Vector v = new Vector(); @@ -1415,10 +1412,10 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { int[] lens = new int[2]; lens[0] = Integer.parseInt(args[1]); lens[1] = Integer.parseInt(args[2]); - kelondroTree fm = new kelondroTree(f, 0x100000, lens, true); + kelondroTree fm = new kelondroTree(f, 0x100000, 10, lens, true); fm.close(); } else if (args[0].equals("-u")) { - kelondroTree fm = new kelondroTree(new File(args[3]), 0x100000); + kelondroTree fm = new kelondroTree(new File(args[3]), 0x100000, 10); ret = fm.put(args[1].getBytes(), args[2].getBytes()); fm.close(); } @@ -1481,7 +1478,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { int steps = 0; while (true) { if (testFile.exists()) testFile.delete(); - tt = new kelondroTree(testFile, 200, 4 ,4, true); + tt = new kelondroTree(testFile, 200, 10, 4 ,4, true); steps = 10 + ((int) System.currentTimeMillis() % 7) * (((int) System.currentTimeMillis() + 17) % 11); t = s; d = ""; @@ -1547,7 +1544,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { File f = new File("test.db"); if (f.exists()) f.delete(); try { - kelondroTree tt = new kelondroTree(f, 0, 4, 4, true); + kelondroTree tt = new kelondroTree(f, 0, 10, 4, 4, true); byte[] b; b = testWord('B'); tt.put(b, b); //tt.print(); b = testWord('C'); tt.put(b, b); //tt.print(); @@ -1581,7 +1578,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { File f = new File("test.db"); if (f.exists()) f.delete(); try { - kelondroTree tt = new kelondroTree(f, 0, 4, 4, true); + kelondroTree tt = new kelondroTree(f, 0, 10, 4, 4, true); byte[] b; for (int i = 0; i < 100; i++) { b = ("T" + i).getBytes(); tt.put(b, b); @@ -1610,7 +1607,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { public static kelondroTree testTree(File f, String testentities) throws IOException { if (f.exists()) f.delete(); - kelondroTree tt = new kelondroTree(f, 0, 4, 4, true); + kelondroTree tt = new kelondroTree(f, 0, 10, 4, 4, true); byte[] b; for (int i = 0; i < testentities.length(); i++) { b = testWord(testentities.charAt(i)); diff --git a/source/de/anomic/plasma/plasmaCrawlEURL.java b/source/de/anomic/plasma/plasmaCrawlEURL.java index 5846e0e3f..b325246ca 100644 --- a/source/de/anomic/plasma/plasmaCrawlEURL.java +++ b/source/de/anomic/plasma/plasmaCrawlEURL.java @@ -77,14 +77,14 @@ public class plasmaCrawlEURL extends plasmaURL { }; if (cachePath.exists()) try { // open existing cache - urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400); + urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent); } catch (IOException e) { cachePath.delete(); - urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, ce, true); + urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true); } else { // create new cache cachePath.getParentFile().mkdirs(); - urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, ce, true); + urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true); } } diff --git a/source/de/anomic/plasma/plasmaCrawlLURL.java b/source/de/anomic/plasma/plasmaCrawlLURL.java index 68bcd773d..dab8bbd39 100644 --- a/source/de/anomic/plasma/plasmaCrawlLURL.java +++ b/source/de/anomic/plasma/plasmaCrawlLURL.java @@ -111,15 +111,15 @@ public final class plasmaCrawlLURL extends plasmaURL { if (cachePath.exists()) { // open existing cache try { - urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400); + urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent); } catch (IOException e) { cachePath.getParentFile().mkdirs(); - urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, ce, true); + urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true); } } else { // create new cache cachePath.getParentFile().mkdirs(); - urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, ce, true); + urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true); } // init result stacks diff --git a/source/de/anomic/plasma/plasmaCrawlNURL.java b/source/de/anomic/plasma/plasmaCrawlNURL.java index 4dcf7a5cd..1c325bd82 100644 --- a/source/de/anomic/plasma/plasmaCrawlNURL.java +++ b/source/de/anomic/plasma/plasmaCrawlNURL.java @@ -163,14 +163,14 @@ public class plasmaCrawlNURL extends plasmaURL { File cacheFile = new File(cacheStacksPath, "urlNotice1.db"); if (cacheFile.exists()) try { // open existing cache - urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400); + urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent); } catch (IOException e) { cacheFile.delete(); - urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, ce, true); + urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true); } else { // create new cache cacheFile.getParentFile().mkdirs(); - urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, ce, true); + urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true); } } diff --git a/source/de/anomic/plasma/plasmaCrawlStacker.java b/source/de/anomic/plasma/plasmaCrawlStacker.java index 735ae2331..daff35276 100644 --- a/source/de/anomic/plasma/plasmaCrawlStacker.java +++ b/source/de/anomic/plasma/plasmaCrawlStacker.java @@ -572,10 +572,10 @@ public final class plasmaCrawlStacker { if (cacheFile.exists()) { // open existing cache try { - this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400); + this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent); } catch (IOException e) { cacheFile.delete(); - this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, plasmaCrawlNURL.ce, true); + this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.ce, true); } try { // loop through the list and fill the messageList with url hashs @@ -597,7 +597,7 @@ public final class plasmaCrawlStacker { // deleting old db and creating a new db try {this.urlEntryCache.close();}catch(Exception ex){} cacheFile.delete(); - this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, plasmaCrawlNURL.ce, true); + this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.ce, true); } catch (IOException e) { /* if we have an error, we start with a fresh database */ plasmaCrawlStacker.this.log.logSevere("Unable to initialize crawl stacker queue, IOException:" + e.getMessage() + ". Reseting DB.\n",e); @@ -605,12 +605,12 @@ public final class plasmaCrawlStacker { // deleting old db and creating a new db try {this.urlEntryCache.close();}catch(Exception ex){} cacheFile.delete(); - this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, plasmaCrawlNURL.ce, true); + this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.ce, true); } } else { // create new cache cacheFile.getParentFile().mkdirs(); - this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, plasmaCrawlNURL.ce, true); + this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.ce, true); } } diff --git a/source/de/anomic/plasma/plasmaWordIndexAssortment.java b/source/de/anomic/plasma/plasmaWordIndexAssortment.java index c6b1a4f91..9dd8eb7d0 100644 --- a/source/de/anomic/plasma/plasmaWordIndexAssortment.java +++ b/source/de/anomic/plasma/plasmaWordIndexAssortment.java @@ -109,7 +109,7 @@ public final class plasmaWordIndexAssortment { if (assortmentFile.exists()) { // open existing assortment tree file try { - assortments = new kelondroTree(assortmentFile, bufferSize); + assortments = new kelondroTree(assortmentFile, bufferSize, kelondroTree.defaultObjectCachePercent); if (log != null) log.logConfig("Opened Assortment Database, " + assortments.size() + " entries, width " + assortmentLength + ", " + bufferkb + "kb buffer"); return; } catch (IOException e){ @@ -120,7 +120,7 @@ public final class plasmaWordIndexAssortment { assortmentFile.delete(); // make space for new one } // create new assortment tree file - assortments = new kelondroTree(assortmentFile, bufferSize, bufferStructure(assortmentLength), true); + assortments = new kelondroTree(assortmentFile, bufferSize, kelondroTree.defaultObjectCachePercent, bufferStructure(assortmentLength), true); if (log != null) log.logConfig("Created new Assortment Database, width " + assortmentLength + ", " + bufferkb + "kb buffer"); } @@ -240,7 +240,7 @@ public final class plasmaWordIndexAssortment { if (!(assortmentFile.delete())) throw new RuntimeException("cannot delete assortment database"); } if (assortmentFile.exists()) assortmentFile.delete(); - assortments = new kelondroTree(assortmentFile, bufferSize, bufferStructure(assortmentLength), true); + assortments = new kelondroTree(assortmentFile, bufferSize, kelondroTree.defaultObjectCachePercent, bufferStructure(assortmentLength), true); } public Iterator hashes(String startWordHash, boolean up, boolean rot) throws IOException { diff --git a/source/de/anomic/plasma/plasmaWordIndexEntity.java b/source/de/anomic/plasma/plasmaWordIndexEntity.java index aaae7dc2a..564f9862a 100644 --- a/source/de/anomic/plasma/plasmaWordIndexEntity.java +++ b/source/de/anomic/plasma/plasmaWordIndexEntity.java @@ -87,13 +87,13 @@ public final class plasmaWordIndexEntity { if (cacheSize > 1048576) cacheSize = 1048576; if (theLocation.exists()) try { // open existing index file - kt = new kelondroTree(theLocation, cacheSize); + kt = new kelondroTree(theLocation, cacheSize, kelondroTree.defaultObjectCachePercent); } catch (IOException e) { theLocation.delete(); - kt = new kelondroTree(theLocation, cacheSize, plasmaURL.urlHashLength, plasmaWordIndexEntry.attrSpace, false); + kt = new kelondroTree(theLocation, cacheSize, kelondroTree.defaultObjectCachePercent, plasmaURL.urlHashLength, plasmaWordIndexEntry.attrSpace, false); } else { // create new index file - kt = new kelondroTree(theLocation, cacheSize, plasmaURL.urlHashLength, plasmaWordIndexEntry.attrSpace, false); + kt = new kelondroTree(theLocation, cacheSize, kelondroTree.defaultObjectCachePercent, plasmaURL.urlHashLength, plasmaWordIndexEntry.attrSpace, false); } return kt; // everyone who get this should close it when finished! } diff --git a/source/de/anomic/yacy/yacyNewsDB.java b/source/de/anomic/yacy/yacyNewsDB.java index 22ef9293c..86fe75786 100644 --- a/source/de/anomic/yacy/yacyNewsDB.java +++ b/source/de/anomic/yacy/yacyNewsDB.java @@ -72,7 +72,7 @@ public class yacyNewsDB { this.bufferkb = bufferkb; if (path.exists()) try { - news = new kelondroTree(path, bufferkb * 0x400); + news = new kelondroTree(path, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent); } catch (IOException e) { news = createDB(path, bufferkb); } else { @@ -81,7 +81,7 @@ public class yacyNewsDB { } private static kelondroTree createDB(File path, int bufferkb) { - return new kelondroTree(path, bufferkb * 0x400, new int[] { + return new kelondroTree(path, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new int[] { yacyNewsRecord.idLength(), // id = created + originator yacyNewsRecord.categoryStringLength, // category yacyCore.universalDateShortPattern.length(), // received