added very experimental object cache for all kelondroTree

databases. This cache now takes up 10% of all cache sizes,
90% goes to the node routing cache we had so far.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2069 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/kelondro/kelondroObjectCache.java

@@ -0,0 +1,186 @@
+// kelondroObjectCache.java
+// ------------------------
+// (C) by Michael Peter Christen; mc@anomic.de
+// first published on http://www.anomic.de
+// Frankfurt, Germany, 2006
+//
+// This is a part of the kelondro database, which is a part of YaCy
+//
+// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
+// $LastChangedRevision: 1986 $
+// $LastChangedBy: orbiter $
+//
+//
+// LICENSE
+// 
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+//
+//
+// A NOTE FROM THE AUTHOR TO THE USERS:
+//
+// Using this software in any meaning (reading, learning, copying, compiling,
+// running) means that you agree that the Author(s) is (are) not responsible
+// for cost, loss of data or any harm that may be caused directly or indirectly
+// by usage of this softare or this documentation. The usage of this software
+// is on your own risk. The installation and usage (starting/running) of this
+// software may allow other people or application to access your computer and
+// any attached devices and is highly dependent on the configuration of the
+// software which must be done by the user of the software; the author(s) is
+// (are) also not responsible for proper configuration and usage of the
+// software, even if provoked by documentation provided together with
+// the software.
+//
+// 
+// A NOTE FROM THE AUTHOR TO DEVELOPERS:
+//
+// Contributions and changes to the program code should be marked as such:
+// Please enter your own (C) notice below; they must be compatible with the GPL.
+// Please mark also all changes in the code; if you don't mark them then they
+// can't be identified; thus all unmarked code belong to the copyright holder
+// as mentioned above. A good documentation of code authorities will also help
+// to maintain the code and the project.
+// A re-distribution must contain the intact and unchanged copyright statement.
+
+
+package de.anomic.kelondro;
+
+import java.util.TreeMap;
+
+public class kelondroObjectCache {
+
+    private final TreeMap cache;
+    private final kelondroMScoreCluster ages;
+    private long  startTime;
+    private int   maxSize;
+    private long  maxAge;
+    private long  minMem;
+    private int   readHit, readMiss, writeUnique, writeDouble;
+    
+    public kelondroObjectCache(int maxSize, long maxAge, long minMem) {
+        this.cache = new TreeMap();
+        this.ages  = new kelondroMScoreCluster();
+        this.startTime = System.currentTimeMillis();
+        this.maxAge = maxAge;
+        this.minMem = minMem;
+        this.readHit = 0;
+        this.readMiss = 0;
+        this.writeUnique = 0;
+        this.writeDouble = 0;
+    }
+
+    public void setMaxAge(long maxAge) {
+        this.maxAge = maxAge;
+    }
+    
+    public void setMaxSize(int maxSize) {
+        this.maxSize = maxSize;
+    }
+    
+    public void setMinMem(int minMem) {
+        this.minMem = minMem;
+    }
+    
+    public long minAge() {
+        if (ages.size() == 0) return 0;
+        return System.currentTimeMillis() - longEmit(ages.getMaxScore());
+    }
+    
+    public long maxAge() {
+        if (ages.size() == 0) return 0;
+        return System.currentTimeMillis() - longEmit(ages.getMinScore());
+    }
+    
+    public int size() {
+        return cache.size();
+    }
+    
+    private int intTime(long longTime) {
+        return (int) Math.max(0, ((longTime - startTime) / 1000));
+    }
+
+    private long longEmit(int intTime) {
+        return (((long) intTime) * (long) 1000) + startTime;
+    }
+    
+    public void put(byte[] key, Object value) {
+        if (key != null) put(new String(key), value);
+    }
+    
+    public void put(String key, Object value) {
+        if (key == null) return;
+        Object prev = null;
+        synchronized(cache) {
+            prev = cache.put(key, value);
+            ages.setScore(key, intTime(System.currentTimeMillis()));
+        }
+        if (prev == null) this.writeUnique++; else this.writeDouble++;
+        flush();
+    }
+    
+    public Object get(byte[] key) {
+        if (key == null) return null;
+        Object r = cache.get(new String(key));
+        flush();
+        if (r == null) this.readMiss++; else this.readHit++;
+        return r;
+    }
+    
+    public Object get(String key) {
+        if (key == null) return null;
+        Object r =  cache.get(key);
+        flush();
+        if (r == null) this.readMiss++; else this.readHit++;
+        return r;
+    }
+    
+    public void remove(byte[] key) {
+        remove(new String(key));
+    }
+    
+    public void remove(String key) {
+        if (key == null) return;
+        synchronized(cache) {
+            cache.remove(key);
+            ages.deleteScore(key);
+        }
+        flush();
+    }
+    
+    public void flush() {
+        String k;
+        synchronized(cache) {
+            while ((ages.size() > 0) &&
+                   ((size() > maxSize) ||
+                   (longEmit(ages.getMinScore()) > maxAge) ||
+                   (Runtime.getRuntime().freeMemory() < minMem)) &&
+                  ((k = bestFlush()) != null)) {
+                cache.remove(k);
+                ages.deleteScore(k);
+                if (Runtime.getRuntime().freeMemory() < minMem) System.gc(); // prevent unnecessary loops
+            }
+        }
+    }
+    
+    public String bestFlush() {
+        if (cache.size() == 0) return null;
+        try {
+            synchronized (cache) {
+                return (String) ages.getMinObject(); // flush oldest entries
+            }
+        } catch (Exception e) {}
+        return null;
+    }
+    
+}

source/de/anomic/kelondro/kelondroRecords.java

@@ -127,6 +127,7 @@ public class kelondroRecords {
     private   int        headchunksize;// overheadsize + key element column size
     private   int        tailchunksize;// sum(all: COLWIDTHS) minus the size of the key element colum
     private   int        recordsize;   // (overhead + sum(all: COLWIDTHS)) = the overall size of a record
+    protected int        objectsize;   // sum(all: COLWIDTHS)) = the size of all data fields
 
     // dynamic run-time seek pointers
     private long POS_HANDLES = 0; // starts after end of POS_COLWIDHS which is POS_COLWIDTHS + COLWIDTHS.length * 4
@@ -247,8 +248,9 @@ public class kelondroRecords {
         
         // store dynamic run-time data
         this.overhead = ohbytec + 4 * ohhandlec;
-        this.recordsize = this.overhead;
+        this.objectsize = 0;
-        for (int i = 0; i < columns.length; i++) this.recordsize += columns[i];
+        for (int i = 0; i < columns.length; i++) this.objectsize += columns[i];
+        this.recordsize = this.overhead + this.objectsize;
         this.headchunksize = overhead + columns[0];
         this.tailchunksize = this.recordsize - this.headchunksize;
 
@@ -412,7 +414,9 @@ public class kelondroRecords {
         // assign remaining values that are only present at run-time
         this.overhead = OHBYTEC + 4 * OHHANDLEC;
         this.recordsize = this.overhead;
-        for (int i = 0; i < COLWIDTHS.length; i++) this.recordsize += COLWIDTHS[i];
+        this.objectsize = 0;
+        for (int i = 0; i < COLWIDTHS.length; i++) this.objectsize += COLWIDTHS[i];
+        this.recordsize = this.overhead + this.objectsize;
         this.headchunksize = this.overhead + COLWIDTHS[0];
         this.tailchunksize = this.recordsize - this.headchunksize;
     }

source/de/anomic/kelondro/kelondroTree.java

@@ -80,11 +80,17 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
 
     private static int root       = 0; // pointer for FHandles-array: pointer to root node
 
+    // calibration of cache
+    private static int nodeCachePercent   = 90;
+    private static int objectCachePercent = 10;
+    
+    // class variables
     private Search writeSearchObj = new Search();
     protected kelondroOrder objectOrder = new kelondroNaturalOrder(true);
     private final kelondroOrder loopDetectionOrder = new kelondroNaturalOrder(true);
     private int readAheadChunkSize = 100;
     private long lastIteratorCount = readAheadChunkSize;
+    private kelondroObjectCache objectCache;
     
     public kelondroTree(File file, long buffersize, int key, int value, boolean exitOnFail) {
         this(file, buffersize, new int[] { key, value }, new kelondroNaturalOrder(true), 1, 8, exitOnFail);
@@ -101,7 +107,10 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
 
     public kelondroTree(File file, long buffersize, int[] columns, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) {
         // this creates a new tree file
-        super(file, buffersize, thisOHBytes, thisOHHandles, columns, thisFHandles, txtProps, txtPropsWidth, exitOnFail);
+        super(file,
+              nodeCachePercent * buffersize / (nodeCachePercent + objectCachePercent),
+              thisOHBytes, thisOHHandles, columns,
+              thisFHandles, txtProps, txtPropsWidth, exitOnFail);
         try {
             setHandle(root, null); // define the root value
         } catch (IOException e) {
@@ -112,6 +121,9 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
         this.objectOrder = objectOrder;
         writeOrderType();
         super.setLogger(log);
+        long objectbuffersize = objectCachePercent * buffersize / (nodeCachePercent + objectCachePercent);
+        long nodecachesize = objectbuffersize / (super.objectsize + 8 * columns.length);
+        this.objectCache = new kelondroObjectCache((int) nodecachesize, nodecachesize * 300 , 4*1024*1024);
     }
     
     public kelondroTree(kelondroRA ra, long buffersize, int[] columns, boolean exitOnFail) {
@@ -121,7 +133,9 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
 
     public kelondroTree(kelondroRA ra, long buffersize, int[] columns, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) {
         // this creates a new tree within a kelondroRA
-        super(ra, buffersize, thisOHBytes, thisOHHandles, columns,
+        super(ra,
+              nodeCachePercent * buffersize / (nodeCachePercent + objectCachePercent),
+              thisOHBytes, thisOHHandles, columns,
               thisFHandles, txtProps, txtPropsWidth, exitOnFail);
         try {
             setHandle(root, null); // define the root value
@@ -133,20 +147,29 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
         this.objectOrder = objectOrder;
         writeOrderType();
         super.setLogger(log);
+        long objectbuffersize = objectCachePercent * buffersize / (nodeCachePercent + objectCachePercent);
+        long nodecachesize = objectbuffersize / (super.objectsize + 8 * columns.length);
+        this.objectCache = new kelondroObjectCache((int) nodecachesize, nodecachesize * 300 , 4*1024*1024);
     }
 
     public kelondroTree(File file, long buffersize) throws IOException {
         // this opens a file with an existing tree file
-        super(file, buffersize);
+        super(file, nodeCachePercent * buffersize / (nodeCachePercent + objectCachePercent));
         readOrderType();
         super.setLogger(log);
+        long objectbuffersize = objectCachePercent * buffersize / (nodeCachePercent + objectCachePercent);
+        long nodecachesize = objectbuffersize / (super.objectsize + 8 * super.columns());
+        this.objectCache = new kelondroObjectCache((int) nodecachesize, nodecachesize * 300 , 4*1024*1024);
     }
 
     public kelondroTree(kelondroRA ra, long buffersize) throws IOException {
         // this opens a file with an existing tree in a kelondroRA
-        super(ra, buffersize);
+        super(ra, nodeCachePercent * buffersize / (nodeCachePercent + objectCachePercent));
         readOrderType();
         super.setLogger(log);
+        long objectbuffersize = objectCachePercent * buffersize / (nodeCachePercent + objectCachePercent);
+        long nodecachesize = objectbuffersize / (super.objectsize + 8 * super.columns());
+        this.objectCache = new kelondroObjectCache((int) nodecachesize, nodecachesize * 300 , 4*1024*1024);
     }
 
     private void writeOrderType() {
@@ -193,7 +216,8 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
     // Returns the value to which this map maps the specified key.
     public byte[][] get(byte[] key) throws IOException {
         // System.out.println("kelondroTree.get " + new String(key) + " in " + filename);
-        byte[][] result = null;
+        byte[][] result = (byte[][]) objectCache.get(key);
+        if (result != null) return result;
         // writeLock.stay(2000, 1000);
         synchronized (writeSearchObj) {
             writeSearchObj.process(key);
@@ -202,6 +226,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
             } else {
                 result = null;
             }
+            objectCache.put(key, result);
         }
         // writeLock.release();
         return result;
@@ -378,6 +403,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
         if (newrow.length != columns()) throw new IllegalArgumentException("put: wrong row length " + newrow.length + "; must be " + columns());
         // first try to find the key element in the database
         synchronized(writeSearchObj) {
+            objectCache.put(newrow[0], newrow);
             writeSearchObj.process(newrow[0]);
             if (writeSearchObj.found()) {
                 // a node with this key exist. simply overwrite the content and return old content
@@ -642,6 +668,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
     // Removes the mapping for this key from this map if present (optional operation).
     public byte[][] remove(byte[] key) throws IOException {
         synchronized(writeSearchObj) {
+            objectCache.remove(key);
             writeSearchObj.process(key);
             if (writeSearchObj.found()) {
                 Node result = writeSearchObj.getMatcher();