From 11951aed418e93ef7bb237b17e8f608936faa1ac Mon Sep 17 00:00:00 2001 From: orbiter Date: Sun, 18 Jun 2006 12:04:41 +0000 Subject: [PATCH] replaced node cache Object type in kelondroTree: kelondroCollectionObjectMap replaces HashMap. the kelondroCollectionObjectMap uses only 1/4 of the memory of HashMap, but is slightly slower than the HashMap. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2213 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- .../de/anomic/kelondro/kelondroRecords.java | 122 ++++++------------ source/de/anomic/kelondro/kelondroTree.java | 14 +- 2 files changed, 47 insertions(+), 89 deletions(-) diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java index 2fa1aa99b..2b0eaf736 100644 --- a/source/de/anomic/kelondro/kelondroRecords.java +++ b/source/de/anomic/kelondro/kelondroRecords.java @@ -70,12 +70,10 @@ package de.anomic.kelondro; import java.io.File; import java.io.IOException; -import java.util.HashMap; import java.util.HashSet; import java.util.NoSuchElementException; import java.util.Random; import java.util.StringTokenizer; -import java.util.Map; import java.util.Iterator; import java.util.logging.Logger; @@ -143,7 +141,7 @@ public class kelondroRecords { private int TXTPROPW; // size of a single TXTPROPS element // caching buffer - private HashMap[] cacheHeaders; // the cache; holds overhead values and key element + private kelondroCollectionObjectMap[] cacheHeaders; // the cache; holds overhead values and key element private int cacheSize; // number of cache records private long cacheStartup; // startup time; for cache aging private kelondroMScoreCluster cacheScore; // controls cache aging @@ -428,7 +426,14 @@ public class kelondroRecords { this.cacheSize = (int) (buffersize / cacheNodeChunkSize(true)); this.cacheScore = new kelondroMScoreCluster(); // cache control of CP_HIGH caches } - this.cacheHeaders = new HashMap[]{new HashMap(), new HashMap(), new HashMap()}; + this.cacheHeaders = new kelondroCollectionObjectMap[]{ + new kelondroCollectionObjectMap(this.headchunksize, 0), + new kelondroCollectionObjectMap(this.headchunksize, 0), + new kelondroCollectionObjectMap(this.headchunksize, 0) + }; + this.cacheHeaders[0].setOrdering(kelondroNaturalOrder.naturalOrder); + this.cacheHeaders[1].setOrdering(kelondroNaturalOrder.naturalOrder); + this.cacheHeaders[2].setOrdering(kelondroNaturalOrder.naturalOrder); } this.cacheStartup = System.currentTimeMillis(); this.readHit = 0; @@ -608,7 +613,7 @@ public class kelondroRecords { if (handle == null) throw new kelondroException(filename, "INTERNAL ERROR: node handle is null."); if (handle.index >= USAGE.allCount()) { if (parentNode == null) { - throw new kelondroException(filename, "INTERNAL ERROR, Node/init: node handle index exceeds size. No auto-fix node was submitted. This is a serious failure."); + throw new kelondroException(filename, "INTERNAL ERROR, Node/init: node handle index " + handle.index + " exceeds size. No auto-fix node was submitted. This is a serious failure."); } else { try { parentNode.setOHHandle(referenceInParent, null); @@ -639,13 +644,13 @@ public class kelondroRecords { } else synchronized(cacheHeaders) { byte[] cacheEntry = null; int cp = CP_HIGH; - cacheEntry = (byte[]) cacheHeaders[CP_HIGH].get(this.handle); // first try + cacheEntry = cacheHeaders[CP_HIGH].get(this.handle); // first try if (cacheEntry == null) { - cacheEntry = (byte[]) cacheHeaders[CP_MEDIUM].get(this.handle); // second try + cacheEntry = cacheHeaders[CP_MEDIUM].get(this.handle); // second try cp = CP_MEDIUM; } if (cacheEntry == null) { - cacheEntry = (byte[]) cacheHeaders[CP_LOW].get(this.handle); // third try + cacheEntry = cacheHeaders[CP_LOW].get(this.handle); // third try cp = CP_LOW; } if (cacheEntry == null) { @@ -670,8 +675,9 @@ public class kelondroRecords { // cache hit, copy overhead and key from cache readHit++; //System.out.println("**CACHE HIT for " + this.handle.index + "**"); - this.headChunk = new byte[headchunksize]; - System.arraycopy(cacheEntry, 0, this.headChunk, 0, headchunksize); + //this.headChunk = new byte[headchunksize]; + //System.arraycopy(cacheEntry, 0, this.headChunk, 0, headchunksize); + this.headChunk = cacheEntry; // update cache scores to announce this cache hit if ((cacheScore != null) && (cp == CP_HIGH)) { cacheScore.setScore(this.handle, (int) ((System.currentTimeMillis() - cacheStartup) / 1000)); @@ -729,26 +735,7 @@ public class kelondroRecords { int h = bytes2int(this.headChunk, OHBYTEC + 4 * i); return (h == NUL) ? null : new Handle(h); } - - /* - public byte[][] setValueCells(byte[][] row) throws IOException { - // if the index is defined, then write values directly to the file, else only to the object - byte[][] result = getValueCells(); // previous value (this loads the values if not already happened) - - // set values - if (this.handle.index != NUL) { - setValue(row[0], 0, ROW.width(0), headChunk, overhead); - int offset = 0; - for (int i = 1; i < row.length; i++) { - setValue(row[i], 0, ROW.width(i), tailChunk, offset); - offset +=ROW.width(i); - } - } - this.headChanged = true; - this.tailChanged = true; - return result; // return previous value - } - */ + public byte[] setValueRow(byte[] row) throws IOException { // if the index is defined, then write values directly to the file, else only to the object assert row.length == ROW.objectsize(); @@ -769,32 +756,6 @@ public class kelondroRecords { return trimCopy(headChunk, overhead, ROW.width(0)); } - /* - public byte[][] getValueCells() throws IOException { - - if (this.tailChunk == null) { - // load all values from the database file - this.tailChunk = new byte[tailchunksize]; - // read values - entryFile.readFully(seekpos(this.handle) + headchunksize, this.tailChunk, 0, this.tailChunk.length); - } - - // create return value - byte[][] values = new byte[ROW.columns()][]; - - // read key - values[0] = trimCopy(headChunk, overhead, ROW.width(0)); - - // read remaining values - int offset = 0; - for (int i = 1; i < ROW.columns(); i++) { - values[i] = trimCopy(tailChunk, offset, ROW.width(i)); - offset += ROW.width(i); - } - - return values; - } -*/ public byte[] getValueRow() throws IOException { if (this.tailChunk == null) { @@ -896,15 +857,13 @@ public class kelondroRecords { long cs = cacheHeaders[CP_LOW].size() + cacheHeaders[CP_MEDIUM].size() + cacheHeaders[CP_HIGH].size(); if (cs == 0) return true; // nothing there to flush if ((cs < cacheSize) && (availableMemory() >= memBlock)) return true; // no need to flush cache space - Handle delkey; // delete one entry. distinguish between different priority cases: if (forPriority == CP_LOW) { // remove only from low-priority cache if (cacheHeaders[CP_LOW].size() != 0) { // just delete any of the low-priority entries - delkey = (Handle) cacheHeaders[CP_LOW].keySet().iterator().next(); - cacheHeaders[CP_LOW].remove(delkey); + cacheHeaders[CP_LOW].removeOne(); cacheFlush++; return true; } else { @@ -914,14 +873,12 @@ public class kelondroRecords { } else if (forPriority == CP_MEDIUM) { if (cacheHeaders[CP_LOW].size() != 0) { // just delete any of the low-priority entries - delkey = (Handle) cacheHeaders[CP_LOW].keySet().iterator().next(); - cacheHeaders[CP_LOW].remove(delkey); + cacheHeaders[CP_LOW].removeOne(); cacheFlush++; return true; } else if (cacheHeaders[CP_MEDIUM].size() != 0) { // just delete any of the medium-priority entries - delkey = (Handle) cacheHeaders[CP_MEDIUM].keySet().iterator().next(); - cacheHeaders[CP_MEDIUM].remove(delkey); + cacheHeaders[CP_MEDIUM].removeOne(); cacheFlush++; return true; } else { @@ -932,14 +889,12 @@ public class kelondroRecords { // request for a high-priority entry if (cacheHeaders[CP_LOW].size() != 0) { // just delete any of the low-priority entries - delkey = (Handle) cacheHeaders[CP_LOW].keySet().iterator().next(); - cacheHeaders[CP_LOW].remove(delkey); + cacheHeaders[CP_LOW].removeOne(); cacheFlush++; return true; } else if (cacheHeaders[CP_MEDIUM].size() != 0) { // just delete any of the medium-priority entries - delkey = (Handle) cacheHeaders[CP_MEDIUM].keySet().iterator().next(); - cacheHeaders[CP_MEDIUM].remove(delkey); + cacheHeaders[CP_MEDIUM].removeOne(); cacheFlush++; return true; } else if (cacheScore == null) { @@ -949,7 +904,7 @@ public class kelondroRecords { } else try { // delete one from the high-priority entries // use the cache-control to find the right object - delkey = (Handle) cacheScore.getMinObject(); + Handle delkey = (Handle) cacheScore.getMinObject(); cacheScore.deleteScore(delkey); cacheHeaders[CP_HIGH].remove(delkey); cacheFlush++; @@ -959,9 +914,12 @@ public class kelondroRecords { // we simply clear the cache String error = "cachScore error: " + e.getMessage() + "; cachesize=" + cacheSize + ", cache.size()=[" + cacheHeaders[0].size() + "," + cacheHeaders[1].size() + "," + cacheHeaders[2].size() + "], cacheScore.size()=" + cacheScore.size(); cacheScore = new kelondroMScoreCluster(); - cacheHeaders[CP_LOW] = new HashMap(); - cacheHeaders[CP_MEDIUM] = new HashMap(); - cacheHeaders[CP_HIGH] = new HashMap(); + cacheHeaders[CP_LOW] = new kelondroCollectionObjectMap(headchunksize, 0); + cacheHeaders[CP_MEDIUM] = new kelondroCollectionObjectMap(headchunksize, 0); + cacheHeaders[CP_HIGH] = new kelondroCollectionObjectMap(headchunksize, 0); + cacheHeaders[0].setOrdering(kelondroNaturalOrder.naturalOrder); + cacheHeaders[1].setOrdering(kelondroNaturalOrder.naturalOrder); + cacheHeaders[2].setOrdering(kelondroNaturalOrder.naturalOrder); throw new kelondroException(filename, error); } @@ -978,8 +936,8 @@ public class kelondroRecords { synchronized (cacheHeaders) { // generate cache entry - byte[] cacheEntry = new byte[headchunksize]; - System.arraycopy(headChunk, 0, cacheEntry, 0, headchunksize); + //byte[] cacheEntry = new byte[headchunksize]; + //System.arraycopy(headChunk, 0, cacheEntry, 0, headchunksize); Handle cacheHandle = new Handle(this.handle.index); // store the cache entry @@ -987,14 +945,14 @@ public class kelondroRecords { if (priority != CP_LOW) upd = upd || (cacheHeaders[CP_LOW].remove(cacheHandle) != null); if (priority != CP_MEDIUM) upd = upd || (cacheHeaders[CP_MEDIUM].remove(cacheHandle) != null); if (priority != CP_HIGH) upd = upd || (cacheHeaders[CP_HIGH].remove(cacheHandle) != null); - cacheHeaders[priority].put(cacheHandle, cacheEntry); + cacheHeaders[priority].put(cacheHandle, headChunk); if ((cacheScore != null) && (priority == CP_HIGH)) { cacheScore.setScore(cacheHandle, (int) ((System.currentTimeMillis() - cacheStartup) / 1000)); } if (upd) writeDouble++; else writeUnique++; // delete the cache entry buffer - cacheEntry = null; + //cacheEntry = null; cacheHandle = null; //System.out.println("kelondroRecords cache4" + filename + ": cache record size = " + (memBefore - Runtime.getRuntime().freeMemory()) + " bytes" + ((newentry) ? " new" : "")); //printCache(); @@ -1019,23 +977,24 @@ public class kelondroRecords { } else { System.out.println("### cache report: [" + cacheHeaders[0].size() + "," + cacheHeaders[0].size() + "," + cacheHeaders[0].size() + "] entries"); for (int cp = 0; cp < 3; cp++) { - Iterator i = cacheHeaders[cp].entrySet().iterator(); - Map.Entry entry; + Iterator i = cacheHeaders[cp].elements(); + byte[] entry; while (i.hasNext()) { - entry = (Map.Entry) i.next(); + entry = (byte[]) i.next(); // print from cache System.out.print("#C " + cp + " "); - printChunk((Handle) entry.getKey(), (byte[]) entry.getValue()); + printChunk((byte[]) entry); System.out.println(); // print from file to compare + /* System.out.print("#F " + cp + " " + ((Handle) entry.getKey()).index + ": "); try { for (int j = 0; j < headchunksize; j++) System.out.print(entryFile.readByte(j + seekpos((Handle) entry.getKey())) + ","); } catch (IOException e) {} - + */ System.out.println(); } } @@ -1043,8 +1002,7 @@ public class kelondroRecords { System.out.println("### end report"); } - private void printChunk(Handle handle, byte[] chunk) { - System.out.print(handle.index + ": "); + private void printChunk(byte[] chunk) { for (int j = 0; j < chunk.length; j++) System.out.print(chunk[j] + ","); } diff --git a/source/de/anomic/kelondro/kelondroTree.java b/source/de/anomic/kelondro/kelondroTree.java index 284ec3156..c5fec8b8c 100644 --- a/source/de/anomic/kelondro/kelondroTree.java +++ b/source/de/anomic/kelondro/kelondroTree.java @@ -1398,10 +1398,10 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { public static void main(String[] args) { //cmd(args); - iterationtest(); + //iterationtest(); //bigtest(Integer.parseInt(args[0])); //randomtest(Integer.parseInt(args[0])); - //smalltest(); + smalltest(); } public static String[] permutations(int letters) { @@ -1507,7 +1507,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { File f = new File("test.db"); if (f.exists()) f.delete(); try { - kelondroTree tt = new kelondroTree(f, 0, 10, 4, 4, true); + kelondroTree tt = new kelondroTree(f, 1000, 10, 4, 4, true); byte[] b; b = testWord('B'); tt.put(b, b); //tt.print(); b = testWord('C'); tt.put(b, b); //tt.print(); @@ -1520,7 +1520,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { b = testWord('E'); tt.put(b, b); b = testWord('F'); tt.put(b, b); b = testWord('G'); tt.put(b, b); - //b = testWord('H'); tt.put(b, b); + b = testWord('H'); tt.put(b, b); b = testWord('I'); tt.put(b, b); b = testWord('J'); tt.put(b, b); b = testWord('K'); tt.put(b, b); @@ -1529,7 +1529,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { System.out.println("elements: " + c); Iterator i = tt.rows(true, true, testWord('G')); for (int j = 0; j < c; j++) { - System.out.println("Row " + j + ": " + new String(((byte[][]) i.next())[0])); + System.out.println("Row " + j + ": " + new String(((kelondroRow.Entry) i.next()).getColBytes(0))); } System.out.println("TERMINATED"); } catch (IOException e) { @@ -1636,10 +1636,10 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { int count = 0; try { Iterator iter = t.rows(true, false, null); - byte[][] row; + kelondroRow.Entry row; while (iter.hasNext()) { count++; - row = (byte[][]) iter.next(); + row = (kelondroRow.Entry) iter.next(); if (row == null) System.out.println("ERROR! null element found"); // else System.out.println("counted element: " + new // String(n.getKey()));