From e6b30911c33515880194a77588f768298f428293 Mon Sep 17 00:00:00 2001 From: orbiter Date: Mon, 19 Sep 2005 15:13:12 +0000 Subject: [PATCH] small changes to caching git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@747 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- .../anomic/kelondro/kelondroBufferedRA.java | 159 ++++++++++++------ source/de/anomic/kelondro/kelondroDyn.java | 21 ++- .../de/anomic/kelondro/kelondroRecords.java | 13 +- source/de/anomic/kelondro/kelondroTree.java | 6 +- 4 files changed, 139 insertions(+), 60 deletions(-) diff --git a/source/de/anomic/kelondro/kelondroBufferedRA.java b/source/de/anomic/kelondro/kelondroBufferedRA.java index 1a5861837..b7c903b25 100644 --- a/source/de/anomic/kelondro/kelondroBufferedRA.java +++ b/source/de/anomic/kelondro/kelondroBufferedRA.java @@ -46,18 +46,29 @@ import java.io.IOException; public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA { - private static final int bufferSizeExp = 10; - private static final int bufferSize = 1 << bufferSizeExp; - private static final int bufferOffsetFilter = bufferSize - 1; - protected kelondroRA ra; protected byte[] buffer; protected int bufferPage; protected boolean bufferWritten; private long seekpos; - public kelondroBufferedRA(kelondroRA ra) throws FileNotFoundException { - this.ra = ra; + private int bufferSizeExp; + private int bufferSize; + private int bufferOffsetFilter; + private int bufferStart; + + public kelondroBufferedRA(kelondroRA ra, int minBufferSize, int bufferStartMin) throws FileNotFoundException { + // calculate buffer organization parameters + this.bufferSizeExp = 0; + minBufferSize--; + while (minBufferSize > 0) {minBufferSize = minBufferSize >> 1; this.bufferSizeExp++;} + this.bufferSize = 1 << this.bufferSizeExp; + this.bufferOffsetFilter = this.bufferSize - 1; + this.bufferStart = 0; + while (this.bufferStart < bufferStartMin) this.bufferStart += this.bufferSize; + + // init buffer + this.ra = ra; this.name = ra.name(); this.buffer = new byte[bufferSize]; this.seekpos = 0; @@ -73,75 +84,121 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA bufferWritten = true; } + /* private void writeBuffer() throws IOException { if ((bufferWritten) || (bufferPage < 0)) return; ra.seek(bufferPage << bufferSizeExp); ra.write(buffer, 0, bufferSize); bufferWritten = true; } + */ private void updateToBuffer(int newPageNr) throws IOException { if (newPageNr != bufferPage) { - writeBuffer(); + //writeBuffer(); readBuffer(newPageNr); } } // pseudo-native method read public int read() throws IOException { - int bn = (int) seekpos >> bufferSizeExp; // buffer page number - int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset - seekpos++; - updateToBuffer(bn); - return 0xFF & buffer[offset]; + if (seekpos < bufferStart) { + // do not use buffer + ra.seek(seekpos); + int r = ra.read(); + seekpos++; + return r; + } else { + int bn = (int) seekpos >> bufferSizeExp; // buffer page number + int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset + seekpos++; + updateToBuffer(bn); + return 0xFF & buffer[offset]; + } } // pseudo-native method write public void write(int b) throws IOException { - int bn = (int) seekpos >> bufferSizeExp; // buffer page number - int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset - seekpos++; - updateToBuffer(bn); - buffer[offset] = (byte) b; - bufferWritten = false; + if (seekpos < bufferStart) { + // do not use buffer + ra.seek(seekpos); + ra.write(b); + seekpos++; + } else { + // write to ra direkt + ra.seek(seekpos); + ra.write(b); + + // and write also to buffer + int bn = (int) seekpos >> bufferSizeExp; // buffer page number + int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset + updateToBuffer(bn); + buffer[offset] = (byte) b; + bufferWritten = false; + + // update seek pos + seekpos++; + } } public int read(byte[] b, int off, int len) throws IOException { - int bn1 = (int) seekpos >> bufferSizeExp; // buffer page number, first position - int bn2 = (int) (seekpos + len - 1) >> bufferSizeExp; // buffer page number, last position - int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset - updateToBuffer(bn1); - if (bn1 == bn2) { - // simple case - System.arraycopy(buffer, offset, b, off, len); + if (seekpos < bufferStart) { + // do not use buffer + ra.seek(seekpos); + int r = ra.read(b, off, len); seekpos += len; - return len; + return r; } else { - // do recursively - int thislen = bufferSize - offset; - System.arraycopy(buffer, offset, b, off, thislen); - seekpos += thislen; - return thislen + read(b, off + thislen, len - thislen); + int bn1 = (int) seekpos >> bufferSizeExp; // buffer page number, first position + int bn2 = (int) (seekpos + len - 1) >> bufferSizeExp; // buffer page number, last position + int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset + updateToBuffer(bn1); + if (bn1 == bn2) { + // simple case + System.arraycopy(buffer, offset, b, off, len); + seekpos += len; + return len; + } else { + // do recursively + int thislen = bufferSize - offset; + System.arraycopy(buffer, offset, b, off, thislen); + seekpos += thislen; + return thislen + read(b, off + thislen, len - thislen); + } } } public void write(byte[] b, int off, int len) throws IOException { - int bn1 = (int) seekpos >> bufferSizeExp; // buffer page number, first position - int bn2 = (int) (seekpos + len - 1) >> bufferSizeExp; // buffer page number, last position - int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset - updateToBuffer(bn1); - if (bn1 == bn2) { - // simple case - System.arraycopy(b, off, buffer, offset, len); - bufferWritten = false; + if (seekpos < bufferStart) { + // do not use buffer + ra.seek(seekpos); + ra.write(b, off, len); seekpos += len; } else { - // do recursively - int thislen = bufferSize - offset; - System.arraycopy(b, off, buffer, offset, thislen); - bufferWritten = false; - seekpos += thislen; - write(b, off + thislen, len - thislen); + int bn1 = (int) seekpos >> bufferSizeExp; // buffer page number, first position + int bn2 = (int) (seekpos + len - 1) >> bufferSizeExp; // buffer page number, last position + int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset + updateToBuffer(bn1); + if (bn1 == bn2) { + // simple case + ra.seek(seekpos); + ra.write(b, off, len); + + System.arraycopy(b, off, buffer, offset, len); + bufferWritten = false; + seekpos += len; + } else { + // do recursively + int thislen = bufferSize - offset; + + ra.seek(seekpos); + ra.write(b, off, thislen); + + System.arraycopy(b, off, buffer, offset, thislen); + bufferWritten = false; + seekpos += thislen; + write(b, off + thislen, len - thislen); + } } } @@ -152,24 +209,26 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA public void close() throws IOException { // write unwritten buffer if (buffer == null) return; - writeBuffer(); + //writeBuffer(); ra.close(); buffer = null; } + /* public void finalize() { try { close(); } catch (IOException e) {} } - + */ + public static void main(String[] args) { try { - kelondroRA file = new kelondroBufferedRA(new kelondroFileRA("testx")); - file.seek(bufferSize - 2); + kelondroRA file = new kelondroBufferedRA(new kelondroFileRA("testx"), 64, 30); + file.seek(1024 - 2); byte[] b = new byte[]{65, 66, 77, 88}; file.write(b); - file.seek(bufferSize * 2 - 30); + file.seek(1024 * 2 - 30); for (int i = 65; i < 150; i++) file.write(i); file.close(); } catch (IOException e) { diff --git a/source/de/anomic/kelondro/kelondroDyn.java b/source/de/anomic/kelondro/kelondroDyn.java index 682483a4f..bcd9fc649 100644 --- a/source/de/anomic/kelondro/kelondroDyn.java +++ b/source/de/anomic/kelondro/kelondroDyn.java @@ -192,6 +192,7 @@ public class kelondroDyn extends kelondroTree { if ((segmentCacheKey != null) && (serverByteBuffer.equals(key, segmentCacheKey))) { // use cache + //System.out.println("cache hit: " + super.filename + "/" + new String(key)); return segmentCacheContent; } else { // read from db @@ -217,6 +218,16 @@ public class kelondroDyn extends kelondroTree { put(key, value); } + public synchronized int getDyn(String key, int pos) throws IOException { + int reccnt = pos / reclen; + // read within a single record + byte[] buf = getValueCached(dynKey(key, reccnt)); + if (buf == null) return -1; + int recpos = pos % reclen; + if (buf.length <= recpos) return -1; + return buf[recpos] & 0xFF; + } + public synchronized byte[] getDyn(String key, int pos, int len) throws IOException { int recpos = pos % reclen; int reccnt = pos / reclen; @@ -235,7 +246,7 @@ public class kelondroDyn extends kelondroTree { buff = null; } //System.out.println("read: buf.length="+buf.length+",recpos="+recpos+",len="+len); - if (len < (reclen - recpos)) { + if (recpos + len <= reclen) { segment1 = new byte[len]; System.arraycopy(buf, recpos, segment1, 0, len); } else { @@ -316,7 +327,8 @@ public class kelondroDyn extends kelondroTree { public synchronized kelondroRA getRA(String filekey) throws IOException { // this returns always a RARecord, even if no existed bevore - return new RARecord(filekey); + //return new kelondroBufferedRA(new RARecord(filekey), 512, 0); + return new RARecord(filekey); } public class RARecord extends kelondroAbstractRA implements kelondroRA { @@ -329,8 +341,9 @@ public class kelondroDyn extends kelondroTree { } public int read() throws IOException { - byte[] b = getDyn(filekey, seekpos++, 1); - return (b == null) ? -1 : b[0] & 0xFF; + return getDyn(filekey, seekpos++); + //byte[] b = getDyn(filekey, seekpos++, 1); + //return (b == null) ? -1 : b[0] & 0xFF; } public void write(int i) throws IOException { diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java index c48a7a8e7..ea9d42466 100644 --- a/source/de/anomic/kelondro/kelondroRecords.java +++ b/source/de/anomic/kelondro/kelondroRecords.java @@ -80,7 +80,7 @@ public class kelondroRecords { // constants private static final int NUL = Integer.MIN_VALUE; // the meta value for the kelondroRecords' NUL abstraction - public static final long memBlock = 5000000; // do not fill cache further if the amount of available memory is less that this + public static final long memBlock = 500000; // do not fill cache further if the amount of available memory is less that this public static final long memKcolb = 10000000; // if the amount of available memory is greater than this, do not use cache size to block, simply use memory // caching flags @@ -551,8 +551,15 @@ public class kelondroRecords { //entryFile.read(this.tailChunk, 0, this.tailChunk.length); } this.headChanged = true; // provoke a cache store - checkCacheSpace(CP_HIGH); - updateNodeCache(CP_HIGH); + int cp = CP_HIGH; + if (OHHANDLEC == 3) { + Handle l = getOHHandle(1); + Handle r = getOHHandle(2); + if ((l == null) && (r == null)) cp = CP_LOW; + else if ((l == null) || (r == null)) cp = CP_MEDIUM; + } + checkCacheSpace(cp); + updateNodeCache(cp); } else { //System.out.println("**CACHE HIT for " + this.handle.index + "**"); // copy cache entry diff --git a/source/de/anomic/kelondro/kelondroTree.java b/source/de/anomic/kelondro/kelondroTree.java index 8d949f7bf..8a9e8b773 100644 --- a/source/de/anomic/kelondro/kelondroTree.java +++ b/source/de/anomic/kelondro/kelondroTree.java @@ -1321,8 +1321,8 @@ public class kelondroTree extends kelondroRecords implements Comparator { public static void main(String[] args) { //cmd(args); - //bigtest(Integer.parseInt(args[0])); - randomtest(Integer.parseInt(args[0])); + bigtest(Integer.parseInt(args[0])); + //randomtest(Integer.parseInt(args[0])); //smalltest(); } @@ -1448,7 +1448,7 @@ public class kelondroTree extends kelondroRecords implements Comparator { public static kelondroTree testTree(File f, String testentities) throws IOException { if (f.exists()) f.delete(); - kelondroTree tt = new kelondroTree(f, 200000, 4, 4); + kelondroTree tt = new kelondroTree(f, 0, 4, 4); byte[] b; for (int i = 0; i < testentities.length(); i++) { b = testWord(testentities.charAt(i));