boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

small changes to caching

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@747 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 20 years ago
parent effbf35f57
commit e6b30911c3
4 changed files with 139 additions and 60 deletions
Show Stats Download Patch File Download Diff File

159
source/de/anomic/kelondro/kelondroBufferedRA.java
Unescape View File

@ -46,18 +46,29 @@ import java.io.IOException;
public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA {
private static final int bufferSizeExp = 10;
private static final int bufferSize = 1 << bufferSizeExp;
private static final int bufferOffsetFilter = bufferSize - 1;
protected kelondroRA ra;
protected byte[] buffer;
protected int bufferPage;
protected boolean bufferWritten;
private long seekpos;
public kelondroBufferedRA(kelondroRA ra) throws FileNotFoundException {
this.ra = ra;
private int bufferSizeExp;
private int bufferSize;
private int bufferOffsetFilter;
private int bufferStart;
public kelondroBufferedRA(kelondroRA ra, int minBufferSize, int bufferStartMin) throws FileNotFoundException {
// calculate buffer organization parameters
this.bufferSizeExp = 0;
minBufferSize--;
while (minBufferSize > 0) {minBufferSize = minBufferSize >> 1; this.bufferSizeExp++;}
this.bufferSize = 1 << this.bufferSizeExp;
this.bufferOffsetFilter = this.bufferSize - 1;
this.bufferStart = 0;
while (this.bufferStart < bufferStartMin) this.bufferStart += this.bufferSize;
// init buffer
this.ra = ra;
this.name = ra.name();
this.buffer = new byte[bufferSize];
this.seekpos = 0;
@ -73,75 +84,121 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA
bufferWritten = true;
}
/*
private void writeBuffer() throws IOException {
if ((bufferWritten) || (bufferPage < 0)) return;
ra.seek(bufferPage << bufferSizeExp);
ra.write(buffer, 0, bufferSize);
bufferWritten = true;
}
*/
private void updateToBuffer(int newPageNr) throws IOException {
if (newPageNr != bufferPage) {
writeBuffer();
//writeBuffer();
readBuffer(newPageNr);
}
}
// pseudo-native method read
public int read() throws IOException {
int bn = (int) seekpos >> bufferSizeExp; // buffer page number
int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset
seekpos++;
updateToBuffer(bn);
return 0xFF & buffer[offset];
if (seekpos < bufferStart) {
// do not use buffer
ra.seek(seekpos);
int r = ra.read();
seekpos++;
return r;
} else {
int bn = (int) seekpos >> bufferSizeExp; // buffer page number
int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset
seekpos++;
updateToBuffer(bn);
return 0xFF & buffer[offset];
}
}
// pseudo-native method write
public void write(int b) throws IOException {
int bn = (int) seekpos >> bufferSizeExp; // buffer page number
int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset
seekpos++;
updateToBuffer(bn);
buffer[offset] = (byte) b;
bufferWritten = false;
if (seekpos < bufferStart) {
// do not use buffer
ra.seek(seekpos);
ra.write(b);
seekpos++;
} else {
// write to ra direkt
ra.seek(seekpos);
ra.write(b);
// and write also to buffer
int bn = (int) seekpos >> bufferSizeExp; // buffer page number
int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset
updateToBuffer(bn);
buffer[offset] = (byte) b;
bufferWritten = false;
// update seek pos
seekpos++;
}
}
public int read(byte[] b, int off, int len) throws IOException {
int bn1 = (int) seekpos >> bufferSizeExp; // buffer page number, first position
int bn2 = (int) (seekpos + len - 1) >> bufferSizeExp; // buffer page number, last position
int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset
updateToBuffer(bn1);
if (bn1 == bn2) {
// simple case
System.arraycopy(buffer, offset, b, off, len);
if (seekpos < bufferStart) {
// do not use buffer
ra.seek(seekpos);
int r = ra.read(b, off, len);
seekpos += len;
return len;
return r;
} else {
// do recursively
int thislen = bufferSize - offset;
System.arraycopy(buffer, offset, b, off, thislen);
seekpos += thislen;
return thislen + read(b, off + thislen, len - thislen);
int bn1 = (int) seekpos >> bufferSizeExp; // buffer page number, first position
int bn2 = (int) (seekpos + len - 1) >> bufferSizeExp; // buffer page number, last position
int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset
updateToBuffer(bn1);
if (bn1 == bn2) {
// simple case
System.arraycopy(buffer, offset, b, off, len);
seekpos += len;
return len;
} else {
// do recursively
int thislen = bufferSize - offset;
System.arraycopy(buffer, offset, b, off, thislen);
seekpos += thislen;
return thislen + read(b, off + thislen, len - thislen);
}
}
}
public void write(byte[] b, int off, int len) throws IOException {
int bn1 = (int) seekpos >> bufferSizeExp; // buffer page number, first position
int bn2 = (int) (seekpos + len - 1) >> bufferSizeExp; // buffer page number, last position
int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset
updateToBuffer(bn1);
if (bn1 == bn2) {
// simple case
System.arraycopy(b, off, buffer, offset, len);
bufferWritten = false;
if (seekpos < bufferStart) {
// do not use buffer
ra.seek(seekpos);
ra.write(b, off, len);
seekpos += len;
} else {
// do recursively
int thislen = bufferSize - offset;
System.arraycopy(b, off, buffer, offset, thislen);
bufferWritten = false;
seekpos += thislen;
write(b, off + thislen, len - thislen);
int bn1 = (int) seekpos >> bufferSizeExp; // buffer page number, first position
int bn2 = (int) (seekpos + len - 1) >> bufferSizeExp; // buffer page number, last position
int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset
updateToBuffer(bn1);
if (bn1 == bn2) {
// simple case
ra.seek(seekpos);
ra.write(b, off, len);
System.arraycopy(b, off, buffer, offset, len);
bufferWritten = false;
seekpos += len;
} else {
// do recursively
int thislen = bufferSize - offset;
ra.seek(seekpos);
ra.write(b, off, thislen);
System.arraycopy(b, off, buffer, offset, thislen);
bufferWritten = false;
seekpos += thislen;
write(b, off + thislen, len - thislen);
}
}
}
@ -152,24 +209,26 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA
public void close() throws IOException {
// write unwritten buffer
if (buffer == null) return;
writeBuffer();
//writeBuffer();
ra.close();
buffer = null;
}
/*
public void finalize() {
try {
close();
} catch (IOException e) {}
}
*/
public static void main(String[] args) {
try {
kelondroRA file = new kelondroBufferedRA(new kelondroFileRA("testx"));
file.seek(bufferSize - 2);
kelondroRA file = new kelondroBufferedRA(new kelondroFileRA("testx"), 64, 30);
file.seek(1024 - 2);
byte[] b = new byte[]{65, 66, 77, 88};
file.write(b);
file.seek(bufferSize * 2 - 30);
file.seek(1024 * 2 - 30);
for (int i = 65; i < 150; i++) file.write(i);
file.close();
} catch (IOException e) {

21
source/de/anomic/kelondro/kelondroDyn.java
Unescape View File

@ -192,6 +192,7 @@ public class kelondroDyn extends kelondroTree {
if ((segmentCacheKey != null) && (serverByteBuffer.equals(key, segmentCacheKey))) {
// use cache
//System.out.println("cache hit: " + super.filename + "/" + new String(key));
return segmentCacheContent;
} else {
// read from db
@ -217,6 +218,16 @@ public class kelondroDyn extends kelondroTree {
put(key, value);
}
public synchronized int getDyn(String key, int pos) throws IOException {
int reccnt = pos / reclen;
// read within a single record
byte[] buf = getValueCached(dynKey(key, reccnt));
if (buf == null) return -1;
int recpos = pos % reclen;
if (buf.length <= recpos) return -1;
return buf[recpos] & 0xFF;
}
public synchronized byte[] getDyn(String key, int pos, int len) throws IOException {
int recpos = pos % reclen;
int reccnt = pos / reclen;
@ -235,7 +246,7 @@ public class kelondroDyn extends kelondroTree {
buff = null;
}
//System.out.println("read: buf.length="+buf.length+",recpos="+recpos+",len="+len);
if (len < (reclen - recpos)) {
if (recpos + len <= reclen) {
segment1 = new byte[len];
System.arraycopy(buf, recpos, segment1, 0, len);
} else {
@ -316,7 +327,8 @@ public class kelondroDyn extends kelondroTree {
public synchronized kelondroRA getRA(String filekey) throws IOException {
// this returns always a RARecord, even if no existed bevore
return new RARecord(filekey);
//return new kelondroBufferedRA(new RARecord(filekey), 512, 0);
return new RARecord(filekey);
}
public class RARecord extends kelondroAbstractRA implements kelondroRA {
@ -329,8 +341,9 @@ public class kelondroDyn extends kelondroTree {
}
public int read() throws IOException {
byte[] b = getDyn(filekey, seekpos++, 1);
return (b == null) ? -1 : b[0] & 0xFF;
return getDyn(filekey, seekpos++);
//byte[] b = getDyn(filekey, seekpos++, 1);
//return (b == null) ? -1 : b[0] & 0xFF;
}
public void write(int i) throws IOException {

13
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -80,7 +80,7 @@ public class kelondroRecords {
// constants
private static final int NUL = Integer.MIN_VALUE; // the meta value for the kelondroRecords' NUL abstraction
public static final long memBlock = 5000000; // do not fill cache further if the amount of available memory is less that this
public static final long memBlock = 500000; // do not fill cache further if the amount of available memory is less that this
public static final long memKcolb = 10000000; // if the amount of available memory is greater than this, do not use cache size to block, simply use memory
// caching flags
@ -551,8 +551,15 @@ public class kelondroRecords {
//entryFile.read(this.tailChunk, 0, this.tailChunk.length);
}
this.headChanged = true; // provoke a cache store
checkCacheSpace(CP_HIGH);
updateNodeCache(CP_HIGH);
int cp = CP_HIGH;
if (OHHANDLEC == 3) {
Handle l = getOHHandle(1);
Handle r = getOHHandle(2);
if ((l == null) && (r == null)) cp = CP_LOW;
else if ((l == null) || (r == null)) cp = CP_MEDIUM;
}
checkCacheSpace(cp);
updateNodeCache(cp);
} else {
//System.out.println("**CACHE HIT for " + this.handle.index + "**");
// copy cache entry

6
source/de/anomic/kelondro/kelondroTree.java
Unescape View File

@ -1321,8 +1321,8 @@ public class kelondroTree extends kelondroRecords implements Comparator {
public static void main(String[] args) {
//cmd(args);
//bigtest(Integer.parseInt(args[0]));
randomtest(Integer.parseInt(args[0]));
bigtest(Integer.parseInt(args[0]));
//randomtest(Integer.parseInt(args[0]));
//smalltest();
}
@ -1448,7 +1448,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
public static kelondroTree testTree(File f, String testentities) throws IOException {
if (f.exists()) f.delete();
kelondroTree tt = new kelondroTree(f, 200000, 4, 4);
kelondroTree tt = new kelondroTree(f, 0, 4, 4);
byte[] b;
for (int i = 0; i < testentities.length(); i++) {
b = testWord(testentities.charAt(i));

Write Preview
Loading…
Cancel
Save