boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added pre-load of node cache entries to kelondroRecords

Browse Source 
this gives the kelondroTree data structure a similar start-up
behaviour like the kelondroFlexTable: the cache is filled with
routing data in such a way that is more performant than
reading node records during normal operation.
The pre-load phase stops automatically after a time-out of 500 milliseconds
of if the cache is full.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2270 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent 31b5d8980e
commit 3b69b35bf2
1 changed files with 33 additions and 19 deletions
Show Stats Download Patch File Download Diff File

50
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -82,6 +82,7 @@ public class kelondroRecords {
private static final int NUL = Integer.MIN_VALUE; // the meta value for the kelondroRecords' NUL abstraction private static final int NUL = Integer.MIN_VALUE; // the meta value for the kelondroRecords' NUL abstraction
private static final long memBlock = 500000; // do not fill cache further if the amount of available memory is less that this private static final long memBlock = 500000; // do not fill cache further if the amount of available memory is less that this
public final static boolean useWriteBuffer = false; public final static boolean useWriteBuffer = false;
public final static long preloadCacheTime = 500; // time that can be wasted to initialize the node cache
// memory calculation // memory calculation
private static final int element_in_cache = 4; // for kelondroCollectionObjectMap: 4; for HashMap: 52 private static final int element_in_cache = 4; // for kelondroCollectionObjectMap: 4; for HashMap: 52
@ -202,7 +203,7 @@ public class kelondroRecords {
kelondroRA raf = new kelondroFileRA(this.filename); kelondroRA raf = new kelondroFileRA(this.filename);
// kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100); // kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100);
// kelondroRA raf = new kelondroNIOFileRA(this.filename, false, 10000); // kelondroRA raf = new kelondroNIOFileRA(this.filename, false, 10000);
init(raf, ohbytec, ohhandlec, rowdef, FHandles, txtProps, txtPropWidth, buffersize / 10); initNewFile(raf, ohbytec, ohhandlec, rowdef, FHandles, txtProps, txtPropWidth, buffersize / 10);
} catch (IOException e) { } catch (IOException e) {
logFailure("cannot create / " + e.getMessage()); logFailure("cannot create / " + e.getMessage());
if (exitOnFail) if (exitOnFail)
@ -217,7 +218,7 @@ public class kelondroRecords {
boolean exitOnFail) { boolean exitOnFail) {
this.filename = null; this.filename = null;
try { try {
init(ra, ohbytec, ohhandlec, rowdef, FHandles, txtProps, txtPropWidth, buffersize / 10); initNewFile(ra, ohbytec, ohhandlec, rowdef, FHandles, txtProps, txtPropWidth, buffersize / 10);
} catch (IOException e) { } catch (IOException e) {
logFailure("cannot create / " + e.getMessage()); logFailure("cannot create / " + e.getMessage());
if (exitOnFail) System.exit(-1); if (exitOnFail) System.exit(-1);
@ -225,7 +226,7 @@ public class kelondroRecords {
initCache(buffersize / 10 * 9); initCache(buffersize / 10 * 9);
} }
private void init(kelondroRA ra, short ohbytec, short ohhandlec, private void initNewFile(kelondroRA ra, short ohbytec, short ohhandlec,
kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth, long writeBufferSize) throws IOException { kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth, long writeBufferSize) throws IOException {
// create new Chunked IO // create new Chunked IO
@ -313,23 +314,23 @@ public class kelondroRecords {
public void logWarning(String message) { public void logWarning(String message) {
if (this.theLogger == null) if (this.theLogger == null)
System.err.println("KELONDRO WARNING for file " + this.filename + ": " + message); System.err.println("KELONDRO WARNING " + this.filename + ": " + message);
else else
this.theLogger.warning("KELONDRO WARNING for file " + this.filename + ": " + message); this.theLogger.warning("KELONDRO WARNING " + this.filename + ": " + message);
} }
public void logFailure(String message) { public void logFailure(String message) {
if (this.theLogger == null) if (this.theLogger == null)
System.err.println("KELONDRO FAILURE for file " + this.filename + ": " + message); System.err.println("KELONDRO FAILURE " + this.filename + ": " + message);
else else
this.theLogger.severe("KELONDRO FAILURE for file " + this.filename + ": " + message); this.theLogger.severe("KELONDRO FAILURE " + this.filename + ": " + message);
} }
public void logFine(String message) { public void logFine(String message) {
if (this.theLogger == null) if (this.theLogger == null)
System.out.println("KELONDRO DEBUG for file " + this.filename + ": " + message); System.out.println("KELONDRO DEBUG " + this.filename + ": " + message);
else else
this.theLogger.fine("KELONDRO DEBUG for file " + this.filename + ": " + message); this.theLogger.fine("KELONDRO DEBUG " + this.filename + ": " + message);
} }
public void clear() throws IOException { public void clear() throws IOException {
@ -351,17 +352,17 @@ public class kelondroRecords {
//kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100); //kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100);
//kelondroRA raf = new kelondroCachedRA(new kelondroFileRA(this.filename), 5000000, 1000); //kelondroRA raf = new kelondroCachedRA(new kelondroFileRA(this.filename), 5000000, 1000);
//kelondroRA raf = new kelondroNIOFileRA(this.filename, (file.length() < 4000000), 10000); //kelondroRA raf = new kelondroNIOFileRA(this.filename, (file.length() < 4000000), 10000);
init(raf, buffersize / 10); initExistingFile(raf, buffersize / 10);
initCache(buffersize / 10 * 9); initCache(buffersize / 10 * 9);
} }
public kelondroRecords(kelondroRA ra, long buffersize) throws IOException{ public kelondroRecords(kelondroRA ra, long buffersize) throws IOException{
this.filename = null; this.filename = null;
init(ra, buffersize / 10); initExistingFile(ra, buffersize / 10);
initCache(buffersize / 10 * 9); initCache(buffersize / 10 * 9);
} }
private void init(kelondroRA ra, long writeBufferSize) throws IOException { private void initExistingFile(kelondroRA ra, long writeBufferSize) throws IOException {
// read from Chunked IO // read from Chunked IO
if (useWriteBuffer) { if (useWriteBuffer) {
this.entryFile = new kelondroBufferedIOChunks(ra, ra.name(), writeBufferSize, 30000 + random.nextLong() % 30000); this.entryFile = new kelondroBufferedIOChunks(ra, ra.name(), writeBufferSize, 30000 + random.nextLong() % 30000);
@ -424,6 +425,21 @@ public class kelondroRecords {
this.writeDouble = 0; this.writeDouble = 0;
this.cacheDelete = 0; this.cacheDelete = 0;
this.cacheFlush = 0; this.cacheFlush = 0;
// pre-load node cache
if ((preloadCacheTime > 0) && (cacheSize > 0)) {
long stop = System.currentTimeMillis() + preloadCacheTime;
Iterator i = contentNodes();
Node n;
int count = 0;
while ((System.currentTimeMillis() < stop) && (cacheHeaders.size() < cacheSize) && (i.hasNext())) {
n = (Node) i.next();
cacheHeaders.addb(n.handle.index, n.headChunk);
count++;
}
cacheHeaders.shape();
logFine("preloaded " + count + " records into cache");
}
} }
private static final long max = Runtime.getRuntime().maxMemory(); private static final long max = Runtime.getRuntime().maxMemory();
@ -567,9 +583,7 @@ public class kelondroRecords {
} }
private Node(Handle handle, byte[] bulkchunk, int offset) throws IOException { private Node(Handle handle, byte[] bulkchunk, int offset) throws IOException {
// create a new empty node and reserve empty space in file for it // this initializer is used to create nodes from bulk-read byte arrays
// use this method only if you want to extend the file with new entries
// without the need to have content in it.
this.handle = handle; this.handle = handle;
// create empty chunks // create empty chunks
@ -595,9 +609,9 @@ public class kelondroRecords {
} }
*/ */
private Node(Handle handle, Node parentNode, int referenceInParent) throws IOException { private Node(Handle handle, Node parentNode, int referenceInParent) throws IOException {
// this creates an entry with an pre-reserved entry position values can be written // this creates an entry with an pre-reserved entry position.
// using the setValues() method but we expect that values are already there in the file // values can be written using the setValues() method,
// ready to be read which we do not here // but we expect that values are already there in the file.
assert (handle != null): "node handle is null"; assert (handle != null): "node handle is null";
assert (handle.index >= 0): "node handle too low: " + handle.index; assert (handle.index >= 0): "node handle too low: " + handle.index;
//assert (handle.index < USAGE.allCount()) : "node handle too high: " + handle.index + ", USEDC=" + USAGE.USEDC + ", FREEC=" + USAGE.FREEC; //assert (handle.index < USAGE.allCount()) : "node handle too high: " + handle.index + ", USEDC=" + USAGE.USEDC + ", FREEC=" + USAGE.FREEC;

Write Preview
Loading…
Cancel
Save