boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

update of cache logging

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2917 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent 8385557672
commit d454ca44ee
2 changed files with 92 additions and 29 deletions
Show Stats Download Patch File Download Diff File

119
source/de/anomic/kelondro/kelondroCache.java
Unescape View File

@ -165,7 +165,10 @@ public class kelondroCache implements kelondroIndex {
if (writeBufferUnique == null) return; if (writeBufferUnique == null) return;
synchronized (writeBufferUnique) { synchronized (writeBufferUnique) {
Iterator i = writeBufferUnique.rows(); Iterator i = writeBufferUnique.rows();
while (i.hasNext()) index.addUnique((kelondroRow.Entry) i.next()); while (i.hasNext()) {
this.index.addUnique((kelondroRow.Entry) i.next());
this.cacheFlush++;
}
writeBufferUnique.clear(); writeBufferUnique.clear();
writeBufferUnique.trim(); writeBufferUnique.trim();
} }
@ -181,7 +184,8 @@ public class kelondroCache implements kelondroIndex {
while ((i.hasNext()) && (maxcount-- > 0)) { while ((i.hasNext()) && (maxcount-- > 0)) {
row = (kelondroRow.Entry) i.next(); row = (kelondroRow.Entry) i.next();
delete.add(row.getColBytes(index.primarykey())); delete.add(row.getColBytes(index.primarykey()));
index.addUnique(row); this.index.addUnique(row);
this.cacheFlush++;
} }
i = delete.rows(); i = delete.rows();
while (i.hasNext()) writeBufferUnique.remove(((kelondroRow.Entry) i.next()).getColBytes(0)); while (i.hasNext()) writeBufferUnique.remove(((kelondroRow.Entry) i.next()).getColBytes(0));
@ -194,7 +198,10 @@ public class kelondroCache implements kelondroIndex {
if (writeBufferDoubles == null) return; if (writeBufferDoubles == null) return;
synchronized (writeBufferDoubles) { synchronized (writeBufferDoubles) {
Iterator i = writeBufferDoubles.rows(); Iterator i = writeBufferDoubles.rows();
while (i.hasNext()) index.put((kelondroRow.Entry) i.next()); while (i.hasNext()) {
this.index.put((kelondroRow.Entry) i.next());
this.cacheFlush++;
}
writeBufferDoubles.clear(); writeBufferDoubles.clear();
writeBufferDoubles.trim(); writeBufferDoubles.trim();
} }
@ -210,7 +217,8 @@ public class kelondroCache implements kelondroIndex {
while ((i.hasNext()) && (maxcount-- > 0)) { while ((i.hasNext()) && (maxcount-- > 0)) {
row = (kelondroRow.Entry) i.next(); row = (kelondroRow.Entry) i.next();
delete.add(row.getColBytes(index.primarykey())); delete.add(row.getColBytes(index.primarykey()));
index.addUnique(row); this.index.addUnique(row);
this.cacheFlush++;
} }
i = delete.rows(); i = delete.rows();
while (i.hasNext()) writeBufferDoubles.remove(((kelondroRow.Entry) i.next()).getColBytes(0)); while (i.hasNext()) writeBufferDoubles.remove(((kelondroRow.Entry) i.next()).getColBytes(0));
@ -273,10 +281,10 @@ public class kelondroCache implements kelondroIndex {
// first look into the miss cache // first look into the miss cache
if (readMissCache != null) { if (readMissCache != null) {
if (readMissCache.get(key) != null) { if (readMissCache.get(key) != null) {
hasnotHit++; this.hasnotHit++;
return null; return null;
} else { } else {
hasnotMiss++; this.hasnotMiss++;
} }
} }
@ -286,36 +294,42 @@ public class kelondroCache implements kelondroIndex {
if (readHitCache != null) { if (readHitCache != null) {
entry = readHitCache.get(key); entry = readHitCache.get(key);
if (entry != null) { if (entry != null) {
readHit++; this.readHit++;
return entry; return entry;
} }
} }
if (writeBufferUnique != null) { if (writeBufferUnique != null) {
entry = writeBufferUnique.get(key); entry = writeBufferUnique.get(key);
if (entry != null) { if (entry != null) {
readHit++; this.readHit++;
return entry; return entry;
} }
} }
if (writeBufferDoubles != null) { if (writeBufferDoubles != null) {
entry = writeBufferDoubles.get(key); entry = writeBufferDoubles.get(key);
if (entry != null) { if (entry != null) {
readHit++; this.readHit++;
return entry; return entry;
} }
} }
// finally ask the backend index // finally ask the backend index
readMiss++; this.readMiss++;
entry = index.get(key); entry = index.get(key);
// learn from result // learn from result
if (entry == null) { if (entry == null) {
checkMissSpace(); checkMissSpace();
if (readMissCache != null) readMissCache.put(readMissCache.row().newEntry(key)); if (readMissCache != null) {
kelondroRow.Entry dummy = readMissCache.put(readMissCache.row().newEntry(key));
if (dummy == null) this.hasnotUnique++; else this.hasnotDouble++;
}
return null; return null;
} else { } else {
checkHitSpace(); checkHitSpace();
if (readHitCache != null) readHitCache.put(entry); if (readHitCache != null) {
kelondroRow.Entry dummy = readHitCache.put(entry);
if (dummy == null) this.writeUnique++; else this.writeDouble++;
}
return entry; return entry;
} }
} }
@ -348,7 +362,10 @@ public class kelondroCache implements kelondroIndex {
} }
assert (writeBufferDoubles == null); assert (writeBufferDoubles == null);
index.put(row); // write to backend index.put(row); // write to backend
if (readHitCache != null) readHitCache.put(row); // learn that entry if (readHitCache != null) {
kelondroRow.Entry dummy = readHitCache.put(row); // learn that entry
if (dummy == null) this.writeUnique++; else this.writeDouble++;
}
return null; return null;
} }
} }
@ -362,13 +379,15 @@ public class kelondroCache implements kelondroIndex {
if (writeBufferDoubles != null) { if (writeBufferDoubles != null) {
// because the entry exists, it must be written in the doubles buffer // because the entry exists, it must be written in the doubles buffer
readHitCache.remove(key); readHitCache.remove(key);
this.cacheDelete++;
writeBufferDoubles.put(row); writeBufferDoubles.put(row);
return entry; return entry;
} else { } else {
// write directly to backend index // write directly to backend index
index.put(row); index.put(row);
// learn from situation // learn from situation
readHitCache.put(row); // overwrite old entry kelondroRow.Entry dummy = readHitCache.put(row); // overwrite old entry
if (dummy == null) this.writeUnique++; else this.writeDouble++;
return entry; return entry;
} }
} }
@ -406,7 +425,10 @@ public class kelondroCache implements kelondroIndex {
// the worst case: we must write to the back-end directly // the worst case: we must write to the back-end directly
entry = index.put(row); entry = index.put(row);
if (readHitCache != null) readHitCache.put(row); // learn that entry if (readHitCache != null) {
kelondroRow.Entry dummy = readHitCache.put(row); // learn that entry
if (dummy == null) this.writeUnique++; else this.writeDouble++;
}
return entry; return entry;
} }
@ -426,11 +448,17 @@ public class kelondroCache implements kelondroIndex {
checkHitSpace(); checkHitSpace();
// remove entry from miss- and hit-cache // remove entry from miss- and hit-cache
if (readMissCache != null) readMissCache.remove(key); if (readMissCache != null) {
this.readMissCache.remove(key);
this.hasnotDelete++;
}
// the worst case: we must write to the backend directly // the worst case: we must write to the backend directly
Entry entry = index.put(row); Entry entry = index.put(row);
if (readHitCache != null) readHitCache.put(row); // learn that entry if (readHitCache != null) {
kelondroRow.Entry dummy = readHitCache.put(row); // learn that entry
if (dummy == null) this.writeUnique++; else this.writeDouble++;
}
return entry; return entry;
} }
@ -444,7 +472,8 @@ public class kelondroCache implements kelondroIndex {
// remove entry from miss- and hit-cache // remove entry from miss- and hit-cache
if (readMissCache != null) { if (readMissCache != null) {
readMissCache.remove(key); this.readMissCache.remove(key);
this.hasnotDelete++;
// the entry does not exist before // the entry does not exist before
if (writeBufferUnique != null) { if (writeBufferUnique != null) {
// since we know that the entry does not exist, we know that new // since we know that the entry does not exist, we know that new
@ -454,7 +483,10 @@ public class kelondroCache implements kelondroIndex {
} }
assert (writeBufferDoubles == null); assert (writeBufferDoubles == null);
index.addUnique(row); // write to backend index.addUnique(row); // write to backend
if (readHitCache != null) readHitCache.put(row); // learn that entry if (readHitCache != null) {
kelondroRow.Entry dummy = readHitCache.put(row); // learn that entry
if (dummy == null) this.writeUnique++; else this.writeDouble++;
}
return; return;
} }
@ -469,7 +501,10 @@ public class kelondroCache implements kelondroIndex {
// the worst case: we must write to the back-end directly // the worst case: we must write to the back-end directly
index.addUnique(row); index.addUnique(row);
if (readHitCache != null) readHitCache.put(row); // learn that entry if (readHitCache != null) {
kelondroRow.Entry dummy = readHitCache.put(row); // learn that entry
if (dummy == null) this.writeUnique++; else this.writeDouble++;
}
} }
public synchronized void addUnique(Entry row, Date entryDate) throws IOException { public synchronized void addUnique(Entry row, Date entryDate) throws IOException {
@ -488,11 +523,17 @@ public class kelondroCache implements kelondroIndex {
checkHitSpace(); checkHitSpace();
// remove entry from miss- and hit-cache // remove entry from miss- and hit-cache
if (readMissCache != null) readMissCache.remove(key); if (readMissCache != null) {
this.readMissCache.remove(key);
this.hasnotDelete++;
}
// the worst case: we must write to the backend directly // the worst case: we must write to the backend directly
index.addUnique(row); index.addUnique(row);
if (readHitCache != null) readHitCache.put(row); // learn that entry if (readHitCache != null) {
kelondroRow.Entry dummy = readHitCache.put(row); // learn that entry
if (dummy == null) this.writeUnique++; else this.writeDouble++;
}
} }
public synchronized Entry remove(byte[] key) throws IOException { public synchronized Entry remove(byte[] key) throws IOException {
@ -502,13 +543,23 @@ public class kelondroCache implements kelondroIndex {
// add entry to miss-cache // add entry to miss-cache
if (readMissCache != null) { if (readMissCache != null) {
// set the miss cache; if there was already an entry we know that the return value must be null // set the miss cache; if there was already an entry we know that the return value must be null
if (readMissCache.put(readMissCache.row().newEntry(key)) != null) return null; kelondroRow.Entry dummy = readMissCache.put(readMissCache.row().newEntry(key));
if (dummy == null) {
this.hasnotUnique++;
} else {
this.hasnotDouble++;
return null;
}
} }
// remove entry from hit-cache // remove entry from hit-cache
if (readHitCache != null) { if (readHitCache != null) {
Entry entry = readHitCache.remove(key); Entry entry = readHitCache.remove(key);
if (entry != null) { if (entry == null) {
this.readMiss++;
} else {
this.readHit++;
this.cacheDelete++;
index.remove(key); index.remove(key);
return entry; return entry;
} }
@ -536,14 +587,20 @@ public class kelondroCache implements kelondroIndex {
if ((writeBufferUnique != null) && (writeBufferUnique.size() > 0)) { if ((writeBufferUnique != null) && (writeBufferUnique.size() > 0)) {
Entry entry = writeBufferUnique.removeOne(); Entry entry = writeBufferUnique.removeOne();
if (readMissCache != null) readMissCache.put(readMissCache.row().newEntry(entry.getColBytes(index.primarykey()))); if (readMissCache != null) {
kelondroRow.Entry dummy = readMissCache.put(readMissCache.row().newEntry(entry.getColBytes(index.primarykey())));
if (dummy == null) this.hasnotUnique++; else this.hasnotDouble++;
}
return entry; return entry;
} }
if ((writeBufferDoubles != null) && (writeBufferDoubles.size() > 0)) { if ((writeBufferDoubles != null) && (writeBufferDoubles.size() > 0)) {
Entry entry = writeBufferDoubles.removeOne(); Entry entry = writeBufferDoubles.removeOne();
byte[] key = entry.getColBytes(index.primarykey()); byte[] key = entry.getColBytes(index.primarykey());
if (readMissCache != null) readMissCache.put(readMissCache.row().newEntry(key)); if (readMissCache != null) {
kelondroRow.Entry dummy = readMissCache.put(readMissCache.row().newEntry(key));
if (dummy == null) this.hasnotUnique++; else this.hasnotDouble++;
}
index.remove(key); index.remove(key);
return entry; return entry;
} }
@ -551,8 +608,14 @@ public class kelondroCache implements kelondroIndex {
Entry entry = index.removeOne(); Entry entry = index.removeOne();
if (entry == null) return null; if (entry == null) return null;
byte[] key = entry.getColBytes(index.primarykey()); byte[] key = entry.getColBytes(index.primarykey());
if (readMissCache != null) readMissCache.put(readMissCache.row().newEntry(key)); if (readMissCache != null) {
if (readHitCache != null) readHitCache.remove(key); kelondroRow.Entry dummy = readMissCache.put(readMissCache.row().newEntry(key));
if (dummy == null) this.hasnotUnique++; else this.hasnotDouble++;
}
if (readHitCache != null) {
kelondroRow.Entry dummy = readHitCache.remove(key);
if (dummy != null) this.cacheDelete++;
}
return entry; return entry;
} }

2
source/de/anomic/kelondro/kelondroRowCollection.java
Unescape View File

@ -150,7 +150,7 @@ public class kelondroRowCollection {
private final void ensureSize(int elements) { private final void ensureSize(int elements) {
int needed = elements * rowdef.objectsize(); int needed = elements * rowdef.objectsize();
if (chunkcache.length >= needed) return; if (chunkcache.length >= needed) return;
byte[] newChunkcache = new byte[needed * 2]; byte[] newChunkcache = new byte[needed * 12 / 10]; // increase space by 20%
System.arraycopy(chunkcache, 0, newChunkcache, 0, chunkcache.length); System.arraycopy(chunkcache, 0, newChunkcache, 0, chunkcache.length);
chunkcache = newChunkcache; chunkcache = newChunkcache;
newChunkcache = null; newChunkcache = null;

Write Preview
Loading…
Cancel
Save