this should reduce IO a lot, because write caches are now actived for all databases - added new caching class that combines a read- and write-cache. - removed old read and write cache classes - removed superfluous RAM index (can be replaced by kelonodroRowSet) - addoped all current classes that used the old caching methods - more asserts, more bugfixes git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2865 6c8d7289-2bf4-0310-a012-ef5d649a1542pull/1/head
orbiter
19 years ago
parent
36f8b2c05a
commit
147d88cf23
@ -1,240 +0,0 @@
|
||||
// kelondroBufferedIndex.java
|
||||
// (C) 2006 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany
|
||||
// first published 16.10.2006 on http://www.anomic.de
|
||||
//
|
||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||
//
|
||||
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
|
||||
// $LastChangedRevision: 1986 $
|
||||
// $LastChangedBy: orbiter $
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package de.anomic.kelondro;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Date;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import de.anomic.server.serverMemory;
|
||||
import de.anomic.server.logging.serverLog;
|
||||
|
||||
public class kelondroBufferedIndex implements kelondroIndex {
|
||||
|
||||
// this implements a write buffer on index objects
|
||||
|
||||
private static final long memBlockLimit = 2000000; // do not fill cache further if the amount of available memory is less that this
|
||||
private static final int bufferFlushLimit = 10000;
|
||||
private static final int bufferFlushMinimum = 1000;
|
||||
private TreeMap buffer;
|
||||
private kelondroIndex index;
|
||||
|
||||
public kelondroBufferedIndex(kelondroIndex theIndex) {
|
||||
index = theIndex;
|
||||
buffer = (theIndex.order() == null) ? new TreeMap() : new TreeMap(theIndex.order());
|
||||
}
|
||||
|
||||
public synchronized void flush() throws IOException {
|
||||
if ((buffer == null) || (buffer.size() == 0)) return;
|
||||
Iterator i = buffer.entrySet().iterator();
|
||||
Map.Entry entry;
|
||||
while (i.hasNext()) {
|
||||
entry = (Map.Entry) i.next();
|
||||
index.put((kelondroRow.Entry) entry.getValue());
|
||||
}
|
||||
buffer.clear();
|
||||
}
|
||||
|
||||
public synchronized void flushOnce() throws IOException {
|
||||
if (buffer.size() == 0) return;
|
||||
Iterator i = buffer.entrySet().iterator();
|
||||
Map.Entry entry;
|
||||
if (i.hasNext()) {
|
||||
entry = (Map.Entry) i.next();
|
||||
System.out.println("*** DEBUG: flushed " + ((kelondroRow.Entry) entry.getValue()).getColString(0, null));
|
||||
index.put((kelondroRow.Entry) entry.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
public void flushSome() throws IOException {
|
||||
if (buffer.size() == 0) return;
|
||||
int flush = Math.max(1, buffer.size() / 10);
|
||||
while (flush-- > 0) flushOnce();
|
||||
}
|
||||
|
||||
public synchronized int size() throws IOException {
|
||||
return buffer.size() + index.size();
|
||||
}
|
||||
|
||||
public int writeBufferSize() {
|
||||
return buffer.size();
|
||||
}
|
||||
|
||||
public synchronized String toString() {
|
||||
try {flush();} catch (IOException e) {}
|
||||
return index.toString();
|
||||
}
|
||||
|
||||
public synchronized kelondroRow.Entry get(byte[] key) throws IOException {
|
||||
long handle = (index instanceof kelondroTree) ? index.profile().startRead() : -1;
|
||||
kelondroRow.Entry entry = null;
|
||||
entry = (kelondroRow.Entry) buffer.get(key);
|
||||
if (entry == null) entry = index.get(key);
|
||||
if (handle >= 0) index.profile().stopRead(handle);
|
||||
return entry;
|
||||
}
|
||||
|
||||
public synchronized kelondroRow.Entry put(kelondroRow.Entry row) throws IOException {
|
||||
return put(row, null);
|
||||
}
|
||||
|
||||
public synchronized kelondroRow.Entry put(kelondroRow.Entry row, Date entryDate) throws IOException {
|
||||
assert (row != null);
|
||||
assert (row.getColBytes(index.primarykey()) != null);
|
||||
assert (!(serverLog.allZero(row.getColBytes(index.primarykey()))));
|
||||
long handle = (index instanceof kelondroTree) ? index.profile().startWrite() : -1;
|
||||
byte[] key = row.getColBytes(index.primarykey());
|
||||
kelondroRow.Entry oldentry = null;
|
||||
oldentry = (kelondroRow.Entry) buffer.get(key);
|
||||
if (oldentry == null) {
|
||||
// try the collection
|
||||
oldentry = index.get(key);
|
||||
if (oldentry == null) {
|
||||
// this was not anywhere
|
||||
if (entryDate == null) {
|
||||
buffer.put(key, row);
|
||||
if (((buffer.size() > bufferFlushMinimum) && (serverMemory.available() > memBlockLimit))
|
||||
|| (buffer.size() > bufferFlushLimit))
|
||||
flush();
|
||||
} else {
|
||||
index.put(row, entryDate);
|
||||
}
|
||||
} else {
|
||||
// replace old entry
|
||||
if (entryDate == null) {
|
||||
index.put(row);
|
||||
} else {
|
||||
index.put(row, entryDate);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// the entry is already in buffer
|
||||
// simply replace old entry
|
||||
if (entryDate == null) {
|
||||
buffer.put(key, row);
|
||||
} else {
|
||||
buffer.remove(key);
|
||||
index.put(row, entryDate);
|
||||
}
|
||||
}
|
||||
if (handle >= 0) index.profile().stopWrite(handle);
|
||||
return oldentry;
|
||||
}
|
||||
|
||||
public synchronized void addUnique(kelondroRow.Entry row) throws IOException {
|
||||
assert (index instanceof kelondroRowSet);
|
||||
((kelondroRowSet) index).addUnique(row);
|
||||
}
|
||||
|
||||
public synchronized void addUnique(kelondroRow.Entry row, Date entryDate) throws IOException {
|
||||
addUnique(row);
|
||||
}
|
||||
|
||||
public synchronized kelondroRow.Entry remove(byte[] key) throws IOException {
|
||||
long handle = (index instanceof kelondroTree) ? index.profile().startDelete() : -1;
|
||||
kelondroRow.Entry oldentry = null;
|
||||
oldentry = (kelondroRow.Entry) buffer.remove(key);
|
||||
if (oldentry == null) {
|
||||
// try the collection
|
||||
return index.remove(key);
|
||||
}
|
||||
if (handle >= 0) index.profile().stopDelete(handle);
|
||||
return oldentry;
|
||||
}
|
||||
|
||||
public synchronized kelondroRow.Entry removeOne() throws IOException {
|
||||
long handle = (index instanceof kelondroTree) ? index.profile().startDelete() : -1;
|
||||
if (buffer.size() > 0) {
|
||||
byte[] key = (byte[]) buffer.keySet().iterator().next();
|
||||
kelondroRow.Entry entry = (kelondroRow.Entry) buffer.remove(key);
|
||||
if (handle >= 0) index.profile().stopDelete(handle);
|
||||
return entry;
|
||||
} else {
|
||||
kelondroRow.Entry entry = index.removeOne();
|
||||
if (handle >= 0) index.profile().stopDelete(handle);
|
||||
return entry;
|
||||
}
|
||||
}
|
||||
|
||||
public kelondroProfile profile() {
|
||||
return index.profile();
|
||||
}
|
||||
|
||||
public synchronized void close() throws IOException {
|
||||
flush();
|
||||
buffer = null;
|
||||
index.close();
|
||||
}
|
||||
|
||||
public kelondroOrder order() {
|
||||
return index.order();
|
||||
}
|
||||
|
||||
public int primarykey() {
|
||||
return index.primarykey();
|
||||
}
|
||||
|
||||
public kelondroRow row() throws IOException {
|
||||
return index.row();
|
||||
}
|
||||
|
||||
public synchronized Iterator rows() throws IOException {
|
||||
return rows(true, false, null);
|
||||
}
|
||||
|
||||
public synchronized Iterator rows(boolean up, boolean rotating, byte[] firstKey) throws IOException {
|
||||
flush();
|
||||
return index.rows(up, rotating, firstKey);
|
||||
}
|
||||
|
||||
public static kelondroBufferedIndex getRAMIndex(kelondroRow rowdef, int initSize) {
|
||||
return new kelondroBufferedIndex(new kelondroRowSet(rowdef, kelondroNaturalOrder.naturalOrder, 0, initSize));
|
||||
}
|
||||
|
||||
public final int cacheObjectChunkSize() {
|
||||
// dummy method
|
||||
return -1;
|
||||
}
|
||||
|
||||
public long[] cacheObjectStatus() {
|
||||
// dummy method
|
||||
return null;
|
||||
}
|
||||
|
||||
public final int cacheNodeChunkSize() {
|
||||
// returns the size that the node cache uses for a single entry
|
||||
return index.cacheNodeChunkSize();
|
||||
}
|
||||
|
||||
public final int[] cacheNodeStatus() {
|
||||
// a collection of different node cache status values
|
||||
return index.cacheNodeStatus();
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,151 +0,0 @@
|
||||
// kelondroCachedIndex
|
||||
// (C) 2006 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany
|
||||
// first published 23.10.2006 on http://www.anomic.de
|
||||
//
|
||||
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
|
||||
// $LastChangedRevision: 1986 $
|
||||
// $LastChangedBy: orbiter $
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package de.anomic.kelondro;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Date;
|
||||
import java.util.Iterator;
|
||||
|
||||
import de.anomic.kelondro.kelondroRow.Entry;
|
||||
import de.anomic.server.logging.serverLog;
|
||||
|
||||
public class kelondroCachedIndex implements kelondroIndex {
|
||||
|
||||
public final static int cacheObjectMissSize = 120;
|
||||
public final static int defaultObjectCachePercent = 10;
|
||||
|
||||
private kelondroObjectCache objectCache;
|
||||
private kelondroIndex theIndex;
|
||||
|
||||
public kelondroCachedIndex(kelondroIndex superIndex, long objectbuffersize) throws IOException {
|
||||
this.theIndex = superIndex;
|
||||
long objecthitcachesize = objectbuffersize * 4 / 5 / cacheObjectChunkSize();
|
||||
long objectmisscachesize = objectbuffersize / 5 / cacheObjectMissSize;
|
||||
this.objectCache = new kelondroObjectCache("generic", (int) objecthitcachesize, (int) objectmisscachesize, objecthitcachesize * 3000 , 4*1024*1024);
|
||||
}
|
||||
|
||||
public final int cacheObjectChunkSize() {
|
||||
try {
|
||||
return this.theIndex.row().objectsize() + /* overhead */ 16 * this.theIndex.row().columns();
|
||||
} catch (IOException e) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
public long[] cacheObjectStatus() {
|
||||
if (this.objectCache == null) return null;
|
||||
return this.objectCache.status();
|
||||
}
|
||||
|
||||
public final int cacheNodeChunkSize() {
|
||||
// returns the size that the node cache uses for a single entry
|
||||
return theIndex.cacheNodeChunkSize();
|
||||
}
|
||||
|
||||
public final int[] cacheNodeStatus() {
|
||||
// a collection of different node cache status values
|
||||
return theIndex.cacheNodeStatus();
|
||||
}
|
||||
|
||||
public void addUnique(Entry row) throws IOException {
|
||||
// the use case for add implies that usually the objects are not needed in the cache
|
||||
// therefore omit an object cache write here
|
||||
this.theIndex.addUnique(row);
|
||||
}
|
||||
|
||||
public void addUnique(Entry row, Date entryDate) throws IOException {
|
||||
this.theIndex.addUnique(row, entryDate);
|
||||
}
|
||||
|
||||
public void close() throws IOException {
|
||||
this.objectCache = null;
|
||||
this.theIndex.close();
|
||||
|
||||
}
|
||||
|
||||
public Entry get(byte[] key) throws IOException {
|
||||
// get result from cache
|
||||
kelondroRow.Entry result = (objectCache == null) ? null : (kelondroRow.Entry) objectCache.get(key);
|
||||
if (result != null) return result;
|
||||
// check if we have an entry in the miss cache
|
||||
if ((objectCache != null) && (objectCache.has(key) == -1)) return null;
|
||||
// finally: get it from the index
|
||||
result = this.theIndex.get(key);
|
||||
if (result == null) objectCache.hasnot(key); else objectCache.put(key, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
public kelondroOrder order() {
|
||||
return this.theIndex.order();
|
||||
}
|
||||
|
||||
public int primarykey() {
|
||||
return this.theIndex.primarykey();
|
||||
}
|
||||
|
||||
public kelondroProfile profile() {
|
||||
return this.theIndex.profile();
|
||||
}
|
||||
|
||||
public Entry put(Entry row) throws IOException {
|
||||
assert (row != null);
|
||||
assert (row.columns() == row().columns());
|
||||
assert (!(serverLog.allZero(row.getColBytes(theIndex.primarykey()))));
|
||||
objectCache.put(row.getColBytes(theIndex.primarykey()), row);
|
||||
return this.theIndex.put(row);
|
||||
}
|
||||
|
||||
public Entry put(Entry row, Date entryDate) throws IOException {
|
||||
assert (row.columns() == row().columns());
|
||||
objectCache.put(row.getColBytes(theIndex.primarykey()), row);
|
||||
return this.theIndex.put(row, entryDate);
|
||||
}
|
||||
|
||||
public Entry remove(byte[] key) throws IOException {
|
||||
if (objectCache.has(key) == -1) return null;
|
||||
objectCache.remove(key);
|
||||
return this.theIndex.remove(key);
|
||||
}
|
||||
|
||||
public Entry removeOne() throws IOException {
|
||||
Entry entry = this.theIndex.removeOne();
|
||||
if (entry == null) return null;
|
||||
this.objectCache.remove(entry.getColBytes(this.theIndex.primarykey()));
|
||||
return entry;
|
||||
}
|
||||
|
||||
public kelondroRow row() throws IOException {
|
||||
return this.theIndex.row();
|
||||
}
|
||||
|
||||
public Iterator rows(boolean up, boolean rotating, byte[] firstKey) throws IOException {
|
||||
return this.theIndex.rows(up, rotating, firstKey);
|
||||
}
|
||||
|
||||
public int size() throws IOException {
|
||||
return this.theIndex.size();
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,307 +0,0 @@
|
||||
// kelondroObjectCache.java
|
||||
// (C) 2006 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany
|
||||
// first published 2006 on http://www.anomic.de
|
||||
//
|
||||
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
|
||||
// $LastChangedRevision: 1986 $
|
||||
// $LastChangedBy: orbiter $
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package de.anomic.kelondro;
|
||||
|
||||
import java.util.TreeMap;
|
||||
|
||||
import de.anomic.server.serverMemory;
|
||||
|
||||
public class kelondroObjectCache {
|
||||
|
||||
private final TreeMap cache;
|
||||
private final kelondroMScoreCluster ages, hasnot;
|
||||
private long startTime;
|
||||
private int maxHitSize, maxMissSize;
|
||||
private long maxAge;
|
||||
private long minMem;
|
||||
private int readHit, readMiss, writeUnique, writeDouble, cacheDelete, cacheFlush;
|
||||
private int hasnotHit, hasnotMiss, hasnotUnique, hasnotDouble, hasnotDelete, hasnotFlush;
|
||||
private String name;
|
||||
|
||||
public kelondroObjectCache(String name, int maxHitSize, int maxMissSize, long maxAge, long minMem) {
|
||||
this.name = name;
|
||||
this.cache = new TreeMap();
|
||||
this.ages = new kelondroMScoreCluster();
|
||||
this.hasnot = new kelondroMScoreCluster();
|
||||
this.startTime = System.currentTimeMillis();
|
||||
this.maxHitSize = Math.max(maxHitSize, 1);
|
||||
this.maxMissSize = Math.max(maxMissSize, 1);
|
||||
this.maxAge = Math.max(maxAge, 10000);
|
||||
this.minMem = Math.max(minMem, 1024 * 1024);
|
||||
this.readHit = 0;
|
||||
this.readMiss = 0;
|
||||
this.writeUnique = 0;
|
||||
this.writeDouble = 0;
|
||||
this.cacheDelete = 0;
|
||||
this.cacheFlush = 0;
|
||||
this.hasnotHit = 0;
|
||||
this.hasnotMiss = 0;
|
||||
this.hasnotUnique = 0;
|
||||
this.hasnotDouble = 0;
|
||||
this.hasnotDelete = 0;
|
||||
this.hasnotFlush = 0;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setMaxAge(long maxAge) {
|
||||
this.maxAge = maxAge;
|
||||
}
|
||||
|
||||
public void setMaxHitSize(int maxSize) {
|
||||
this.maxHitSize = maxSize;
|
||||
}
|
||||
|
||||
public void setMaxMissSize(int maxSize) {
|
||||
this.maxMissSize = maxSize;
|
||||
}
|
||||
|
||||
public int maxHitSize() {
|
||||
return this.maxHitSize;
|
||||
}
|
||||
|
||||
public int maxMissSize() {
|
||||
return this.maxMissSize;
|
||||
}
|
||||
|
||||
public void setMinMem(int minMem) {
|
||||
this.minMem = minMem;
|
||||
}
|
||||
|
||||
public long minAge() {
|
||||
if (ages.size() == 0) return 0;
|
||||
return System.currentTimeMillis() - longEmit(ages.getMaxScore());
|
||||
}
|
||||
|
||||
public long maxAge() {
|
||||
if (ages.size() == 0) return 0;
|
||||
return System.currentTimeMillis() - longEmit(ages.getMinScore());
|
||||
}
|
||||
|
||||
public int hitsize() {
|
||||
return cache.size();
|
||||
}
|
||||
|
||||
public int misssize() {
|
||||
return hasnot.size();
|
||||
}
|
||||
|
||||
public long[] status() {
|
||||
return new long[]{
|
||||
(long) maxHitSize(),
|
||||
(long) maxMissSize(),
|
||||
(long) hitsize(),
|
||||
(long) misssize(),
|
||||
this.maxAge,
|
||||
minAge(),
|
||||
maxAge(),
|
||||
(long) readHit,
|
||||
(long) readMiss,
|
||||
(long) writeUnique,
|
||||
(long) writeDouble,
|
||||
(long) cacheDelete,
|
||||
(long) cacheFlush,
|
||||
(long) hasnotHit,
|
||||
(long) hasnotMiss,
|
||||
(long) hasnotUnique,
|
||||
(long) hasnotDouble,
|
||||
(long) hasnotDelete,
|
||||
(long) hasnotFlush
|
||||
};
|
||||
}
|
||||
|
||||
private static long[] combinedStatus(long[] a, long[] b) {
|
||||
return new long[]{
|
||||
a[0] + b[0],
|
||||
a[1] + b[1],
|
||||
a[2] + b[2],
|
||||
a[3] + b[3],
|
||||
Math.max(a[4], b[4]),
|
||||
Math.min(a[5], b[5]),
|
||||
Math.max(a[6], b[6]),
|
||||
a[7] + b[7],
|
||||
a[8] + b[8],
|
||||
a[9] + b[9],
|
||||
a[10] + b[10],
|
||||
a[11] + b[11],
|
||||
a[12] + b[12],
|
||||
a[13] + b[13],
|
||||
a[14] + b[14],
|
||||
a[15] + b[15],
|
||||
a[16] + b[16],
|
||||
a[17] + b[17],
|
||||
a[18] + b[18]
|
||||
};
|
||||
}
|
||||
|
||||
public static long[] combinedStatus(long[][] a, int l) {
|
||||
if ((a == null) || (a.length == 0) || (l == 0)) return null;
|
||||
if ((a.length >= 1) && (l == 1)) return a[0];
|
||||
if ((a.length >= 2) && (l == 2)) return combinedStatus(a[0], a[1]);
|
||||
return combinedStatus(combinedStatus(a, l - 1), a[l - 1]);
|
||||
}
|
||||
|
||||
private int intTime(long longTime) {
|
||||
return (int) Math.max(0, ((longTime - startTime) / 1000));
|
||||
}
|
||||
|
||||
private long longEmit(int intTime) {
|
||||
return (((long) intTime) * (long) 1000) + startTime;
|
||||
}
|
||||
|
||||
public void put(byte[] key, Object value) {
|
||||
if (key != null) put(new String(key), value);
|
||||
}
|
||||
|
||||
public void put(String key, Object value) {
|
||||
if ((key == null) || (value == null)) return;
|
||||
Object prev = null;
|
||||
synchronized(cache) {
|
||||
prev = cache.put(key, value);
|
||||
ages.setScore(key, intTime(System.currentTimeMillis()));
|
||||
if (hasnot.deleteScore(key) != 0) hasnotDelete++;
|
||||
}
|
||||
if (prev == null) this.writeUnique++; else this.writeDouble++;
|
||||
flushc();
|
||||
}
|
||||
|
||||
public Object get(byte[] key) {
|
||||
return get(new String(key));
|
||||
}
|
||||
|
||||
public Object get(String key) {
|
||||
if (key == null) return null;
|
||||
Object r = null;
|
||||
synchronized(cache) {
|
||||
r = cache.get(key);
|
||||
if (r == null) {
|
||||
this.readMiss++;
|
||||
} else {
|
||||
this.readHit++;
|
||||
ages.setScore(key, intTime(System.currentTimeMillis())); // renew cache update time
|
||||
}
|
||||
}
|
||||
flushc();
|
||||
return r;
|
||||
}
|
||||
|
||||
public void hasnot(byte[] key) {
|
||||
hasnot(new String(key));
|
||||
}
|
||||
|
||||
public void hasnot(String key) {
|
||||
if (key == null) return;
|
||||
int prev = 0;
|
||||
synchronized(cache) {
|
||||
if (cache.remove(key) != null) cacheDelete++;
|
||||
ages.deleteScore(key);
|
||||
prev = hasnot.getScore(key);
|
||||
hasnot.setScore(key, intTime(System.currentTimeMillis()));
|
||||
}
|
||||
if (prev == 0) this.hasnotUnique++; else this.hasnotDouble++;
|
||||
flushh();
|
||||
}
|
||||
|
||||
public int has(byte[] key) {
|
||||
return has(new String(key));
|
||||
}
|
||||
|
||||
public int has(String key) {
|
||||
// returns a 3-value boolean:
|
||||
// 1 = key definitely exists
|
||||
// -1 = key definitely does not exist
|
||||
// 0 = unknown, if key exists
|
||||
if (key == null) return 0;
|
||||
synchronized(cache) {
|
||||
if (hasnot.getScore(key) > 0) {
|
||||
hasnot.setScore(key, intTime(System.currentTimeMillis())); // renew cache update time
|
||||
this.hasnotHit++;
|
||||
return -1;
|
||||
}
|
||||
this.hasnotMiss++;
|
||||
if (cache.get(key) != null) return 1;
|
||||
}
|
||||
flushh();
|
||||
return 0;
|
||||
}
|
||||
|
||||
public void remove(byte[] key) {
|
||||
remove(new String(key));
|
||||
}
|
||||
|
||||
public void remove(String key) {
|
||||
if (key == null) return;
|
||||
synchronized(cache) {
|
||||
if (cache.remove(key) != null) cacheDelete++;
|
||||
ages.deleteScore(key);
|
||||
hasnot.setScore(key, intTime(System.currentTimeMillis()));
|
||||
}
|
||||
flushh();
|
||||
}
|
||||
|
||||
public void flushc() {
|
||||
String k;
|
||||
synchronized(cache) {
|
||||
while ((ages.size() > 0) &&
|
||||
((k = (String) ages.getMinObject()) != null) &&
|
||||
((ages.size() > maxHitSize) ||
|
||||
(((System.currentTimeMillis() - longEmit(ages.getScore(k))) > maxAge) &&
|
||||
(serverMemory.available() < minMem)))
|
||||
) {
|
||||
cache.remove(k);
|
||||
ages.deleteScore(k);
|
||||
cacheFlush++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void flushh() {
|
||||
String k;
|
||||
synchronized(cache) {
|
||||
while ((hasnot.size() > 0) &&
|
||||
((k = (String) hasnot.getMinObject()) != null) &&
|
||||
((hasnot.size() > maxMissSize) ||
|
||||
(((System.currentTimeMillis() - longEmit(hasnot.getScore(k))) > maxAge) &&
|
||||
(serverMemory.available() < minMem)))
|
||||
) {
|
||||
hasnot.deleteScore(k);
|
||||
hasnotFlush++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
// test to measure memory usage of miss cache
|
||||
kelondroMScoreCluster t = new kelondroMScoreCluster();
|
||||
System.gc(); long s0 = Runtime.getRuntime().freeMemory();
|
||||
int loop = 200000;
|
||||
for (int i = 0; i < loop; i++) t.setScore((Integer.toString(i) + "000000000000").substring(0, 12), i);
|
||||
System.gc(); long s1 = Runtime.getRuntime().freeMemory();
|
||||
System.out.println((s1 - s0) / loop);
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,126 +0,0 @@
|
||||
// kelondroRAMIndex.java
|
||||
// (C) 2006 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany
|
||||
// first published 12.08.2006 on http://www.anomic.de
|
||||
//
|
||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||
//
|
||||
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
|
||||
// $LastChangedRevision: 1986 $
|
||||
// $LastChangedBy: orbiter $
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package de.anomic.kelondro;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Date;
|
||||
import java.util.Iterator;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import de.anomic.kelondro.kelondroRow.Entry;
|
||||
|
||||
public class kelondroRAMIndex implements kelondroIndex {
|
||||
|
||||
private TreeMap index;
|
||||
private kelondroOrder order;
|
||||
private kelondroRow rowdef;
|
||||
private kelondroProfile profile;
|
||||
|
||||
public kelondroRAMIndex(kelondroOrder defaultOrder, kelondroRow rowdef) {
|
||||
this.index = new TreeMap(defaultOrder);
|
||||
this.order = defaultOrder;
|
||||
this.rowdef = rowdef;
|
||||
this.profile = new kelondroProfile();
|
||||
}
|
||||
|
||||
public kelondroOrder order() {
|
||||
return this.order;
|
||||
}
|
||||
|
||||
public int primarykey() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
public synchronized int size() {
|
||||
return this.index.size();
|
||||
}
|
||||
|
||||
public kelondroRow row() {
|
||||
return this.rowdef;
|
||||
}
|
||||
|
||||
public synchronized Entry get(byte[] key) {
|
||||
return (kelondroRow.Entry) index.get(key);
|
||||
}
|
||||
|
||||
public kelondroRow.Entry put(kelondroRow.Entry row, Date entryDate) throws IOException {
|
||||
return put(row);
|
||||
}
|
||||
|
||||
public synchronized Entry put(Entry row) {
|
||||
return (kelondroRow.Entry) index.put(row.getColBytes(0), row);
|
||||
}
|
||||
|
||||
public synchronized void addUnique(kelondroRow.Entry row) throws IOException {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
public synchronized void addUnique(kelondroRow.Entry row, Date entryDate) throws IOException {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
public synchronized Entry remove(byte[] key) {
|
||||
return (kelondroRow.Entry) index.remove(key);
|
||||
}
|
||||
|
||||
public synchronized Entry removeOne() {
|
||||
if (this.index.size() == 0) return null;
|
||||
return remove((byte[]) index.keySet().iterator().next());
|
||||
}
|
||||
|
||||
public synchronized Iterator rows(boolean up, boolean rotating, byte[] firstKey) {
|
||||
return index.values().iterator();
|
||||
}
|
||||
|
||||
public void close() {
|
||||
index = null;
|
||||
}
|
||||
|
||||
public kelondroProfile profile() {
|
||||
return profile;
|
||||
}
|
||||
|
||||
public final int cacheObjectChunkSize() {
|
||||
// dummy method
|
||||
return -1;
|
||||
}
|
||||
|
||||
public long[] cacheObjectStatus() {
|
||||
// dummy method
|
||||
return null;
|
||||
}
|
||||
|
||||
public final int cacheNodeChunkSize() {
|
||||
// returns the size that the node cache uses for a single entry
|
||||
return -1;
|
||||
}
|
||||
|
||||
public final int[] cacheNodeStatus() {
|
||||
// a collection of different node cache status values
|
||||
return new int[]{0,0,0,0,0,0,0,0,0,0};
|
||||
}
|
||||
}
|
||||
Loading…
Reference in new issue