a better caching with less overhead and more appropriate

synchronisation use in more than 10 different data objects

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6250 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent 2e01bd955d
commit 5cc17ccf8a

@ -36,31 +36,25 @@ import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.Map; import java.util.Map;
import de.anomic.kelondro.index.SimpleARC;
import de.anomic.kelondro.order.CloneableIterator; import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.order.NaturalOrder; import de.anomic.kelondro.order.NaturalOrder;
import de.anomic.kelondro.order.RotateIterator; import de.anomic.kelondro.order.RotateIterator;
import de.anomic.kelondro.util.DateFormatter; import de.anomic.kelondro.util.DateFormatter;
import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.FileUtils;
import de.anomic.kelondro.util.ScoreCluster;
import de.anomic.kelondro.util.kelondroException; import de.anomic.kelondro.util.kelondroException;
public class MapView { public class MapView {
private BLOB blob; private BLOB blob;
private ScoreCluster<String> cacheScore; private SimpleARC<String, Map<String, String>> cache;
private HashMap<String, Map<String, String>> cache;
private final long startup;
private final int cachesize;
private final char fillchar; private final char fillchar;
public MapView(final Heap blob, final int cachesize, char fillchar) { public MapView(final Heap blob, final int cachesize, char fillchar) {
this.blob = blob; this.blob = blob;
this.cache = new HashMap<String, Map<String, String>>(); this.cache = new SimpleARC<String, Map<String, String>>(cachesize);
this.cacheScore = new ScoreCluster<String>();
this.startup = System.currentTimeMillis();
this.cachesize = cachesize;
this.fillchar = fillchar; this.fillchar = fillchar;
/* /*
// debug // debug
@ -100,8 +94,7 @@ public class MapView {
*/ */
public synchronized void clear() throws IOException { public synchronized void clear() throws IOException {
this.blob.clear(); this.blob.clear();
this.cache = new HashMap<String, Map<String, String>>(); this.cache.clear();
this.cacheScore = new ScoreCluster<String>();
} }
private static String map2string(final Map<String, String> map, final String comment) { private static String map2string(final Map<String, String> map, final String comment) {
@ -141,41 +134,39 @@ public class MapView {
* @param newMap * @param newMap
* @throws IOException * @throws IOException
*/ */
public synchronized void put(String key, final Map<String, String> newMap) throws IOException { public void put(String key, final Map<String, String> newMap) throws IOException {
assert (key != null); assert (key != null);
assert (key.length() > 0); assert (key.length() > 0);
assert (newMap != null); assert (newMap != null);
if (cacheScore == null) return; // may appear during shutdown
key = normalizeKey(key); key = normalizeKey(key);
synchronized (this) {
// write entry // write entry
blob.put(key.getBytes("UTF-8"), map2string(newMap, "W" + DateFormatter.formatShortSecond() + " ").getBytes("UTF-8")); blob.put(key.getBytes("UTF-8"), map2string(newMap, "W" + DateFormatter.formatShortSecond() + " ").getBytes("UTF-8"));
// check for space in cache
checkCacheSpace();
// write map to cache // write map to cache
cacheScore.setScore(key, (int) ((System.currentTimeMillis() - startup) / 1000));
cache.put(key, newMap); cache.put(key, newMap);
} }
}
/** /**
* remove a Map * remove a Map
* @param key the primary key * @param key the primary key
* @throws IOException * @throws IOException
*/ */
public synchronized void remove(String key) throws IOException { public void remove(String key) throws IOException {
// update elementCount // update elementCount
if (key == null) return; if (key == null) return;
key = normalizeKey(key); key = normalizeKey(key);
synchronized (this) {
// remove from cache // remove from cache
cacheScore.deleteScore(key);
cache.remove(key); cache.remove(key);
// remove from file // remove from file
blob.remove(key.getBytes()); blob.remove(key.getBytes());
} }
}
/** /**
* check if a specific key is in the database * check if a specific key is in the database
@ -183,13 +174,15 @@ public class MapView {
* @return * @return
* @throws IOException * @throws IOException
*/ */
public synchronized boolean has(String key) throws IOException { public boolean has(String key) throws IOException {
assert key != null; assert key != null;
if (cache == null) return false; // case may appear during shutdown if (cache == null) return false; // case may appear during shutdown
key = normalizeKey(key); key = normalizeKey(key);
synchronized (this) {
if (this.cache.containsKey(key)) return true; if (this.cache.containsKey(key)) return true;
return this.blob.has(key.getBytes()); return this.blob.has(key.getBytes());
} }
}
/** /**
* retrieve the whole Map from the table * retrieve the whole Map from the table
@ -197,7 +190,7 @@ public class MapView {
* @return * @return
* @throws IOException * @throws IOException
*/ */
public synchronized Map<String, String> get(final String key) throws IOException { public Map<String, String> get(final String key) throws IOException {
if (key == null) return null; if (key == null) return null;
return get(key, true); return get(key, true);
} }
@ -208,12 +201,13 @@ public class MapView {
return key; return key;
} }
protected synchronized Map<String, String> get(String key, final boolean storeCache) throws IOException { protected Map<String, String> get(String key, final boolean storeCache) throws IOException {
// load map from cache // load map from cache
assert key != null; assert key != null;
if (cache == null) return null; // case may appear during shutdown if (cache == null) return null; // case may appear during shutdown
key = normalizeKey(key); key = normalizeKey(key);
synchronized (this) {
Map<String, String> map = cache.get(key); Map<String, String> map = cache.get(key);
if (map != null) return map; if (map != null) return map;
@ -226,26 +220,13 @@ public class MapView {
map = string2map(new String(b, "UTF-8")); map = string2map(new String(b, "UTF-8"));
if (storeCache) { if (storeCache) {
// cache it also
checkCacheSpace();
// write map to cache // write map to cache
cacheScore.setScore(key, (int) ((System.currentTimeMillis() - startup) / 1000));
cache.put(key, map); cache.put(key, map);
} }
// return value // return value
return map; return map;
} }
private synchronized void checkCacheSpace() {
// check for space in cache
if (cache == null) return; // may appear during shutdown
if (cache.size() >= cachesize) {
// delete one entry
final String delkey = cacheScore.getMinObject();
cacheScore.deleteScore(delkey);
cache.remove(delkey);
}
} }
/** /**
@ -298,9 +279,8 @@ public class MapView {
/** /**
* close the Map table * close the Map table
*/ */
public void close() { public synchronized void close() {
cache = null; cache = null;
cacheScore = null;
// close file // close file
if (blob != null) blob.close(true); if (blob != null) blob.close(true);

@ -1,4 +1,5 @@
// SimpleARC.java // SimpleARC.java
// a Simple Adaptive Replacement Cache
// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany // (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 17.04.2009 on http://yacy.net // first published 17.04.2009 on http://yacy.net
// //
@ -90,6 +91,27 @@ public class SimpleARC <K, V> {
return v; return v;
} }
/**
* check if the map contains the key
* @param s
* @return
*/
public boolean containsKey(K s) {
if (this.levelB.containsKey(s)) return true;
return this.levelA.containsKey(s);
}
/**
* remove an entry from the cache
* @param s
* @return the old value
*/
public V remove(K s) {
V r = this.levelB.remove(s);
if (r != null) return r;
return this.levelA.remove(s);
}
/** /**
* clear the cache * clear the cache
*/ */

Loading…
Cancel
Save