From 5cc17ccf8a5bf4d2cb018a5b96d2a5055ebf8114 Mon Sep 17 00:00:00 2001 From: orbiter Date: Fri, 7 Aug 2009 11:55:32 +0000 Subject: [PATCH] a better caching with less overhead and more appropriate synchronisation use in more than 10 different data objects git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6250 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/de/anomic/kelondro/blob/MapView.java | 112 +++++++----------- .../de/anomic/kelondro/index/SimpleARC.java | 22 ++++ 2 files changed, 68 insertions(+), 66 deletions(-) diff --git a/source/de/anomic/kelondro/blob/MapView.java b/source/de/anomic/kelondro/blob/MapView.java index ab247ac29..eaedeba45 100644 --- a/source/de/anomic/kelondro/blob/MapView.java +++ b/source/de/anomic/kelondro/blob/MapView.java @@ -36,31 +36,25 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Map; +import de.anomic.kelondro.index.SimpleARC; import de.anomic.kelondro.order.CloneableIterator; import de.anomic.kelondro.order.NaturalOrder; import de.anomic.kelondro.order.RotateIterator; import de.anomic.kelondro.util.DateFormatter; import de.anomic.kelondro.util.FileUtils; -import de.anomic.kelondro.util.ScoreCluster; import de.anomic.kelondro.util.kelondroException; public class MapView { private BLOB blob; - private ScoreCluster cacheScore; - private HashMap> cache; - private final long startup; - private final int cachesize; + private SimpleARC> cache; private final char fillchar; public MapView(final Heap blob, final int cachesize, char fillchar) { this.blob = blob; - this.cache = new HashMap>(); - this.cacheScore = new ScoreCluster(); - this.startup = System.currentTimeMillis(); - this.cachesize = cachesize; + this.cache = new SimpleARC>(cachesize); this.fillchar = fillchar; /* // debug @@ -100,8 +94,7 @@ public class MapView { */ public synchronized void clear() throws IOException { this.blob.clear(); - this.cache = new HashMap>(); - this.cacheScore = new ScoreCluster(); + this.cache.clear(); } private static String map2string(final Map map, final String comment) { @@ -141,22 +134,19 @@ public class MapView { * @param newMap * @throws IOException */ - public synchronized void put(String key, final Map newMap) throws IOException { + public void put(String key, final Map newMap) throws IOException { assert (key != null); assert (key.length() > 0); assert (newMap != null); - if (cacheScore == null) return; // may appear during shutdown key = normalizeKey(key); - // write entry - blob.put(key.getBytes("UTF-8"), map2string(newMap, "W" + DateFormatter.formatShortSecond() + " ").getBytes("UTF-8")); - - // check for space in cache - checkCacheSpace(); - - // write map to cache - cacheScore.setScore(key, (int) ((System.currentTimeMillis() - startup) / 1000)); - cache.put(key, newMap); + synchronized (this) { + // write entry + blob.put(key.getBytes("UTF-8"), map2string(newMap, "W" + DateFormatter.formatShortSecond() + " ").getBytes("UTF-8")); + + // write map to cache + cache.put(key, newMap); + } } /** @@ -164,17 +154,18 @@ public class MapView { * @param key the primary key * @throws IOException */ - public synchronized void remove(String key) throws IOException { + public void remove(String key) throws IOException { // update elementCount if (key == null) return; key = normalizeKey(key); - // remove from cache - cacheScore.deleteScore(key); - cache.remove(key); - - // remove from file - blob.remove(key.getBytes()); + synchronized (this) { + // remove from cache + cache.remove(key); + + // remove from file + blob.remove(key.getBytes()); + } } /** @@ -183,12 +174,14 @@ public class MapView { * @return * @throws IOException */ - public synchronized boolean has(String key) throws IOException { + public boolean has(String key) throws IOException { assert key != null; if (cache == null) return false; // case may appear during shutdown key = normalizeKey(key); - if (this.cache.containsKey(key)) return true; - return this.blob.has(key.getBytes()); + synchronized (this) { + if (this.cache.containsKey(key)) return true; + return this.blob.has(key.getBytes()); + } } /** @@ -197,7 +190,7 @@ public class MapView { * @return * @throws IOException */ - public synchronized Map get(final String key) throws IOException { + public Map get(final String key) throws IOException { if (key == null) return null; return get(key, true); } @@ -208,43 +201,31 @@ public class MapView { return key; } - protected synchronized Map get(String key, final boolean storeCache) throws IOException { + protected Map get(String key, final boolean storeCache) throws IOException { // load map from cache assert key != null; if (cache == null) return null; // case may appear during shutdown key = normalizeKey(key); - Map map = cache.get(key); - if (map != null) return map; - - // load map - if (!(blob.has(key.getBytes()))) return null; - - // read object - final byte[] b = blob.get(key.getBytes()); - if (b == null) return null; - map = string2map(new String(b, "UTF-8")); - - if (storeCache) { - // cache it also - checkCacheSpace(); - // write map to cache - cacheScore.setScore(key, (int) ((System.currentTimeMillis() - startup) / 1000)); - cache.put(key, map); - } - - // return value - return map; - } + synchronized (this) { + Map map = cache.get(key); + if (map != null) return map; + + // load map + if (!(blob.has(key.getBytes()))) return null; + + // read object + final byte[] b = blob.get(key.getBytes()); + if (b == null) return null; + map = string2map(new String(b, "UTF-8")); + + if (storeCache) { + // write map to cache + cache.put(key, map); + } - private synchronized void checkCacheSpace() { - // check for space in cache - if (cache == null) return; // may appear during shutdown - if (cache.size() >= cachesize) { - // delete one entry - final String delkey = cacheScore.getMinObject(); - cacheScore.deleteScore(delkey); - cache.remove(delkey); + // return value + return map; } } @@ -298,9 +279,8 @@ public class MapView { /** * close the Map table */ - public void close() { + public synchronized void close() { cache = null; - cacheScore = null; // close file if (blob != null) blob.close(true); diff --git a/source/de/anomic/kelondro/index/SimpleARC.java b/source/de/anomic/kelondro/index/SimpleARC.java index d7425d96b..7e7231591 100644 --- a/source/de/anomic/kelondro/index/SimpleARC.java +++ b/source/de/anomic/kelondro/index/SimpleARC.java @@ -1,4 +1,5 @@ // SimpleARC.java +// a Simple Adaptive Replacement Cache // (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany // first published 17.04.2009 on http://yacy.net // @@ -90,6 +91,27 @@ public class SimpleARC { return v; } + /** + * check if the map contains the key + * @param s + * @return + */ + public boolean containsKey(K s) { + if (this.levelB.containsKey(s)) return true; + return this.levelA.containsKey(s); + } + + /** + * remove an entry from the cache + * @param s + * @return the old value + */ + public V remove(K s) { + V r = this.levelB.remove(s); + if (r != null) return r; + return this.levelA.remove(s); + } + /** * clear the cache */