boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

implemented the Map interface in the ARC classes so it will be possible to instantiate ARCs as

Browse Source 
Map<byte[], Map<String, byte[]>>
Because such Maps with byte[] keys cannot be stored in hash maps (bad hashing on byte[])
another ARC with comparable Maps has been added

This will make it possible to move the HTCache class 'Cache' into the cora package because that
class may be used either with RAM caches (ARCs) or with file-based caches (BEncodedHeaps)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7071 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent c60d0282fd
commit cf07b34c2d
6 changed files with 286 additions and 44 deletions
Show Stats Download Patch File Download Diff File

36
source/net/yacy/cora/storage/ARC.java
Unescape View File

@ -21,6 +21,10 @@
package net.yacy.cora.storage;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
/**
* This is a simple cache using two generations of hashtables to store the content with a LFU strategy.
* The Algorithm is described in a slightly more complex version as Adaptive Replacement Cache, "ARC".
@ -30,7 +34,7 @@ package net.yacy.cora.storage;
* at the same size.
*/
public interface ARC<K, V> {
public interface ARC<K, V> extends Iterable<Map.Entry<K, V>> {
/**
* get the size of the ARC. this returns the sum of main and ghost cache
@ -43,7 +47,14 @@ public interface ARC<K, V> {
* @param s
* @param v
*/
public void put(K s, V v);
public void insert(K s, V v);
/**
* put a value to the cache.
* @param s
* @param v
*/
public V put(K s, V v);
/**
* get a value from the cache.
@ -70,4 +81,25 @@ public interface ARC<K, V> {
* clear the cache
*/
public void clear();
/**
* iterator implements the Iterable interface
* the method can easily be implemented using the entrySet method
*/
public Iterator<Map.Entry<K, V>> iterator();
/**
* Return a Set view of the mappings contained in this map.
* This method is the basis for all methods that are implemented
* by a AbstractMap implementation
*
* @return a set view of the mappings contained in this map
*/
public Set<Map.Entry<K, V>> entrySet();
/**
* a hash code for this ARC
* @return a hash code
*/
int hashCode();
}

68
source/net/yacy/cora/storage/ComparableARC.java
Unescape View File

@ -0,0 +1,68 @@
/**
* ComparableARC
* an Adaptive Replacement Cache for comparable objects
* Copyright 2010 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany
* First released 24.08.2010 at http://yacy.net
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.cora.storage;
import java.util.Comparator;
import java.util.LinkedList;
import java.util.Map;
import java.util.TreeMap;
public final class ComparableARC<K, V> extends SimpleARC<K, V> implements Map<K, V>, Iterable<Map.Entry<K, V>>, ARC<K, V> {
public ComparableARC(final int cacheSize, Comparator<? super K> comparator) {
super.cacheSize = cacheSize / 2;
super.levelA = new LimitedTreeMap<K, V>(this.cacheSize, comparator);
super.levelB = new LimitedTreeMap<K, V>(this.cacheSize, comparator);
}
private static class LimitedTreeMap<K, V> extends TreeMap<K, V> {
private static final long serialVersionUID = -2276429187676080820L;
int limit;
LinkedList<K> keys;
public LimitedTreeMap(final int cacheSize, Comparator<? super K> comparator) {
super(comparator);
this.limit = cacheSize;
this.keys = new LinkedList<K>();
}
public V put(K k, V v) {
V r = super.put(k, v);
keys.add(k);
if (keys.size() > this.limit) {
K w = this.keys.removeFirst();
V t = super.remove(w);
assert t != null;
}
return r;
}
public V remove(Object k) {
V r = super.remove(k);
this.keys.remove(k);
return r;
}
public void clear() {
super.clear();
this.keys.clear();
}
}
}

86
source/net/yacy/cora/storage/ConcurrentARC.java
Unescape View File

@ -20,6 +20,13 @@
package net.yacy.cora.storage;
import java.util.AbstractMap;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
/**
* This is a simple cache using two generations of hashtables to store the content with a LFU strategy.
@ -30,7 +37,7 @@ package net.yacy.cora.storage;
* at the same size.
*/
public final class ConcurrentARC<K, V> implements ARC<K, V> {
public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, Iterable<Map.Entry<K, V>>, ARC<K, V> {
private final int mask;
private final ARC<K, V> arc[];
@ -39,19 +46,38 @@ public final class ConcurrentARC<K, V> implements ARC<K, V> {
public ConcurrentARC(final int cacheSize, final int partitions) {
int m = 1;
while (m < partitions) m = m * 2;
this.arc = new SimpleARC[m];
for (int i = 0; i < this.arc.length; i++) this.arc[i] = new SimpleARC<K, V>(cacheSize / m);
this.arc = new HashARC[m];
for (int i = 0; i < this.arc.length; i++) this.arc[i] = new HashARC<K, V>(cacheSize / m);
m -= 1;
this.mask = m;
}
@SuppressWarnings("unchecked")
public ConcurrentARC(final int cacheSize, final int partitions, Comparator<? super K> comparator) {
int m = 1;
while (m < partitions) m = m * 2;
this.arc = new ComparableARC[m];
for (int i = 0; i < this.arc.length; i++) this.arc[i] = new ComparableARC<K, V>(cacheSize / m, comparator);
m -= 1;
this.mask = m;
}
/**
* put a value to the cache.
* @param s
* @param v
*/
public final void insert(final K s, final V v) {
this.arc[s.hashCode() & mask].put(s, v);
}
/**
* put a value to the cache.
* @param s
* @param v
*/
public final void put(final K s, final V v) {
this.arc[s.hashCode() & mask].put(s, v);
public final V put(final K s, final V v) {
return this.arc[s.hashCode() & mask].put(s, v);
}
/**
@ -59,8 +85,9 @@ public final class ConcurrentARC<K, V> implements ARC<K, V> {
* @param s
* @return the value
*/
public final V get(final K s) {
return this.arc[s.hashCode() & mask].get(s);
@SuppressWarnings("unchecked")
public final V get(final Object s) {
return this.arc[s.hashCode() & mask].get((K) s);
}
/**
@ -68,8 +95,9 @@ public final class ConcurrentARC<K, V> implements ARC<K, V> {
* @param s
* @return
*/
public final boolean containsKey(final K s) {
return this.arc[s.hashCode() & mask].containsKey(s);
@SuppressWarnings("unchecked")
public final boolean containsKey(final Object s) {
return this.arc[s.hashCode() & mask].containsKey((K) s);
}
/**
@ -77,8 +105,9 @@ public final class ConcurrentARC<K, V> implements ARC<K, V> {
* @param s
* @return the old value
*/
public final V remove(final K s) {
return this.arc[s.hashCode() & mask].remove(s);
@SuppressWarnings("unchecked")
public final V remove(final Object s) {
return this.arc[s.hashCode() & mask].remove((K) s);
}
/**
@ -88,9 +117,44 @@ public final class ConcurrentARC<K, V> implements ARC<K, V> {
for (ARC<K, V> a: this.arc) a.clear();
}
/**
* get the size of the ARC.
* @return the complete number of entries in the ARC cache
*/
public final int size() {
int s = 0;
for (ARC<K, V> a: this.arc) s += a.size();
return s;
}
/**
* iterator implements the Iterable interface
*/
public Iterator<java.util.Map.Entry<K, V>> iterator() {
return entrySet().iterator();
}
/**
* Return a Set view of the mappings contained in this map.
* This method is the basis for all methods that are implemented
* by a AbstractMap implementation
*
* @return a set view of the mappings contained in this map
*/
@Override
public Set<java.util.Map.Entry<K, V>> entrySet() {
Set<Map.Entry<K, V>> m = new HashSet<Map.Entry<K, V>>();
for (ARC<K, V> a: this.arc) {
for (Map.Entry<K, V> entry: a.entrySet()) m.add(entry);
}
return m;
}
/**
* a hash code for this ARC
* @return a hash code
*/
public int hashCode() {
return this.arc.hashCode();
}
}

46
source/net/yacy/cora/storage/HashARC.java
Unescape View File

@ -0,0 +1,46 @@
/**
* HashARC
* an Adaptive Replacement Cache for objects that can be compared using hashing
* Copyright 2009 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany
* First released 17.04.2009 at http://yacy.net
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.cora.storage;
import java.util.LinkedHashMap;
import java.util.Map;
public final class HashARC<K, V> extends SimpleARC<K, V> implements Map<K, V>, Iterable<Map.Entry<K, V>>, ARC<K, V> {
public final static boolean accessOrder = false; // if false, then a insertion-order is used
public HashARC(final int cacheSize) {
this.cacheSize = cacheSize / 2;
super.levelA = new LinkedHashMap<K, V>(cacheSize, 0.1f, accessOrder) {
private static final long serialVersionUID = 1L;
@Override protected boolean removeEldestEntry(final Map.Entry<K, V> eldest) {
return size() > HashARC.this.cacheSize;
}
};
this.levelB = new LinkedHashMap<K, V>(cacheSize, 0.1f, accessOrder) {
private static final long serialVersionUID = 1L;
@Override protected boolean removeEldestEntry(final Map.Entry<K, V> eldest) {
return size() > HashARC.this.cacheSize;
}
};
}
}

90
source/net/yacy/cora/storage/SimpleARC.java
Unescape View File

@ -21,8 +21,11 @@
package net.yacy.cora.storage;
import java.util.LinkedHashMap;
import java.util.AbstractMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
/**
@ -32,37 +35,21 @@ import java.util.Map;
* or http://en.wikipedia.org/wiki/Adaptive_Replacement_Cache
* This version omits the ghost entry handling which is described in ARC, and keeps both cache levels
* at the same size.
*
* This class is defined abstract because it shall be used with either the HashARC or the ComparableARC classes
*/
public final class SimpleARC<K, V> implements ARC<K, V> {
abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, Iterable<Map.Entry<K, V>>, ARC<K, V> {
public final static boolean accessOrder = false; // if false, then a insertion-order is used
protected final int cacheSize;
private final Map<K, V> levelA, levelB;
public SimpleARC(final int cacheSize) {
this.cacheSize = cacheSize / 2;
this.levelA = new LinkedHashMap<K, V>(cacheSize, 0.1f, accessOrder) {
private static final long serialVersionUID = 1L;
@Override protected boolean removeEldestEntry(final Map.Entry<K, V> eldest) {
return size() > SimpleARC.this.cacheSize;
}
};
this.levelB = new LinkedHashMap<K, V>(cacheSize, 0.1f, accessOrder) {
private static final long serialVersionUID = 1L;
@Override protected boolean removeEldestEntry(final Map.Entry<K, V> eldest) {
return size() > SimpleARC.this.cacheSize;
}
};
}
protected int cacheSize;
protected Map<K, V> levelA, levelB;
/**
* put a value to the cache.
* @param s
* @param v
*/
public final synchronized void put(final K s, final V v) {
public final synchronized void insert(final K s, final V v) {
if (this.levelB.containsKey(s)) {
this.levelB.put(s, v);
assert (this.levelB.size() <= cacheSize); // the cache should shrink automatically
@ -72,19 +59,37 @@ public final class SimpleARC<K, V> implements ARC<K, V> {
}
}
/**
* put a value to the cache.
* @param s
* @param v
*/
public final synchronized V put(final K s, final V v) {
V r = null;
if (this.levelB.containsKey(s)) {
r = this.levelB.put(s, v);
assert (this.levelB.size() <= cacheSize); // the cache should shrink automatically
} else {
r = this.levelA.put(s, v);
assert (this.levelA.size() <= cacheSize); // the cache should shrink automatically
}
return r;
}
/**
* get a value from the cache.
* @param s
* @return the value
*/
public final synchronized V get(final K s) {
@SuppressWarnings("unchecked")
public final synchronized V get(final Object s) {
V v = this.levelB.get(s);
if (v != null) return v;
v = this.levelA.remove(s);
if (v == null) return null;
// move value from A to B; since it was already removed from A, just put it to B
//System.out.println("ARC: moving A->B, size(A) = " + this.levelA.size() + ", size(B) = " + this.levelB.size());
this.levelB.put(s, v);
this.levelB.put((K) s, v);
assert (this.levelB.size() <= cacheSize); // the cache should shrink automatically
return v;
}
@ -94,17 +99,18 @@ public final class SimpleARC<K, V> implements ARC<K, V> {
* @param s
* @return
*/
public final synchronized boolean containsKey(final K s) {
public final synchronized boolean containsKey(final Object s) {
if (this.levelB.containsKey(s)) return true;
return this.levelA.containsKey(s);
}
/**
* remove an entry from the cache
* @param s
* @return the old value
*/
public final synchronized V remove(final K s) {
public final synchronized V remove(final Object s) {
final V r = this.levelB.remove(s);
if (r != null) return r;
return this.levelA.remove(s);
@ -125,4 +131,34 @@ public final class SimpleARC<K, V> implements ARC<K, V> {
public final synchronized int size() {
return this.levelA.size() + this.levelB.size();
}
/**
* iterator implements the Iterable interface
*/
public Iterator<Map.Entry<K, V>> iterator() {
return entrySet().iterator();
}
/**
* Return a Set view of the mappings contained in this map.
* This method is the basis for all methods that are implemented
* by a AbstractMap implementation
*
* @return a set view of the mappings contained in this map
*/
@Override
public Set<Map.Entry<K, V>> entrySet() {
Set<Map.Entry<K, V>> m = new HashSet<Map.Entry<K, V>>();
for (Map.Entry<K, V> entry: this.levelA.entrySet()) m.add(entry);
for (Map.Entry<K, V> entry: this.levelB.entrySet()) m.add(entry);
return m;
}
/**
* a hash code for this ARC
* @return the hash code of one of the ARC partial hash tables
*/
public int hashCode() {
return this.levelA.hashCode();
}
}

4
source/net/yacy/kelondro/blob/BEncodedHeap.java
Unescape View File

@ -187,7 +187,6 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
* @param name
* @return true if the row exists
*/
@Override
public boolean containsKey(Object key) {
if (key instanceof byte[]) return containsKey((byte[]) key);
return false;
@ -197,7 +196,6 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
* the containsValue method cannot be used in this method
* and is only here to implement the Map interface
*/
@Override
public boolean containsValue(Object value) {
// this method shall not be used because it is not appropriate for this kind of data
throw new UnsupportedOperationException();
@ -222,7 +220,6 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
* @param name
* @return the map if one found or NULL if no entry exists or the entry is corrupt
*/
@Override
public Map<String, byte[]> get(Object key) {
if (key instanceof byte[])
try {
@ -315,7 +312,6 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
return value;
}
@Override
public Map<String, byte[]> remove(Object key) {
if (key instanceof byte[])
try {

Write Preview
Loading…
Cancel
Save