boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added BEncodedHeapBag and BEncodedHeapShard which are storage container

Browse Source 
for a new metadata store. An abstraction of the content for this storage
is defined with MapStore. A MapStore is an abstraction of a RDF Node
store.
pull/1/head
Michael Peter Christen 13 years ago
parent 6fecd0db88
commit 613ab6a69d
5 changed files with 842 additions and 3 deletions
Show Stats Download Patch File Download Diff File

111
source/net/yacy/kelondro/blob/AbstractMapStore.java
Unescape View File

@ -0,0 +1,111 @@
// AbstractMapStore.java
// (C) 2011 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 16.11.2011 on http://yacy.net
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.kelondro.blob;
import java.util.AbstractMap;
import java.util.Collection;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import net.yacy.cora.document.UTF8;
public abstract class AbstractMapStore implements MapStore {
@Override
public boolean containsValue(Object arg0) {
throw new UnsupportedOperationException("ContainsValue() not appropriate, use outer indexing");
}
@Override
public Set<java.util.Map.Entry<byte[], Map<String, byte[]>>> entrySet() {
throw new UnsupportedOperationException("entrySet() not appropriate, use an iterator");
}
@Override
public Set<byte[]> keySet() {
throw new UnsupportedOperationException("keySet() not appropriate, use an iterator");
}
@Override
public void putAll(Map<? extends byte[], ? extends Map<String, byte[]>> entries) {
if (entries instanceof MapStore) {
Iterator<Map.Entry<byte[], Map<String, byte[]>>> i = ((MapStore) entries).iterator();
Map.Entry<? extends byte[], ? extends Map<String, byte[]>> entry;
while (i.hasNext()) {
entry = i.next();
this.put(entry.getKey(), entry.getValue());
}
} else {
for (Map.Entry<? extends byte[], ? extends Map<String, byte[]>> e: entries.entrySet()) {
this.put(e.getKey(), e.getValue());
}
}
}
@Override
public Collection<Map<String, byte[]>> values() {
throw new UnsupportedOperationException("values() not appropriate, use an iterator");
}
@Override
public Iterator<Map.Entry<byte[], Map<String, byte[]>>> iterator() {
final Iterator<byte[]> k = this.keyIterator();
return new Iterator<Map.Entry<byte[], Map<String, byte[]>>>(){
@Override
public boolean hasNext() {
return k.hasNext();
}
@Override
public Map.Entry<byte[], Map<String, byte[]>> next() {
byte[] key = k.next();
if (key == null) return null;
return new AbstractMap.SimpleImmutableEntry<byte[], Map<String, byte[]>>(key, AbstractMapStore.this.get(key));
}
@Override
public void remove() {
k.remove();
}
};
}
public static String map2String(Map<String, byte[]> map) {
StringBuilder sb = new StringBuilder(map.size() * 50);
sb.append("<map>\n");
for (Map.Entry<String, byte[]> entry: map.entrySet()) {
sb.append('<').append(entry.getKey()).append('>');
sb.append(UTF8.String(entry.getValue()));
sb.append("</").append(entry.getKey()).append(">\n");
}
sb.append("</map>\n");
return sb.toString();
}
}

44
source/net/yacy/kelondro/blob/BEncodedHeap.java
Unescape View File

@ -42,6 +42,7 @@ import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.order.ByteOrder; import net.yacy.kelondro.order.ByteOrder;
import net.yacy.kelondro.order.CloneableIterator;
import net.yacy.kelondro.order.Digest; import net.yacy.kelondro.order.Digest;
import net.yacy.kelondro.order.NaturalOrder; import net.yacy.kelondro.order.NaturalOrder;
import net.yacy.kelondro.util.BDecoder; import net.yacy.kelondro.util.BDecoder;
@ -53,9 +54,7 @@ import net.yacy.kelondro.util.FileUtils;
* store a table of properties (instead of fixed-field entries) this is realized using blobs and BEncoded * store a table of properties (instead of fixed-field entries) this is realized using blobs and BEncoded
* property lists * property lists
*/ */
public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, public class BEncodedHeap implements MapStore {
Iterable<Map.Entry<byte[], Map<String, byte[]>>>
{
private Heap table; private Heap table;
private final LinkedHashSet<String> columnames; private final LinkedHashSet<String> columnames;
@ -89,6 +88,42 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>,
this.columnames = new LinkedHashSet<String>(); this.columnames = new LinkedHashSet<String>();
} }
@Override
public ByteOrder getOrdering() {
return this.table.ordering;
}
@Override
public CloneableIterator<byte[]> keyIterator() {
try {
return this.table.keys(true, false);
} catch (IOException e) {
Log.logSevere("BEncodedHeap", "returning empty iterator for failed key iteration: " + e.getMessage(), e);
return new CloneableIterator<byte[]>(){
@Override
public boolean hasNext() {
return false;
}
@Override
public byte[] next() {
return null;
}
@Override
public void remove() {
}
@Override
public CloneableIterator<byte[]> clone(Object modifier) {
return this;
}
};
}
}
public byte[] encodedKey(final String key) { public byte[] encodedKey(final String key) {
return Base64Order.enhancedCoder.encodeSubstring(Digest.encodeMD5Raw(key), this.table.keylength); return Base64Order.enhancedCoder.encodeSubstring(Digest.encodeMD5Raw(key), this.table.keylength);
} }
@ -487,7 +522,10 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>,
* are flushed * are flushed
*/ */
public void close() { public void close() {
int s = this.size();
File f = this.table.heapFile;
this.table.close(); this.table.close();
if (s == 0) f.delete();
} }
/** /**

338
source/net/yacy/kelondro/blob/BEncodedHeapBag.java
Unescape View File

@ -0,0 +1,338 @@
// BEncodedHeapBag.java
// (C) 2011 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 16.11.2011 on http://yacy.net
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.kelondro.blob;
import java.io.File;
import java.io.IOException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.UTF8;
import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.order.ByteOrder;
import net.yacy.kelondro.order.CloneableIterator;
import net.yacy.kelondro.order.MergeIterator;
import net.yacy.kelondro.util.FileUtils;
public class BEncodedHeapBag extends AbstractMapStore implements MapStore {
private Map<String, BEncodedHeap> bag; // a map from a date string to a kelondroIndex object
private final File baseDir;
private final String prefix;
private final int keylength, buffermax;
private final ByteOrder entryOrder;
private String current;
private final long fileAgeLimit;
private final long fileSizeLimit;
public BEncodedHeapBag(
final File path,
final String prefix,
final int keylength,
final ByteOrder ordering,
final int buffermax,
final long fileAgeLimit,
final long fileSizeLimit) {
this.baseDir = path;
this.prefix = prefix;
this.keylength = keylength;
this.buffermax = buffermax;
this.entryOrder = ordering;
this.fileAgeLimit = fileAgeLimit;
this.fileSizeLimit = fileSizeLimit;
init();
}
private void init() {
this.current = null;
// initialized tables map
this.bag = new HashMap<String, BEncodedHeap>();
if (!(this.baseDir.exists())) this.baseDir.mkdirs();
String[] tablefile = this.baseDir.list();
// first pass: find tables
final HashMap<String, Long> t = new HashMap<String, Long>();
long ram, time, maxtime = 0;
Date d;
File f;
for (final String element : tablefile) {
if ((element.startsWith(this.prefix)) &&
(element.length() > this.prefix.length()) &&
(element.charAt(this.prefix.length()) == '.') &&
(element.length() == this.prefix.length() + 23)) {
f = new File(this.baseDir, element);
try {
d = GenericFormatter.SHORT_MILSEC_FORMATTER.parse(element.substring(this.prefix.length() + 1, this.prefix.length() + 18));
} catch (final ParseException e) {
Log.logSevere("BEncodedHeapBag", "", e);
continue;
}
time = d.getTime();
if (time > maxtime) {
this.current = element;
assert this.current != null;
maxtime = time;
}
t.put(element, f.length());
}
}
// second pass: open tables
Iterator<Map.Entry<String, Long>> i;
Map.Entry<String, Long> entry;
String maxf;
long maxram;
while (!t.isEmpty()) {
// find maximum table
maxram = 0;
maxf = null;
i = t.entrySet().iterator();
while (i.hasNext()) {
entry = i.next();
ram = entry.getValue().longValue();
if (maxf == null || ram > maxram) {
maxf = entry.getKey();
maxram = ram;
}
}
// open next biggest table
t.remove(maxf);
f = new File(this.baseDir, maxf);
try {
Log.logInfo("BEncodedHeapBag", "opening partial heap " + f);
BEncodedHeap heap = new BEncodedHeap(f, this.keylength, this.entryOrder, this.buffermax);
this.bag.put(maxf, heap);
} catch (IOException e) {
Log.logSevere("BEncodedHeapBag", "error opening partial heap " + f);
}
}
}
@Override
public synchronized void close() {
if (this.bag == null) return;
final Iterator<BEncodedHeap> i = this.bag.values().iterator();
while (i.hasNext()) {
i.next().close();
}
this.bag = null;
}
@Override
public void clear() {
close();
final String[] l = this.baseDir.list();
for (final String element : l) {
if (element.startsWith(this.prefix)) {
final File f = new File(this.baseDir, element);
if (!f.isDirectory()) FileUtils.deletedelete(f);
}
}
init();
}
private MapStore keeperOf(final byte[] key) {
if (key == null) return null;
if (this.bag == null) return null;
for (final MapStore oi: this.bag.values()) {
if (oi.containsKey(key)) return oi;
}
return null;
}
private String newFilename() {
return this.prefix + "." + GenericFormatter.SHORT_MILSEC_FORMATTER.format() + ".heap";
}
private MapStore newHeap() {
this.current = newFilename();
final File f = new File(this.baseDir, this.current);
BEncodedHeap heap;
try {
heap = new BEncodedHeap(f, this.keylength, this.entryOrder, this.buffermax);
} catch (IOException e) {
Log.logSevere("BEncodedHeapBag", "unable to open new heap file: " + e.getMessage(), e);
return null;
}
this.bag.put(this.current, heap);
return heap;
}
private MapStore checkHeap(final BEncodedHeap heap) {
// check size and age of given table; in case it is too large or too old
// create a new table
assert heap != null;
long t = System.currentTimeMillis();
if (((t / 1000) % 10) != 0) return heap; // we check only every 10 seconds because all these file and parser operations are very expensive
final String name = heap.getFile().getName();
long d;
try {
d = GenericFormatter.SHORT_MILSEC_FORMATTER.parse(name.substring(this.prefix.length() + 1, this.prefix.length() + 18)).getTime();
} catch (final ParseException e) {
Log.logSevere("BEncodedHeapBag", "", e);
d = 0;
}
if (d + this.fileAgeLimit < t || new File(this.baseDir, name).length() >= this.fileSizeLimit) {
return newHeap();
}
return heap;
}
@Override
public boolean containsKey(Object key) {
if (key == null || !(key instanceof byte[])) return false;
synchronized (this.bag) {
return keeperOf((byte[]) key) != null;
}
}
@Override
public Map<String, byte[]> put(byte[] key, Map<String, byte[]> map) {
if (this.bag == null) return null;
MapStore keeper = null;
synchronized (this.bag) {
keeper = keeperOf(key);
}
if (keeper != null) {
return keeper.put(key, map);
}
synchronized (this.bag) {
keeper = keeperOf(key); // we must check that again because it could have changed in between
if (keeper != null) return keeper.put(key, map);
if (this.current == null) {
keeper = newHeap();
return keeper.put(key, map);
}
keeper = checkHeap(this.bag.get(this.current));
}
return keeper.put(key, map);
}
@Override
public Map<String, byte[]> get(Object key) {
if (key == null || !(key instanceof byte[])) return null;
synchronized (this.bag) {
return keeperOf((byte[]) key).get(key);
}
}
@Override
public boolean isEmpty() {
final Iterator<BEncodedHeap> i = this.bag.values().iterator();
while (i.hasNext()) if (!i.next().isEmpty()) return false;
return true;
}
@Override
public int size() {
final Iterator<BEncodedHeap> i = this.bag.values().iterator();
int s = 0;
while (i.hasNext()) s += i.next().size();
return s;
}
@Override
public Map<String, byte[]> remove(Object key) {
if (key == null || !(key instanceof byte[])) return null;
final MapStore heap;
synchronized (this.bag) {
heap = keeperOf((byte[]) key);
}
if (heap == null) return null;
return heap.remove(key);
}
@Override
public ByteOrder getOrdering() {
return this.entryOrder;
}
@Override
public CloneableIterator<byte[]> keyIterator() {
final List<CloneableIterator<byte[]>> c = new ArrayList<CloneableIterator<byte[]>>(this.bag.size());
final Iterator<BEncodedHeap> i = this.bag.values().iterator();
CloneableIterator<byte[]> k;
while (i.hasNext()) {
k = i.next().keyIterator();
if (k != null && k.hasNext()) c.add(k);
}
return MergeIterator.cascade(c, this.entryOrder, MergeIterator.simpleMerge, true);
}
protected static Map<String, byte[]> testMap(int i) {
HashMap<String, byte[]> t = new HashMap<String, byte[]>();
t.put("rdf:about", UTF8.getBytes("http://abc.de/testmap#" + i));
t.put("dc:title", UTF8.getBytes("test nr " + i));
return t;
}
private static BEncodedHeapBag testHeapBag(File f) {
return new BEncodedHeapBag(
f,
"testbag",
12,
Base64Order.enhancedCoder,
10,
ArrayStack.oneMonth, 100 /*Integer.MAX_VALUE*/);
}
public static void main(String[] args) {
File f = new File("/tmp");
BEncodedHeapBag hb = testHeapBag(f);
for (int i = 0; i < 10000; i++) {
hb.put(Word.word2hash(Integer.toString(i)), testMap(i));
}
System.out.println("test size after put = " + hb.size());
hb.close();
hb = testHeapBag(f);
Iterator<Map.Entry<byte[], Map<String, byte[]>>> mi = hb.iterator();
int c = 1000;
Map.Entry<byte[], Map<String, byte[]>> entry;
while (mi.hasNext() && c-- > 0) {
entry = mi.next();
System.out.println(UTF8.String(entry.getKey()) + ": " + AbstractMapStore.map2String(entry.getValue()));
}
for (int i = 10000; i > 0; i--) {
hb.remove(Word.word2hash(Integer.toString(i - 1)));
}
System.out.println("test size after remove = " + hb.size());
hb.close();
Log.shutdown();
}
}

296
source/net/yacy/kelondro/blob/BEncodedHeapShard.java
Unescape View File

@ -0,0 +1,296 @@
// BEncodedHeapShard.java
// (C) 2011 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 16.11.2011 on http://yacy.net
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.kelondro.blob;
import java.io.File;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.UTF8;
import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.order.ByteOrder;
import net.yacy.kelondro.order.CloneableIterator;
import net.yacy.kelondro.order.MergeIterator;
import net.yacy.kelondro.util.FileUtils;
public class BEncodedHeapShard extends AbstractMapStore implements MapStore {
public interface Method {
/**
* a sharding method produces a filename from a given key
* @param key
* @return
*/
public String filename(byte[] key);
/**
* get the maximum key length for access keys
* @return
*/
public int getKeylength();
/**
* get the byte order on the keys
* @return
*/
public ByteOrder getOrdering();
/**
* check if the given file name is a part of the shard
* @param filename
* @return true if the file is part of the shar
*/
public boolean isShardPart(String filename);
public String getShardName(String filename);
}
public static class B64ShardMethod implements Method {
private final int keylength;
private final ByteOrder ordering;
private final byte[] template;
private final int charpos;
private final String prefix;
public B64ShardMethod(
final int keylength,
final ByteOrder ordering,
final String prefix) {
this.keylength = keylength;
this.ordering = ordering;
this.template = ASCII.getBytes(prefix + ".?");
this.charpos = ASCII.getBytes(prefix).length + 1;
this.prefix = prefix;
}
@Override
public String filename(byte[] key) {
byte[] s = new byte[this.template.length];
System.arraycopy(this.template, 0, s, 0, s.length);
s[this.charpos] = key[0];
return ASCII.String(s);
}
@Override
public int getKeylength() {
return this.keylength;
}
@Override
public ByteOrder getOrdering() {
return this.ordering;
}
@Override
public boolean isShardPart(String filename) {
// TODO Auto-generated method stub
return filename.startsWith(this.prefix) &&
filename.charAt(this.prefix.length()) == '.' &&
filename.endsWith(".heap");
}
public String getShardName(String filename) {
return filename.substring(0, this.template.length);
}
}
private ConcurrentHashMap<String, MapStore> shard;
private File baseDir;
private Method shardMethod;
public BEncodedHeapShard(File baseDir, Method shardMethod) {
this.shard = new ConcurrentHashMap<String, MapStore>();
this.baseDir = baseDir;
this.shardMethod = shardMethod;
init();
}
private void init() {
// initialized tables map
this.shard = new ConcurrentHashMap<String, MapStore>();
if (!(this.baseDir.exists())) this.baseDir.mkdirs();
String[] tablefile = this.baseDir.list();
// open all tables of this shard
for (final String element : tablefile) {
if (this.shardMethod.isShardPart(element)) {
Log.logInfo("BEncodedHeapShard", "opening partial shard " + element);
MapStore bag = openBag(element);
this.shard.put(this.shardMethod.getShardName(element), bag);
}
}
}
@Override
public void close() {
if (this.shard == null) return;
final Iterator<MapStore> i = this.shard.values().iterator();
while (i.hasNext()) {
i.next().close();
}
this.shard = null;
}
@Override
public void clear() {
close();
final String[] l = this.baseDir.list();
for (final String element : l) {
if (this.shardMethod.isShardPart(element)) {
final File f = new File(this.baseDir, element);
if (!f.isDirectory()) FileUtils.deletedelete(f);
}
}
init();
}
private MapStore keeperOf(final byte[] key) {
String shardfile = this.shardMethod.filename(key);
MapStore bag = this.shard.get(shardfile);
if (bag != null) return bag;
bag = openBag(shardfile);
this.shard.put(shardfile, bag);
return bag;
}
public MapStore openBag(String shardfile) {
MapStore bag = new BEncodedHeapBag(
this.baseDir,
shardfile,
this.shardMethod.getKeylength(),
this.shardMethod.getOrdering(),
10,
ArrayStack.oneMonth * 12,
Integer.MAX_VALUE);
return bag;
}
@Override
public boolean containsKey(Object key) {
if (key == null || !(key instanceof byte[])) return false;
String shardfile = this.shardMethod.filename((byte[]) key);
MapStore bag = this.shard.get(shardfile);
if (bag == null) return false;
return bag.containsKey(key);
}
@Override
public Map<String, byte[]> put(byte[] key, Map<String, byte[]> map) {
if (this.shard == null) return null;
MapStore keeper = null;
synchronized (this.shard) {
keeper = keeperOf(key);
}
return keeper.put(key, map);
}
@Override
public Map<String, byte[]> get(Object key) {
if (key == null || !(key instanceof byte[])) return null;
String shardfile = this.shardMethod.filename((byte[]) key);
MapStore bag = this.shard.get(shardfile);
if (bag == null) return null;
return bag.get(key);
}
@Override
public boolean isEmpty() {
final Iterator<MapStore> i = this.shard.values().iterator();
while (i.hasNext()) if (!i.next().isEmpty()) return false;
return true;
}
@Override
public int size() {
final Iterator<MapStore> i = this.shard.values().iterator();
int s = 0;
while (i.hasNext()) s += i.next().size();
return s;
}
@Override
public Map<String, byte[]> remove(Object key) {
if (key == null || !(key instanceof byte[])) return null;
final MapStore bag;
synchronized (this.shard) {
bag = keeperOf((byte[]) key);
}
if (bag == null) return null;
return bag.remove(key);
}
@Override
public ByteOrder getOrdering() {
return this.shardMethod.getOrdering();
}
@Override
public CloneableIterator<byte[]> keyIterator() {
final List<CloneableIterator<byte[]>> c = new ArrayList<CloneableIterator<byte[]>>(this.shard.size());
final Iterator<MapStore> i = this.shard.values().iterator();
CloneableIterator<byte[]> k;
while (i.hasNext()) {
k = i.next().keyIterator();
if (k != null && k.hasNext()) c.add(k);
}
return MergeIterator.cascade(c, this.shardMethod.getOrdering(), MergeIterator.simpleMerge, true);
}
private static BEncodedHeapShard testHeapShard(File f) {
return new BEncodedHeapShard(f, new B64ShardMethod(12, Base64Order.enhancedCoder, "testshard"));
}
public static void main(String[] args) {
File f = new File("/tmp");
BEncodedHeapShard hb = testHeapShard(f);
for (int i = 0; i < 10000; i++) {
hb.put(Word.word2hash(Integer.toString(i)), BEncodedHeapBag.testMap(i));
}
System.out.println("test size after put = " + hb.size());
hb.close();
hb = testHeapShard(f);
Iterator<Map.Entry<byte[], Map<String, byte[]>>> mi = hb.iterator();
int c = 100;
Map.Entry<byte[], Map<String, byte[]>> entry;
while (mi.hasNext() && c-- > 0) {
entry = mi.next();
System.out.println(UTF8.String(entry.getKey()) + ": " + AbstractMapStore.map2String(entry.getValue()));
}
for (int i = 10000; i > 0; i--) {
hb.remove(Word.word2hash(Integer.toString(i - 1)));
}
System.out.println("test size after remove = " + hb.size());
hb.close();
Log.shutdown();
}
}

56
source/net/yacy/kelondro/blob/MapStore.java
Unescape View File

@ -0,0 +1,56 @@
// MapMap.java
// (C) 2011 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 16.11.2011 on http://yacy.net
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.kelondro.blob;
import java.util.Map;
import net.yacy.kelondro.order.ByteOrder;
import net.yacy.kelondro.order.CloneableIterator;
/**
* this is a placeholder interface
* for the complex expressionMap<byte[], Map<String, byte[]>>
*
*/
public interface MapStore extends Map<byte[], Map<String, byte[]>>, Iterable<Map.Entry<byte[], Map<String, byte[]>>> {
/**
* the map should have an ordering on the key elements
* @return a byte order on the key elements
*/
public ByteOrder getOrdering();
/**
* the keys of the map should be iterable
* @return an iterator on the map keys
*/
public CloneableIterator<byte[]> keyIterator();
/**
* most of the MapMap implementations are file-based, so we should consider a close method
*/
public void close();
}
Write Preview
Loading…
Cancel
Save