boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

better memory management and slightly less (in total and temporary) RAM allocation:

Browse Source 
- confirm that database objects that are not supposed to grow do not have a index memory management that is designed for growth
- changed index sorting method in such a way that it allocates less objects during quicksort
- database classes classes renaming (shorter, naming addresses that objects hold in RAM)
- added a large number of asserts to check if objects actually take the RAM that they should have


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7019 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent 5924a0d851
commit 6388a58fc7
26 changed files with 482 additions and 366 deletions
Show Stats Download Patch File Download Diff File

8
htroot/PerformanceMemory_p.java
Unescape View File

@ -30,7 +30,7 @@ import java.util.Iterator;
import java.util.Map; import java.util.Map;
import net.yacy.kelondro.index.Cache; import net.yacy.kelondro.index.Cache;
import net.yacy.kelondro.index.ObjectIndexCache; import net.yacy.kelondro.index.RAMIndex;
import net.yacy.kelondro.table.Table; import net.yacy.kelondro.table.Table;
import net.yacy.kelondro.util.Domains; import net.yacy.kelondro.util.Domains;
import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.FileUtils;
@ -118,11 +118,11 @@ public class PerformanceMemory_p {
prop.putNum("EcoIndexTotalMem", totalmem / (1024 * 1024d)); prop.putNum("EcoIndexTotalMem", totalmem / (1024 * 1024d));
// write object cache table // write object cache table
Iterator<Map.Entry<String, ObjectIndexCache>> oi = ObjectIndexCache.objects(); Iterator<Map.Entry<String, RAMIndex>> oi = RAMIndex.objects();
c = 0; c = 0;
mem = 0; mem = 0;
Map.Entry<String, ObjectIndexCache> oie; Map.Entry<String, RAMIndex> oie;
ObjectIndexCache cache; RAMIndex cache;
long hitmem, totalhitmem = 0; long hitmem, totalhitmem = 0;
while (oi.hasNext()) { while (oi.hasNext()) {
oie = oi.next(); oie = oi.next();

4
source/de/anomic/crawler/ZURL.java
Unescape View File

@ -35,7 +35,7 @@ import java.util.concurrent.ConcurrentLinkedQueue;
import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.index.ObjectIndex; import net.yacy.kelondro.index.Index;
import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.Row;
import net.yacy.kelondro.index.RowSet; import net.yacy.kelondro.index.RowSet;
import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.index.RowSpaceExceededException;
@ -63,7 +63,7 @@ public class ZURL implements Iterable<ZURL.Entry> {
); );
// the class object // the class object
protected ObjectIndex urlIndex; protected Index urlIndex;
protected final ConcurrentLinkedQueue<byte[]> stack; protected final ConcurrentLinkedQueue<byte[]> stack;
public ZURL( public ZURL(

2
source/de/anomic/http/client/Cache.java
Unescape View File

@ -84,7 +84,7 @@ public final class Cache {
Log.logException(e); Log.logException(e);
} }
try { try {
fileDBunbuffered = new ArrayStack(new File(cachePath, FILE_DB_NAME), prefix, 12, Base64Order.enhancedCoder, 1024 * 1024 * 2); fileDBunbuffered = new ArrayStack(new File(cachePath, FILE_DB_NAME), prefix, 12, Base64Order.enhancedCoder, 1024 * 1024 * 2, false);
fileDBunbuffered.setMaxSize(maxCacheSize); fileDBunbuffered.setMaxSize(maxCacheSize);
fileDB = new Compressor(fileDBunbuffered, 2 * 1024 * 1024); fileDB = new Compressor(fileDBunbuffered, 2 * 1024 * 1024);
} catch (IOException e) { } catch (IOException e) {

6
source/de/anomic/search/MetadataRepository.java
Unescape View File

@ -46,7 +46,7 @@ import net.yacy.kelondro.data.meta.URIMetadataRow;
import net.yacy.kelondro.data.word.WordReferenceVars; import net.yacy.kelondro.data.word.WordReferenceVars;
import net.yacy.kelondro.index.Cache; import net.yacy.kelondro.index.Cache;
import net.yacy.kelondro.index.HandleSet; import net.yacy.kelondro.index.HandleSet;
import net.yacy.kelondro.index.ObjectIndex; import net.yacy.kelondro.index.Index;
import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.Row;
import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
@ -61,7 +61,7 @@ import net.yacy.repository.Blacklist;
public final class MetadataRepository implements Iterable<byte[]> { public final class MetadataRepository implements Iterable<byte[]> {
// class objects // class objects
protected ObjectIndex urlIndexFile; protected Index urlIndexFile;
private Export exportthread; // will have a export thread assigned if exporter is running private Export exportthread; // will have a export thread assigned if exporter is running
private File location; private File location;
private ArrayList<hostStat> statsDump; private ArrayList<hostStat> statsDump;
@ -72,7 +72,7 @@ public final class MetadataRepository implements Iterable<byte[]> {
final boolean useTailCache, final boolean useTailCache,
final boolean exceed134217727) { final boolean exceed134217727) {
this.location = path; this.location = path;
ObjectIndex backupIndex = null; Index backupIndex = null;
try { try {
backupIndex = new SplitTable(this.location, tablename, URIMetadataRow.rowdef, useTailCache, exceed134217727); backupIndex = new SplitTable(this.location, tablename, URIMetadataRow.rowdef, useTailCache, exceed134217727);
} catch (RowSpaceExceededException e) { } catch (RowSpaceExceededException e) {

6
source/de/anomic/search/blockrank/CRProcess.java
Unescape View File

@ -32,7 +32,7 @@ import java.util.Date;
import java.util.Iterator; import java.util.Iterator;
import net.yacy.kelondro.data.word.WordReference; import net.yacy.kelondro.data.word.WordReference;
import net.yacy.kelondro.index.ObjectIndex; import net.yacy.kelondro.index.Index;
import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.Row;
import net.yacy.kelondro.index.RowSet; import net.yacy.kelondro.index.RowSet;
import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.index.RowSpaceExceededException;
@ -169,7 +169,7 @@ public class CRProcess {
return true; return true;
} }
public static boolean accumulate_upd(final File f, final ObjectIndex acc) throws IOException, RowSpaceExceededException { public static boolean accumulate_upd(final File f, final Index acc) throws IOException, RowSpaceExceededException {
// open file // open file
AttrSeq source_cr = null; AttrSeq source_cr = null;
try { try {
@ -275,7 +275,7 @@ public class CRProcess {
// open target file // open target file
AttrSeq acc = null; AttrSeq acc = null;
ObjectIndex newacc = null; Index newacc = null;
IndexCell<WordReference> newseq = null; IndexCell<WordReference> newseq = null;
if (newdb) { if (newdb) {
final File path = to_file.getParentFile(); // path to storage place final File path = to_file.getParentFile(); // path to storage place

4
source/de/anomic/yacy/yacyNewsDB.java
Unescape View File

@ -55,7 +55,7 @@ import java.util.Properties;
import java.util.Map.Entry; import java.util.Map.Entry;
import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.index.ObjectIndex; import net.yacy.kelondro.index.Index;
import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.Row;
import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
@ -73,7 +73,7 @@ public class yacyNewsDB {
private final File path; private final File path;
private final Row rowdef; private final Row rowdef;
protected final int attributesMaxLength; protected final int attributesMaxLength;
protected ObjectIndex news; protected Index news;
private static final int categoryStringLength = 8; private static final int categoryStringLength = 8;
public static final int idLength = DateFormatter.PATTERN_SHORT_SECOND.length() + Word.commonHashLength; public static final int idLength = DateFormatter.PATTERN_SHORT_SECOND.length() + Word.commonHashLength;

24
source/net/yacy/dbtest.java
Unescape View File

@ -11,7 +11,7 @@ import java.util.Random;
import javax.imageio.ImageIO; import javax.imageio.ImageIO;
import net.yacy.kelondro.index.ObjectIndex; import net.yacy.kelondro.index.Index;
import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.Row;
import net.yacy.kelondro.index.RowSet; import net.yacy.kelondro.index.RowSet;
import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.index.RowSpaceExceededException;
@ -89,21 +89,21 @@ public class dbtest {
} }
public static abstract class STJob implements Runnable { public static abstract class STJob implements Runnable {
private final ObjectIndex table_test, table_reference; private final Index table_test, table_reference;
private final long source; private final long source;
public STJob(final ObjectIndex table_test, final ObjectIndex table_reference, final long aSource) { public STJob(final Index table_test, final Index table_reference, final long aSource) {
this.table_test = table_test; this.table_test = table_test;
this.table_reference = table_reference; this.table_reference = table_reference;
this.source = aSource; this.source = aSource;
} }
public ObjectIndex getTable_test() { public Index getTable_test() {
return this.table_test; return this.table_test;
} }
public ObjectIndex getTable_reference() { public Index getTable_reference() {
return this.table_reference; return this.table_reference;
} }
@ -115,7 +115,7 @@ public class dbtest {
} }
public static final class WriteJob extends STJob { public static final class WriteJob extends STJob {
public WriteJob(final ObjectIndex table_test, final ObjectIndex table_reference, final long aSource) { public WriteJob(final Index table_test, final Index table_reference, final long aSource) {
super(table_test, table_reference, aSource); super(table_test, table_reference, aSource);
} }
@ -138,7 +138,7 @@ public class dbtest {
} }
public static final class RemoveJob extends STJob { public static final class RemoveJob extends STJob {
public RemoveJob(final ObjectIndex table_test, final ObjectIndex table_reference, final long aSource) { public RemoveJob(final Index table_test, final Index table_reference, final long aSource) {
super(table_test, table_reference, aSource); super(table_test, table_reference, aSource);
} }
@ -157,7 +157,7 @@ public class dbtest {
} }
public static final class ReadJob extends STJob { public static final class ReadJob extends STJob {
public ReadJob(final ObjectIndex table_test, final ObjectIndex table_reference, final long aSource) { public ReadJob(final Index table_test, final Index table_reference, final long aSource) {
super(table_test, table_reference, aSource); super(table_test, table_reference, aSource);
} }
@ -194,7 +194,7 @@ public class dbtest {
} }
} }
public static ObjectIndex selectTableType(final String dbe, final String tablename, final Row testRow) throws Exception { public static Index selectTableType(final String dbe, final String tablename, final Row testRow) throws Exception {
if (dbe.equals("kelondroRowSet")) { if (dbe.equals("kelondroRowSet")) {
return new RowSet(testRow, 0); return new RowSet(testRow, 0);
} }
@ -214,7 +214,7 @@ public class dbtest {
return null; return null;
} }
public static boolean checkEquivalence(final ObjectIndex test, final ObjectIndex reference) throws IOException { public static boolean checkEquivalence(final Index test, final Index reference) throws IOException {
if (reference == null) return true; if (reference == null) return true;
if (test.size() == reference.size()) { if (test.size() == reference.size()) {
System.out.println("* Testing equivalence of test table to reference table, " + test.size() + " entries"); System.out.println("* Testing equivalence of test table to reference table, " + test.size() + " entries");
@ -288,8 +288,8 @@ public class dbtest {
// create the database access // create the database access
final Row testRow = new Row("byte[] key-" + keylength + ", byte[] dummy-" + keylength + ", value-" + valuelength, Base64Order.enhancedCoder); final Row testRow = new Row("byte[] key-" + keylength + ", byte[] dummy-" + keylength + ", value-" + valuelength, Base64Order.enhancedCoder);
final ObjectIndex table_test = selectTableType(dbe_test, tablename_test, testRow); final Index table_test = selectTableType(dbe_test, tablename_test, testRow);
final ObjectIndex table_reference = (dbe_reference == null) ? null : selectTableType(dbe_reference, tablename_reference, testRow); final Index table_reference = (dbe_reference == null) ? null : selectTableType(dbe_reference, tablename_reference, testRow);
final long afterinit = System.currentTimeMillis(); final long afterinit = System.currentTimeMillis();
System.out.println("Test for db-engine " + dbe_test + " started to create file " + tablename_test + " with test " + command); System.out.println("Test for db-engine " + dbe_test + " started to create file " + tablename_test + " with test " + command);

72
source/net/yacy/kelondro/blob/ArrayStack.java
Unescape View File

@ -91,6 +91,7 @@ public class ArrayStack implements BLOB {
protected List<blobItem> blobs; protected List<blobItem> blobs;
private final String prefix; private final String prefix;
private final int buffersize; private final int buffersize;
private final boolean trimall;
// the thread pool for the keeperOf executor service // the thread pool for the keeperOf executor service
private final ExecutorService executor; private final ExecutorService executor;
@ -100,7 +101,8 @@ public class ArrayStack implements BLOB {
final String prefix, final String prefix,
final int keylength, final int keylength,
final ByteOrder ordering, final ByteOrder ordering,
final int buffersize) throws IOException { final int buffersize,
final boolean trimall) throws IOException {
this.keylength = keylength; this.keylength = keylength;
this.prefix = prefix; this.prefix = prefix;
this.ordering = ordering; this.ordering = ordering;
@ -110,6 +112,7 @@ public class ArrayStack implements BLOB {
this.fileSizeLimit = (long) Integer.MAX_VALUE; this.fileSizeLimit = (long) Integer.MAX_VALUE;
this.repositoryAgeMax = Long.MAX_VALUE; this.repositoryAgeMax = Long.MAX_VALUE;
this.repositorySizeMax = Long.MAX_VALUE; this.repositorySizeMax = Long.MAX_VALUE;
this.trimall = trimall;
// init the thread pool for the keeperOf executor service // init the thread pool for the keeperOf executor service
this.executor = new ThreadPoolExecutor( this.executor = new ThreadPoolExecutor(
@ -187,7 +190,12 @@ public class ArrayStack implements BLOB {
d = DateFormatter.parseShortMilliSecond(files[i].substring(prefix.length() + 1, prefix.length() + 18)); d = DateFormatter.parseShortMilliSecond(files[i].substring(prefix.length() + 1, prefix.length() + 18));
f = new File(heapLocation, files[i]); f = new File(heapLocation, files[i]);
time = d.getTime(); time = d.getTime();
oneBlob = (time == maxtime) ? new Heap(f, keylength, ordering, buffersize) : new HeapModifier(f, keylength, ordering); if (time == maxtime && !trimall) {
oneBlob = new Heap(f, keylength, ordering, buffersize);
} else {
oneBlob = new HeapModifier(f, keylength, ordering);
oneBlob.trim(); // no writings here, can be used with minimum memory
}
sortedItems.put(Long.valueOf(time), new blobItem(d, f, oneBlob)); sortedItems.put(Long.valueOf(time), new blobItem(d, f, oneBlob));
} catch (ParseException e) {continue;} } catch (ParseException e) {continue;}
} }
@ -200,6 +208,19 @@ public class ArrayStack implements BLOB {
} }
} }
public long mem() {
long m = 0;
for (blobItem b: this.blobs) m += b.blob.mem();
return m;
}
public void trim() {
// trim shall not be called for ArrayStacks because the characteristics of an ArrayStack is that the 'topmost' BLOB on the stack
// is used for write operations and all other shall be trimmed automatically since they are not used for writing. And the
// topmost BLOB must not be trimmed to support fast writings.
throw new UnsupportedOperationException();
}
/** /**
* add a blob file to the array. * add a blob file to the array.
* note that this file must be generated with a file name from newBLOB() * note that this file must be generated with a file name from newBLOB()
@ -213,7 +234,13 @@ public class ArrayStack implements BLOB {
} catch (ParseException e) { } catch (ParseException e) {
throw new IOException("date parse problem with file " + location.toString() + ": " + e.getMessage()); throw new IOException("date parse problem with file " + location.toString() + ": " + e.getMessage());
} }
BLOB oneBlob = (full && buffersize > 0) ? new Heap(location, keylength, ordering, buffersize) : new HeapModifier(location, keylength, ordering); BLOB oneBlob;
if (full && buffersize > 0 && !trimall) {
oneBlob = new Heap(location, keylength, ordering, buffersize);
} else {
oneBlob = new HeapModifier(location, keylength, ordering);
oneBlob.trim();
}
blobs.add(new blobItem(d, location, oneBlob)); blobs.add(new blobItem(d, location, oneBlob));
} }
@ -321,25 +348,6 @@ public class ArrayStack implements BLOB {
return unmount(idx); return unmount(idx);
} }
/*
public synchronized File unmountSimilarSizeBLOB(long otherSize) {
if (this.blobs.isEmpty() || otherSize == 0) return null;
blobItem b;
double delta, bestDelta = Double.MAX_VALUE;
int bestIndex = -1;
for (int i = 0; i < this.blobs.size(); i++) {
b = this.blobs.get(i);
if (b.location.length() == 0) continue;
delta = ((double) b.location.length()) / ((double) otherSize);
if (delta < 1.0) delta = 1.0 / delta;
if (delta < bestDelta) {
bestDelta = delta;
bestIndex = i;
}
}
return unmount(bestIndex);
}
*/
/** /**
* return the number of BLOB files in this array * return the number of BLOB files in this array
* @return * @return
@ -684,7 +692,7 @@ public class ArrayStack implements BLOB {
} }
/** /**
* replace a BLOB entry with another which must be smaller or same size * replace a BLOB entry with another
* @param key the primary key * @param key the primary key
* @throws IOException * @throws IOException
* @throws RowSpaceExceededException * @throws RowSpaceExceededException
@ -697,13 +705,29 @@ public class ArrayStack implements BLOB {
return d; return d;
} }
/**
* replace a BLOB entry with another which must be smaller or same size
* @param key the primary key
* @throws IOException
* @throws RowSpaceExceededException
*/
public synchronized int reduce(byte[] key, Reducer reduce) throws IOException, RowSpaceExceededException {
int d = 0;
for (blobItem bi: blobs) {
d += bi.blob.reduce(key, reduce);
}
return d;
}
/** /**
* remove a BLOB * remove a BLOB
* @param key the primary key * @param key the primary key
* @throws IOException * @throws IOException
*/ */
public synchronized void remove(byte[] key) throws IOException { public synchronized void remove(byte[] key) throws IOException {
long m = this.mem();
for (blobItem bi: blobs) bi.blob.remove(key); for (blobItem bi: blobs) bi.blob.remove(key);
assert this.mem() <= m : "m = " + m + ", mem() = " + mem();
} }
/** /**
@ -971,7 +995,7 @@ public class ArrayStack implements BLOB {
final File f = new File("/Users/admin/blobarraytest"); final File f = new File("/Users/admin/blobarraytest");
try { try {
//f.delete(); //f.delete();
final ArrayStack heap = new ArrayStack(f, "test", 12, NaturalOrder.naturalOrder, 512 * 1024); final ArrayStack heap = new ArrayStack(f, "test", 12, NaturalOrder.naturalOrder, 512 * 1024, false);
heap.put("aaaaaaaaaaaa".getBytes(), "eins zwei drei".getBytes()); heap.put("aaaaaaaaaaaa".getBytes(), "eins zwei drei".getBytes());
heap.put("aaaaaaaaaaab".getBytes(), "vier fuenf sechs".getBytes()); heap.put("aaaaaaaaaaab".getBytes(), "vier fuenf sechs".getBytes());
heap.put("aaaaaaaaaaac".getBytes(), "sieben acht neun".getBytes()); heap.put("aaaaaaaaaaac".getBytes(), "sieben acht neun".getBytes());

34
source/net/yacy/kelondro/blob/BLOB.java
Unescape View File

@ -59,6 +59,18 @@ public interface BLOB {
*/ */
public void clear() throws IOException; public void clear() throws IOException;
/**
* trim the index of the database: this releases memory not currently used
* @throws IOException
*/
public void trim();
/**
* calculate the memory in RAM that the BLOB occupies
* @return number of bytes that is used
*/
public long mem();
/** /**
* ask for the number of entries * ask for the number of entries
* @return the number of entries in the table * @return the number of entries in the table
@ -139,12 +151,22 @@ public interface BLOB {
* It is therefore necessary that it is known that the new entry will be smaller than the * It is therefore necessary that it is known that the new entry will be smaller than the
* old entry before calling this method. * old entry before calling this method.
* @param key the primary key * @param key the primary key
* @param b * @param rewriter
* @return the number of bytes that the rewriter reduced the BLOB * @return the number of bytes that the rewriter reduced the BLOB
* @throws IOException * @throws IOException
* @throws RowSpaceExceededException * @throws RowSpaceExceededException
*/ */
public int replace(byte[] key, Rewriter rewriter) throws IOException, RowSpaceExceededException; public int replace(byte[] key, Rewriter rewriter) throws IOException, RowSpaceExceededException;
/**
* a reduce method is the same as the replace. A replace subsumes a reduce method. A reduce method may be more simple.
* @param key the primary key
* @param reducer
* @return the number of bytes that the rewriter reduced the BLOB
* @throws IOException
* @throws RowSpaceExceededException
*/
public int reduce(byte[] key, Reducer reducer) throws IOException, RowSpaceExceededException;
/** /**
* remove a BLOB * remove a BLOB
@ -171,4 +193,14 @@ public interface BLOB {
} }
public interface Reducer extends Rewriter {
/**
* A Reducer is a rewriter that reduced the content. There are no additional methods in this interface.
* The interface shall be used in replacement of a Rewriter to simply state the fact that the rewritement
* also reduces the content of a BLOB entry or may also keep the size the same;
*/
}
} }

19
source/net/yacy/kelondro/blob/Compressor.java
Unescape View File

@ -70,6 +70,14 @@ public class Compressor implements BLOB {
initBuffer(); initBuffer();
} }
public long mem() {
return backend.mem();
}
public void trim() {
this.backend.trim();
}
private static class Entity implements Map.Entry<String, byte[]> { private static class Entity implements Map.Entry<String, byte[]> {
private String key; private String key;
private byte[] payload; private byte[] payload;
@ -361,5 +369,16 @@ public class Compressor implements BLOB {
this.put(key, c); this.put(key, c);
return reduction; return reduction;
} }
public int reduce(byte[] key, Reducer reducer) throws IOException, RowSpaceExceededException {
byte[] b = get(key);
if (b == null) return 0;
byte[] c = reducer.rewrite(b);
int reduction = c.length - b.length;
assert reduction >= 0;
if (reduction == 0) return 0;
this.put(key, c);
return reduction;
}
} }

20
source/net/yacy/kelondro/blob/HeapModifier.java
Unescape View File

@ -239,14 +239,20 @@ public class HeapModifier extends HeapReader implements BLOB {
} }
public int replace(byte[] key, final Rewriter rewriter) throws IOException { public int replace(byte[] key, final Rewriter rewriter) throws IOException {
key = normalizeKey(key); throw new UnsupportedOperationException();
assert key.length == this.keylength; }
// pre-check before synchronization public int reduce(byte[] key, final Reducer reducer) throws IOException {
key = normalizeKey(key);
assert key.length == this.keylength;
// pre-check before synchronization
long pos = index.get(key); long pos = index.get(key);
if (pos < 0) return 0; if (pos < 0) return 0;
synchronized (this) { synchronized (this) {
long m = this.mem();
// check again if the index contains the key // check again if the index contains the key
pos = index.get(key); pos = index.get(key);
if (pos < 0) return 0; if (pos < 0) return 0;
@ -271,7 +277,7 @@ public class HeapModifier extends HeapReader implements BLOB {
file.readFully(blob, 0, blob.length); file.readFully(blob, 0, blob.length);
// rewrite the entry // rewrite the entry
blob = rewriter.rewrite(blob); blob = reducer.rewrite(blob);
int reduction = len - blob.length; int reduction = len - blob.length;
if (reduction == 0) { if (reduction == 0) {
// even if the reduction is zero then it is still be possible that the record has been changed // even if the reduction is zero then it is still be possible that the record has been changed
@ -302,8 +308,10 @@ public class HeapModifier extends HeapReader implements BLOB {
// add a new free entry // add a new free entry
this.free.put(pos + 4 + blob.length + key.length, newfreereclen); this.free.put(pos + 4 + blob.length + key.length, newfreereclen);
assert mem() <= m : "m = " + m + ", mem() = " + mem();
return reduction; return reduction;
} }
} }
} }

8
source/net/yacy/kelondro/blob/HeapReader.java
Unescape View File

@ -112,7 +112,15 @@ public class HeapReader {
this.file.close(); this.file.close();
// the file will be opened again automatically when the next access to it comes. // the file will be opened again automatically when the next access to it comes.
} }
public long mem() {
return index.mem(); // don't add the memory for free here since then the asserts for memory management don't work
}
public void trim() {
this.index.trim();
}
protected byte[] normalizeKey(byte[] key) { protected byte[] normalizeKey(byte[] key) {
// check size of key: zero-filled keys are only possible of the ordering is // check size of key: zero-filled keys are only possible of the ordering is
// an instance of the natural ordering. Base64-orderings cannot use zeros in keys. // an instance of the natural ordering. Base64-orderings cannot use zeros in keys.

10
source/net/yacy/kelondro/index/BufferedObjectIndex.java
Unescape View File

@ -3,7 +3,9 @@
* Copyright 2010 by Michael Peter Christen * Copyright 2010 by Michael Peter Christen
* First released 18.4.2010 at http://yacy.net * First released 18.4.2010 at http://yacy.net
* *
* This file is part of YaCy * $LastChangedDate: 2010-06-16 17:11:21 +0200 (Mi, 16 Jun 2010) $
* $LastChangedRevision: 6922 $
* $LastChangedBy: orbiter $
* *
* This library is free software; you can redistribute it and/or * This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public * modify it under the terms of the GNU Lesser General Public
@ -37,14 +39,14 @@ import net.yacy.kelondro.order.MergeIterator;
* @author Michael Peter Christen * @author Michael Peter Christen
* *
*/ */
public class BufferedObjectIndex implements ObjectIndex, Iterable<Row.Entry> { public class BufferedObjectIndex implements Index, Iterable<Row.Entry> {
private final ObjectIndex backend; private final Index backend;
private final RowSet buffer; private final RowSet buffer;
private final int buffersize; private final int buffersize;
private final Row.EntryComparator entryComparator; private final Row.EntryComparator entryComparator;
public BufferedObjectIndex(ObjectIndex backend, int buffersize) { public BufferedObjectIndex(Index backend, int buffersize) {
this.backend = backend; this.backend = backend;
this.buffersize = buffersize; this.buffersize = buffersize;
this.buffer = new RowSet(backend.row()); this.buffer = new RowSet(backend.row());

55
source/net/yacy/kelondro/index/Cache.java
Unescape View File

@ -1,29 +1,26 @@
// Cache.java /**
// (C) 2006 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany * Cache.java
// first published 26.10.2006 on http://www.anomic.de * Copyright 2006 by Michael Peter Christen
// * First released 26.10.2006 at http://yacy.net
// This is a part of YaCy, a peer-to-peer based web search engine *
// * $LastChangedDate: 2010-06-16 17:11:21 +0200 (Mi, 16 Jun 2010) $
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ * $LastChangedRevision: 6922 $
// $LastChangedRevision: 1986 $ * $LastChangedBy: orbiter $
// $LastChangedBy: orbiter $ *
// * This library is free software; you can redistribute it and/or
// LICENSE * modify it under the terms of the GNU Lesser General Public
// * License as published by the Free Software Foundation; either
// This program is free software; you can redistribute it and/or modify * version 2.1 of the License, or (at your option) any later version.
// it under the terms of the GNU General Public License as published by *
// the Free Software Foundation; either version 2 of the License, or * This library is distributed in the hope that it will be useful,
// (at your option) any later version. * but WITHOUT ANY WARRANTY; without even the implied warranty of
// * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// This program is distributed in the hope that it will be useful, * Lesser General Public License for more details.
// but WITHOUT ANY WARRANTY; without even the implied warranty of *
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * You should have received a copy of the GNU Lesser General Public License
// GNU General Public License for more details. * along with this program in the file lgpl21.txt
// * If not, see <http://www.gnu.org/licenses/>.
// You should have received a copy of the GNU General Public License */
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.kelondro.index; package net.yacy.kelondro.index;
@ -41,7 +38,7 @@ import net.yacy.kelondro.order.CloneableIterator;
import net.yacy.kelondro.util.MemoryControl; import net.yacy.kelondro.util.MemoryControl;
public final class Cache implements ObjectIndex, Iterable<Row.Entry> { public final class Cache implements Index, Iterable<Row.Entry> {
// this is a combined read cache and write buffer // this is a combined read cache and write buffer
// we maintain four tables: // we maintain four tables:
@ -57,7 +54,7 @@ public final class Cache implements ObjectIndex, Iterable<Row.Entry> {
private static final long memStartShrink = 20 * 1024 * 1024; // a limit for the node cache to start with shrinking if less than this memory amount is available private static final long memStartShrink = 20 * 1024 * 1024; // a limit for the node cache to start with shrinking if less than this memory amount is available
// class objects // class objects
private final ObjectIndex index; // the back-end of the cache private final Index index; // the back-end of the cache
private RowSet readHitCache; // contains a complete copy of the cached objects private RowSet readHitCache; // contains a complete copy of the cached objects
private RowSet readMissCache; // contains only the keys of the objects that had been a miss private RowSet readMissCache; // contains only the keys of the objects that had been a miss
private Row keyrow; private Row keyrow;
@ -72,7 +69,7 @@ public final class Cache implements ObjectIndex, Iterable<Row.Entry> {
* @param hitLimit a limit of cache hit entries. If given as value <= 0, then only the RAM limits the size * @param hitLimit a limit of cache hit entries. If given as value <= 0, then only the RAM limits the size
* @param missLimit a limit of cache miss entries. If given as value <= 0, then only the RAM limits the size * @param missLimit a limit of cache miss entries. If given as value <= 0, then only the RAM limits the size
*/ */
public Cache(final ObjectIndex backupIndex, final int hitLimit, final int missLimit) { public Cache(final Index backupIndex, final int hitLimit, final int missLimit) {
this.index = backupIndex; this.index = backupIndex;
this.hitLimit = hitLimit; this.hitLimit = hitLimit;
this.missLimit = missLimit; this.missLimit = missLimit;

61
source/net/yacy/kelondro/index/HandleMap.java
Unescape View File

@ -1,26 +1,26 @@
// HandleMap.java /**
// (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany * HandleMap
// first published 08.04.2008 on http://yacy.net * Copyright 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// * First released 08.04.2008 at http://yacy.net
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ *
// $LastChangedRevision: 1986 $ * $LastChangedDate: 2010-06-16 17:11:21 +0200 (Mi, 16 Jun 2010) $
// $LastChangedBy: orbiter $ * $LastChangedRevision: 6922 $
// * $LastChangedBy: orbiter $
// LICENSE *
// * This library is free software; you can redistribute it and/or
// This program is free software; you can redistribute it and/or modify * modify it under the terms of the GNU Lesser General Public
// it under the terms of the GNU General Public License as published by * License as published by the Free Software Foundation; either
// the Free Software Foundation; either version 2 of the License, or * version 2.1 of the License, or (at your option) any later version.
// (at your option) any later version. *
// * This library is distributed in the hope that it will be useful,
// This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of
// but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * Lesser General Public License for more details.
// GNU General Public License for more details. *
// * You should have received a copy of the GNU Lesser General Public License
// You should have received a copy of the GNU General Public License * along with this program in the file lgpl21.txt
// along with this program; if not, write to the Free Software * If not, see <http://www.gnu.org/licenses/>.
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */
package net.yacy.kelondro.index; package net.yacy.kelondro.index;
@ -53,7 +53,7 @@ import net.yacy.kelondro.order.CloneableIterator;
public final class HandleMap implements Iterable<Row.Entry> { public final class HandleMap implements Iterable<Row.Entry> {
private final Row rowdef; private final Row rowdef;
private RowSetArray index; private RAMIndexCluster index;
/** /**
* initialize a HandleMap * initialize a HandleMap
@ -65,7 +65,7 @@ public final class HandleMap implements Iterable<Row.Entry> {
*/ */
public HandleMap(final int keylength, final ByteOrder objectOrder, final int idxbytes, final int expectedspace, String name) { public HandleMap(final int keylength, final ByteOrder objectOrder, final int idxbytes, final int expectedspace, String name) {
this.rowdef = new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, keylength, "key"), new Column("long c-" + idxbytes + " {b256}")}, objectOrder); this.rowdef = new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, keylength, "key"), new Column("long c-" + idxbytes + " {b256}")}, objectOrder);
this.index = new RowSetArray(name, rowdef, spread(expectedspace)); this.index = new RAMIndexCluster(name, rowdef, spread(expectedspace));
} }
/** /**
@ -95,6 +95,10 @@ public final class HandleMap implements Iterable<Row.Entry> {
assert this.index.size() == file.length() / (keylength + idxbytes); assert this.index.size() == file.length() / (keylength + idxbytes);
} }
public void trim() {
this.index.trim();
}
public long mem() { public long mem() {
return index.mem(); return index.mem();
} }
@ -266,7 +270,14 @@ public final class HandleMap implements Iterable<Row.Entry> {
public final synchronized long remove(final byte[] key) { public final synchronized long remove(final byte[] key) {
assert (key != null); assert (key != null);
final boolean exist = index.has(key);
if (!exist) return -1;
final int s = index.size();
final long m = index.mem();
final Row.Entry indexentry = index.remove(key); final Row.Entry indexentry = index.remove(key);
assert (indexentry != null);
assert index.size() < s : "s = " + s + ", index.size() = " + index.size();
assert index.mem() <= m : "m = " + m + ", index.mem() = " + index.mem();
if (indexentry == null) return -1; if (indexentry == null) return -1;
return indexentry.getColLong(1); return indexentry.getColLong(1);
} }

53
source/net/yacy/kelondro/index/ObjectIndex.java → source/net/yacy/kelondro/index/Index.java
Unescape View File

@ -1,32 +1,25 @@
// ObjectIndex.java /**
// ------------------ * Index
// part of the Kelondro Database * Copyright 2005 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// (C) by Michael Peter Christen; mc@yacy.net * First released 26.10.2005 at http://yacy.net
// first published on http://www.anomic.de *
// Frankfurt, Germany, 2005 * $LastChangedDate: 2010-06-16 17:11:21 +0200 (Mi, 16 Jun 2010) $
// created: 26.10.2005 * $LastChangedRevision: 6922 $
// * $LastChangedBy: orbiter $
// This program is free software; you can redistribute it and/or modify *
// it under the terms of the GNU General Public License as published by * This library is free software; you can redistribute it and/or
// the Free Software Foundation; either version 2 of the License, or * modify it under the terms of the GNU Lesser General Public
// (at your option) any later version. * License as published by the Free Software Foundation; either
// * version 2.1 of the License, or (at your option) any later version.
// This program is distributed in the hope that it will be useful, *
// but WITHOUT ANY WARRANTY; without even the implied warranty of * This library is distributed in the hope that it will be useful,
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * but WITHOUT ANY WARRANTY; without even the implied warranty of
// GNU General Public License for more details. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// * Lesser General Public License for more details.
// You should have received a copy of the GNU General Public License *
// along with this program; if not, write to the Free Software * You should have received a copy of the GNU Lesser General Public License
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA * along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
/* A kelondroIndex is a table with indexed access on the first column
Elements may be selected from the table with logarithmic computation time
using the get-method. Inserts have also the same computation order and
can be done with the put-method.
The kelondro Database provides two implementations of this interface:
kelondroTree and kelondroHashtable
*/ */
package net.yacy.kelondro.index; package net.yacy.kelondro.index;
@ -39,7 +32,7 @@ import java.util.List;
import net.yacy.kelondro.order.CloneableIterator; import net.yacy.kelondro.order.CloneableIterator;
public interface ObjectIndex extends Iterable<Row.Entry> { public interface Index extends Iterable<Row.Entry> {
public String filename(); // returns a unique identified for this index; can be a real or artificial file name public String filename(); // returns a unique identified for this index; can be a real or artificial file name
public int size(); public int size();

69
source/net/yacy/kelondro/index/ObjectIndexCache.java → source/net/yacy/kelondro/index/RAMIndex.java
Unescape View File

@ -1,26 +1,26 @@
// ObjectIndexCache.java /**
// (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany * RAMIndex
// first published 07.01.2008 on http://yacy.net * Copyright 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// * First released 07.01.2008 at http://yacy.net
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ *
// $LastChangedRevision: 1986 $ * $LastChangedDate: 2010-06-16 17:11:21 +0200 (Mi, 16 Jun 2010) $
// $LastChangedBy: orbiter $ * $LastChangedRevision: 6922 $
// * $LastChangedBy: orbiter $
// LICENSE *
// * This library is free software; you can redistribute it and/or
// This program is free software; you can redistribute it and/or modify * modify it under the terms of the GNU Lesser General Public
// it under the terms of the GNU General Public License as published by * License as published by the Free Software Foundation; either
// the Free Software Foundation; either version 2 of the License, or * version 2.1 of the License, or (at your option) any later version.
// (at your option) any later version. *
// * This library is distributed in the hope that it will be useful,
// This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of
// but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * Lesser General Public License for more details.
// GNU General Public License for more details. *
// * You should have received a copy of the GNU Lesser General Public License
// You should have received a copy of the GNU General Public License * along with this program in the file lgpl21.txt
// along with this program; if not, write to the Free Software * If not, see <http://www.gnu.org/licenses/>.
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */
package net.yacy.kelondro.index; package net.yacy.kelondro.index;
@ -37,9 +37,9 @@ import net.yacy.kelondro.order.MergeIterator;
import net.yacy.kelondro.order.StackIterator; import net.yacy.kelondro.order.StackIterator;
public final class ObjectIndexCache implements ObjectIndex, Iterable<Row.Entry> { public final class RAMIndex implements Index, Iterable<Row.Entry> {
private static final TreeMap<String, ObjectIndexCache> objectTracker = new TreeMap<String, ObjectIndexCache>(); private static final TreeMap<String, RAMIndex> objectTracker = new TreeMap<String, RAMIndex>();
private final String name; private final String name;
private final Row rowdef; private final Row rowdef;
@ -48,7 +48,7 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable<Row.Entry>
private final Row.EntryComparator entryComparator; private final Row.EntryComparator entryComparator;
//private final int spread; //private final int spread;
public ObjectIndexCache(String name, final Row rowdef, final int expectedspace) { public RAMIndex(String name, final Row rowdef, final int expectedspace) {
this.name = name; this.name = name;
this.rowdef = rowdef; this.rowdef = rowdef;
this.entryComparator = new Row.EntryComparator(rowdef.objectOrder); this.entryComparator = new Row.EntryComparator(rowdef.objectOrder);
@ -57,7 +57,7 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable<Row.Entry>
objectTracker.put(name, this); objectTracker.put(name, this);
} }
private ObjectIndexCache(String name, final Row rowdef, RowSet index0, RowSet index1, Row.EntryComparator entryComparator) { private RAMIndex(String name, final Row rowdef, RowSet index0, RowSet index1, Row.EntryComparator entryComparator) {
this.name = name; this.name = name;
this.rowdef = rowdef; this.rowdef = rowdef;
this.index0 = index0; this.index0 = index0;
@ -66,18 +66,23 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable<Row.Entry>
objectTracker.put(name, this); objectTracker.put(name, this);
} }
public static final Iterator<Map.Entry<String, ObjectIndexCache>> objects() { public static final Iterator<Map.Entry<String, RAMIndex>> objects() {
return objectTracker.entrySet().iterator(); return objectTracker.entrySet().iterator();
} }
public ObjectIndexCache clone() { public RAMIndex clone() {
return new ObjectIndexCache(this.name + ".clone", this.rowdef, index0.clone(), index1.clone(), entryComparator); return new RAMIndex(this.name + ".clone", this.rowdef, index0.clone(), index1.clone(), entryComparator);
} }
public void clear() { public void clear() {
reset(); reset();
} }
public void trim() {
if (this.index0 != null) this.index0.trim();
if (this.index1 != null) this.index1.trim();
}
public final synchronized void reset() { public final synchronized void reset() {
this.index0 = null; // first flush RAM to make room this.index0 = null; // first flush RAM to make room
this.index0 = new RowSet(rowdef); this.index0 = new RowSet(rowdef);
@ -223,13 +228,17 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable<Row.Entry>
public final synchronized Row.Entry remove(final byte[] key) { public final synchronized Row.Entry remove(final byte[] key) {
finishInitialization(); finishInitialization();
// if the new entry is within the initialization part, just delete it // if the new entry is within the initialization part, just delete it
int s = index0.size();
final Row.Entry indexentry = index0.remove(key); final Row.Entry indexentry = index0.remove(key);
if (indexentry != null) { if (indexentry != null) {
assert index0.size() < s: "s = " + s + ", index0.size() = " + index0.size();
assert index0.get(key) == null; // check if remove worked assert index0.get(key) == null; // check if remove worked
return indexentry; return indexentry;
} }
// else remove it from the index1 // else remove it from the index1
s = index1.size();
final Row.Entry removed = index1.remove(key); final Row.Entry removed = index1.remove(key);
assert removed == null || index1.size() < s: "s = " + s + ", index1.size() = " + index1.size();
assert index1.get(key) == null : "removed " + ((removed == null) ? " is null" : " is not null") + ", and index entry still exists"; // check if remove worked assert index1.get(key) == null : "removed " + ((removed == null) ? " is null" : " is not null") + ", and index entry still exists"; // check if remove worked
return removed; return removed;
} }

182
source/net/yacy/kelondro/index/RowSetArray.java → source/net/yacy/kelondro/index/RAMIndexCluster.java
Unescape View File

@ -1,23 +1,26 @@
// RowSetArray.java /**
// -------------------------- * RAMIndexCluster
// (C) by Michael Peter Christen; mc@yacy.net * Copyright 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published on http://yacy.net * First released 12.03.2009 at http://yacy.net
// Frankfurt, Germany, 2009 *
// last major change: 12.03.2009 * $LastChangedDate: 2010-06-16 17:11:21 +0200 (Mi, 16 Jun 2010) $
// * $LastChangedRevision: 6922 $
// This program is free software; you can redistribute it and/or modify * $LastChangedBy: orbiter $
// it under the terms of the GNU General Public License as published by *
// the Free Software Foundation; either version 2 of the License, or * This library is free software; you can redistribute it and/or
// (at your option) any later version. * modify it under the terms of the GNU Lesser General Public
// * License as published by the Free Software Foundation; either
// This program is distributed in the hope that it will be useful, * version 2.1 of the License, or (at your option) any later version.
// but WITHOUT ANY WARRANTY; without even the implied warranty of *
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * This library is distributed in the hope that it will be useful,
// GNU General Public License for more details. * but WITHOUT ANY WARRANTY; without even the implied warranty of
// * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// You should have received a copy of the GNU General Public License * Lesser General Public License for more details.
// along with this program; if not, write to the Free Software *
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA * You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.kelondro.index; package net.yacy.kelondro.index;
@ -34,48 +37,52 @@ import net.yacy.kelondro.order.MergeIterator;
import net.yacy.kelondro.order.StackIterator; import net.yacy.kelondro.order.StackIterator;
public final class RowSetArray implements ObjectIndex, Iterable<Row.Entry>, Cloneable { public final class RAMIndexCluster implements Index, Iterable<Row.Entry>, Cloneable {
private final String name; private final String name;
private final Row rowdef; private final Row rowdef;
private final ObjectIndexCache[] array; private final RAMIndex[] cluster;
public RowSetArray(String name, final Row rowdef, final int arraySize) { public RAMIndexCluster(String name, final Row rowdef, final int clusterSize) {
//assert arraySize < 100 : arraySize; //assert arraySize < 100 : arraySize;
this.name = name; this.name = name;
this.array = new ObjectIndexCache[arraySize]; this.cluster = new RAMIndex[clusterSize];
this.rowdef = rowdef; this.rowdef = rowdef;
for (int i = 0; i < arraySize; i++) { for (int i = 0; i < clusterSize; i++) {
this.array[i] = new ObjectIndexCache(name + "." + i, rowdef, 0); this.cluster[i] = new RAMIndex(name + "." + i, rowdef, 0);
} }
} }
private RowSetArray(String name, final Row rowdef, final ObjectIndexCache[] array) { private RAMIndexCluster(String name, final Row rowdef, final RAMIndex[] array) {
this.name = name; this.name = name;
this.array = array; this.cluster = array;
this.rowdef = rowdef; this.rowdef = rowdef;
} }
public RowSetArray clone() { public void trim() {
ObjectIndexCache[] a = new ObjectIndexCache[this.array.length]; for (RAMIndex i: this.cluster) if (i != null) i.trim();
for (int i = 0; i < this.array.length; i++) { }
a[i] = this.array[i].clone();
public RAMIndexCluster clone() {
RAMIndex[] a = new RAMIndex[this.cluster.length];
for (int i = 0; i < this.cluster.length; i++) {
a[i] = this.cluster[i].clone();
} }
return new RowSetArray(this.name + ".clone", this.rowdef, a); return new RAMIndexCluster(this.name + ".clone", this.rowdef, a);
} }
private final int indexFor(final byte[] key) { private final int indexFor(final byte[] key) {
return (int) ((this.rowdef.objectOrder.cardinal(key) / 17) % ((long) array.length)); return (int) ((this.rowdef.objectOrder.cardinal(key) / 17) % ((long) cluster.length));
} }
private final int indexFor(final Entry row) { private final int indexFor(final Entry row) {
return (int) ((this.rowdef.objectOrder.cardinal(row.bytes(), 0, row.getPrimaryKeyLength()) / 17) % ((long) array.length)); return (int) ((this.rowdef.objectOrder.cardinal(row.bytes(), 0, row.getPrimaryKeyLength()) / 17) % ((long) cluster.length));
} }
public final byte[] smallestKey() { public final byte[] smallestKey() {
HandleSet keysort = new HandleSet(this.rowdef.primaryKeyLength, this.rowdef.objectOrder, this.array.length); HandleSet keysort = new HandleSet(this.rowdef.primaryKeyLength, this.rowdef.objectOrder, this.cluster.length);
synchronized (this.array) { synchronized (this.cluster) {
for (ObjectIndexCache rs: this.array) try { for (RAMIndex rs: this.cluster) try {
keysort.put(rs.smallestKey()); keysort.put(rs.smallestKey());
} catch (RowSpaceExceededException e) { } catch (RowSpaceExceededException e) {
Log.logException(e); Log.logException(e);
@ -85,9 +92,9 @@ public final class RowSetArray implements ObjectIndex, Iterable<Row.Entry>, Clon
} }
public final byte[] largestKey() { public final byte[] largestKey() {
HandleSet keysort = new HandleSet(this.rowdef.primaryKeyLength, this.rowdef.objectOrder, this.array.length); HandleSet keysort = new HandleSet(this.rowdef.primaryKeyLength, this.rowdef.objectOrder, this.cluster.length);
synchronized (this.array) { synchronized (this.cluster) {
for (ObjectIndexCache rs: this.array) try { for (RAMIndex rs: this.cluster) try {
keysort.put(rs.largestKey()); keysort.put(rs.largestKey());
} catch (RowSpaceExceededException e) { } catch (RowSpaceExceededException e) {
Log.logException(e); Log.logException(e);
@ -96,11 +103,11 @@ public final class RowSetArray implements ObjectIndex, Iterable<Row.Entry>, Clon
return keysort.largestKey(); return keysort.largestKey();
} }
private final ObjectIndexCache accessArray(final int i) { private final RAMIndex accessArray(final int i) {
ObjectIndexCache r = this.array[i]; RAMIndex r = this.cluster[i];
if (r == null) synchronized (this.array) { if (r == null) synchronized (this.cluster) {
r = new ObjectIndexCache(name + "." + i, this.rowdef, 0); r = new RAMIndex(name + "." + i, this.rowdef, 0);
this.array[i] = r; this.cluster[i] = r;
} }
return r; return r;
} }
@ -116,15 +123,15 @@ public final class RowSetArray implements ObjectIndex, Iterable<Row.Entry>, Clon
} }
public final void clear() { public final void clear() {
synchronized (this.array) { synchronized (this.cluster) {
for (ObjectIndexCache c: this.array) if (c != null) c.clear(); for (RAMIndex c: this.cluster) if (c != null) c.clear();
} }
} }
public final void close() { public final void close() {
clear(); clear();
synchronized (this.array) { synchronized (this.cluster) {
for (ObjectIndexCache c: this.array) if (c != null) c.close(); for (RAMIndex c: this.cluster) if (c != null) c.close();
} }
} }
@ -140,7 +147,7 @@ public final class RowSetArray implements ObjectIndex, Iterable<Row.Entry>, Clon
public final Entry get(final byte[] key) { public final Entry get(final byte[] key) {
final int i = indexFor(key); final int i = indexFor(key);
if (i < 0) return null; if (i < 0) return null;
final ObjectIndexCache r = this.array[i]; final RAMIndex r = this.cluster[i];
if (r == null) return null; if (r == null) return null;
return r.get(key); return r.get(key);
} }
@ -148,17 +155,17 @@ public final class RowSetArray implements ObjectIndex, Iterable<Row.Entry>, Clon
public final boolean has(final byte[] key) { public final boolean has(final byte[] key) {
final int i = indexFor(key); final int i = indexFor(key);
if (i < 0) return false; if (i < 0) return false;
final ObjectIndexCache r = this.array[i]; final RAMIndex r = this.cluster[i];
if (r == null) return false; if (r == null) return false;
return r.has(key); return r.has(key);
} }
public final CloneableIterator<byte[]> keys(final boolean up, final byte[] firstKey) { public final CloneableIterator<byte[]> keys(final boolean up, final byte[] firstKey) {
synchronized (this.array) { synchronized (this.cluster) {
final Collection<CloneableIterator<byte[]>> col = new ArrayList<CloneableIterator<byte[]>>(); final Collection<CloneableIterator<byte[]>> col = new ArrayList<CloneableIterator<byte[]>>();
for (int i = 0; i < this.array.length; i++) { for (int i = 0; i < this.cluster.length; i++) {
if (this.array[i] != null) { if (this.cluster[i] != null) {
col.add(this.array[i].keys(up, firstKey)); col.add(this.cluster[i].keys(up, firstKey));
} }
} }
return MergeIterator.cascade(col, this.rowdef.objectOrder, MergeIterator.simpleMerge, up); return MergeIterator.cascade(col, this.rowdef.objectOrder, MergeIterator.simpleMerge, up);
@ -185,11 +192,11 @@ public final class RowSetArray implements ObjectIndex, Iterable<Row.Entry>, Clon
public final ArrayList<RowCollection> removeDoubles() throws RowSpaceExceededException { public final ArrayList<RowCollection> removeDoubles() throws RowSpaceExceededException {
final ArrayList<RowCollection> col = new ArrayList<RowCollection>(); final ArrayList<RowCollection> col = new ArrayList<RowCollection>();
synchronized (this.array) { synchronized (this.cluster) {
for (int i = 0; i < this.array.length; i++) { for (int i = 0; i < this.cluster.length; i++) {
if (this.array[i] != null) { if (this.cluster[i] != null) {
col.addAll(this.array[i].removeDoubles()); col.addAll(this.cluster[i].removeDoubles());
if (this.array[i].isEmpty()) this.array[i] = null; if (this.cluster[i].isEmpty()) this.cluster[i] = null;
} }
} }
} }
@ -197,11 +204,11 @@ public final class RowSetArray implements ObjectIndex, Iterable<Row.Entry>, Clon
} }
public final Entry removeOne() { public final Entry removeOne() {
synchronized (this.array) { synchronized (this.cluster) {
for (int i = 0; i < this.array.length; i++) { for (int i = 0; i < this.cluster.length; i++) {
if (this.array[i] != null) { if (this.cluster[i] != null) {
final Entry entry = this.array[i].removeOne(); final Entry entry = this.cluster[i].removeOne();
if (this.array[i].isEmpty()) this.array[i] = null; if (this.cluster[i].isEmpty()) this.cluster[i] = null;
return entry; return entry;
} }
} }
@ -211,11 +218,11 @@ public final class RowSetArray implements ObjectIndex, Iterable<Row.Entry>, Clon
public List<Row.Entry> top(int count) { public List<Row.Entry> top(int count) {
List<Row.Entry> list = new ArrayList<Row.Entry>(); List<Row.Entry> list = new ArrayList<Row.Entry>();
synchronized (this.array) { synchronized (this.cluster) {
for (int i = 0; i < this.array.length; i++) { for (int i = 0; i < this.cluster.length; i++) {
if (this.array[i] != null) { if (this.cluster[i] != null) {
try { try {
List<Row.Entry> list0 = this.array[i].top(count - list.size()); List<Row.Entry> list0 = this.cluster[i].top(count - list.size());
list.addAll(list0); list.addAll(list0);
} catch (IOException e) { } catch (IOException e) {
continue; continue;
@ -239,13 +246,13 @@ public final class RowSetArray implements ObjectIndex, Iterable<Row.Entry>, Clon
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public final CloneableIterator<Entry> rows(final boolean up, final byte[] firstKey) { public final CloneableIterator<Entry> rows(final boolean up, final byte[] firstKey) {
synchronized (this.array) { synchronized (this.cluster) {
final CloneableIterator<Entry>[] col = new CloneableIterator[this.array.length]; final CloneableIterator<Entry>[] col = new CloneableIterator[this.cluster.length];
for (int i = 0; i < this.array.length; i++) { for (int i = 0; i < this.cluster.length; i++) {
if (this.array[i] == null) { if (this.cluster[i] == null) {
col[i] = null; col[i] = null;
} else { } else {
col[i] = this.array[i].rows(up, firstKey); col[i] = this.cluster[i].rows(up, firstKey);
} }
} }
return StackIterator.stack(col); return StackIterator.stack(col);
@ -258,36 +265,23 @@ public final class RowSetArray implements ObjectIndex, Iterable<Row.Entry>, Clon
public final int size() { public final int size() {
int c = 0; int c = 0;
synchronized (this.array) { synchronized (this.cluster) {
for (int i = 0; i < this.array.length; i++) { for (RAMIndex i: this.cluster) if (i != null) c += i.size();
if (this.array[i] != null) {
c += this.array[i].size();
}
}
} }
return c; return c;
} }
public long mem() { public long mem() {
long m = 0; long m = 0;
synchronized (this.array) { synchronized (this.cluster) {
for (int i = 0; i < this.array.length; i++) { for (RAMIndex i: this.cluster) if (i != null) m += i.mem();
if (this.array[i] != null) {
m += this.array[i].mem();
}
}
} }
return m; return m;
} }
public final boolean isEmpty() { public final boolean isEmpty() {
synchronized (this.array) { synchronized (this.cluster) {
for (int i = 0; i < this.array.length; i++) { for (RAMIndex i: this.cluster) if (i != null && !i.isEmpty()) return false;
if (this.array[i] != null) {
if (!this.array[i].isEmpty()) return false;
}
}
} }
return true; return true;
} }

13
source/net/yacy/kelondro/index/RowCollection.java
Unescape View File

@ -877,15 +877,14 @@ public class RowCollection implements Iterable<Row.Entry>, Cloneable {
} }
private final int picMiddle(final int a, final int b, final int c) { private final int picMiddle(final int a, final int b, final int c) {
if (a > b) { if (compare(a, b) > 0) {
if (c > a) return a; if (compare(c, a) > 0) return a;
if (c < b) return b; if (compare(b, c) > 0) return b;
return c;
} else { } else {
if (c < a) return a; if (compare(a, c) > 0) return a;
if (c > b) return b; if (compare(c, b) > 0) return b;
return c;
} }
return c;
//if (c < a && a < b || a > b && c > a) return a; //if (c < a && a < b || a > b && c > a) return a;
//if (a < b && c > b || c < b && a > b) return b; //if (a < b && c > b || c < b && a > b) return b;
} }

48
source/net/yacy/kelondro/index/RowSet.java
Unescape View File

@ -1,26 +1,26 @@
// RowSet.java /**
// (C) 2006 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany * RowSet
// first published 20.06.2006 on http://www.anomic.de * Copyright 2006 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// * First released 20.06.2006 at http://yacy.net
// $LastChangedDate$ *
// $LastChangedRevision$ * $LastChangedDate$
// $LastChangedBy$ * $LastChangedRevision$
// * $LastChangedBy$
// LICENSE *
// * This library is free software; you can redistribute it and/or
// This program is free software; you can redistribute it and/or modify * modify it under the terms of the GNU Lesser General Public
// it under the terms of the GNU General Public License as published by * License as published by the Free Software Foundation; either
// the Free Software Foundation; either version 2 of the License, or * version 2.1 of the License, or (at your option) any later version.
// (at your option) any later version. *
// * This library is distributed in the hope that it will be useful,
// This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of
// but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * Lesser General Public License for more details.
// GNU General Public License for more details. *
// * You should have received a copy of the GNU Lesser General Public License
// You should have received a copy of the GNU General Public License * along with this program in the file lgpl21.txt
// along with this program; if not, write to the Free Software * If not, see <http://www.gnu.org/licenses/>.
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */
package net.yacy.kelondro.index; package net.yacy.kelondro.index;
@ -33,7 +33,7 @@ import net.yacy.kelondro.order.CloneableIterator;
import net.yacy.kelondro.order.NaturalOrder; import net.yacy.kelondro.order.NaturalOrder;
public class RowSet extends RowCollection implements ObjectIndex, Iterable<Row.Entry> { public class RowSet extends RowCollection implements Index, Iterable<Row.Entry> {
private static final int collectionReSortLimit = 300; private static final int collectionReSortLimit = 300;

16
source/net/yacy/kelondro/rwi/IndexCell.java
Unescape View File

@ -249,36 +249,40 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
public int remove(byte[] termHash, HandleSet urlHashes) throws IOException { public int remove(byte[] termHash, HandleSet urlHashes) throws IOException {
int removed = this.ram.remove(termHash, urlHashes); int removed = this.ram.remove(termHash, urlHashes);
int reduced; int reduced;
//final long am = this.array.mem();
try { try {
reduced = this.array.replace(termHash, new RemoveRewriter<ReferenceType>(urlHashes)); reduced = this.array.reduce(termHash, new RemoveReducer<ReferenceType>(urlHashes));
} catch (RowSpaceExceededException e) { } catch (RowSpaceExceededException e) {
reduced = 0; reduced = 0;
Log.logWarning("IndexCell", "not possible to remove urlHashes from a RWI because of too low memory. Remove was not applied. Please increase RAM assignment"); Log.logWarning("IndexCell", "not possible to remove urlHashes from a RWI because of too low memory. Remove was not applied. Please increase RAM assignment");
} }
//assert this.array.mem() <= am : "am = " + am + ", array.mem() = " + this.array.mem();
return removed + (reduced / this.array.rowdef().objectsize); return removed + (reduced / this.array.rowdef().objectsize);
} }
public boolean remove(byte[] termHash, byte[] urlHashBytes) throws IOException { public boolean remove(byte[] termHash, byte[] urlHashBytes) throws IOException {
boolean removed = this.ram.remove(termHash, urlHashBytes); boolean removed = this.ram.remove(termHash, urlHashBytes);
int reduced; int reduced;
//final long am = this.array.mem();
try { try {
reduced = this.array.replace(termHash, new RemoveRewriter<ReferenceType>(urlHashBytes)); reduced = this.array.reduce(termHash, new RemoveReducer<ReferenceType>(urlHashBytes));
} catch (RowSpaceExceededException e) { } catch (RowSpaceExceededException e) {
reduced = 0; reduced = 0;
Log.logWarning("IndexCell", "not possible to remove urlHashes from a RWI because of too low memory. Remove was not applied. Please increase RAM assignment"); Log.logWarning("IndexCell", "not possible to remove urlHashes from a RWI because of too low memory. Remove was not applied. Please increase RAM assignment");
} }
//assert this.array.mem() <= am : "am = " + am + ", array.mem() = " + this.array.mem();
return removed || (reduced > 0); return removed || (reduced > 0);
} }
private static class RemoveRewriter<ReferenceType extends Reference> implements ReferenceContainerArray.ContainerRewriter<ReferenceType> { private static class RemoveReducer<ReferenceType extends Reference> implements ReferenceContainerArray.ContainerReducer<ReferenceType> {
HandleSet urlHashes; HandleSet urlHashes;
public RemoveRewriter(HandleSet urlHashes) { public RemoveReducer(HandleSet urlHashes) {
this.urlHashes = urlHashes; this.urlHashes = urlHashes;
} }
public RemoveRewriter(byte[] urlHashBytes) { public RemoveReducer(byte[] urlHashBytes) {
this.urlHashes = new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0); this.urlHashes = new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0);
try { try {
this.urlHashes.put(urlHashBytes); this.urlHashes.put(urlHashBytes);
@ -287,7 +291,7 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
} }
} }
public ReferenceContainer<ReferenceType> rewrite(ReferenceContainer<ReferenceType> container) { public ReferenceContainer<ReferenceType> reduce(ReferenceContainer<ReferenceType> container) {
container.sort(); container.sort();
container.removeEntries(urlHashes); container.removeEntries(urlHashes);
return container; return container;

29
source/net/yacy/kelondro/rwi/ReferenceContainerArray.java
Unescape View File

@ -71,7 +71,8 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
prefix, prefix,
payloadrow.primaryKeyLength, payloadrow.primaryKeyLength,
termOrder, termOrder,
0); 0,
true);
assert merger != null; assert merger != null;
this.merger = merger; this.merger = merger;
} }
@ -84,6 +85,10 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
this.array.clear(); this.array.clear();
} }
public long mem() {
return array.mem();
}
public int[] sizes() { public int[] sizes() {
return (this.array == null) ? new int[0] : this.array.sizes(); return (this.array == null) ? new int[0] : this.array.sizes();
} }
@ -249,34 +254,36 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
array.remove(termHash); array.remove(termHash);
} }
public int replace(final byte[] termHash, ContainerRewriter<ReferenceType> rewriter) throws IOException, RowSpaceExceededException { public int reduce(final byte[] termHash, ContainerReducer<ReferenceType> reducer) throws IOException, RowSpaceExceededException {
return array.replace(termHash, new BLOBRewriter(termHash, rewriter)); return array.reduce(termHash, new BLOBReducer(termHash, reducer));
} }
public class BLOBRewriter implements BLOB.Rewriter { public class BLOBReducer implements BLOB.Reducer {
ContainerRewriter<ReferenceType> rewriter; ContainerReducer<ReferenceType> rewriter;
byte[] wordHash; byte[] wordHash;
public BLOBRewriter(byte[] wordHash, ContainerRewriter<ReferenceType> rewriter) { public BLOBReducer(byte[] wordHash, ContainerReducer<ReferenceType> rewriter) {
this.rewriter = rewriter; this.rewriter = rewriter;
this.wordHash = wordHash; this.wordHash = wordHash;
} }
public byte[] rewrite(byte[] b) { public byte[] rewrite(byte[] b) {
if (b == null) return null; if (b == null) return null;
ReferenceContainer<ReferenceType> c = rewriter.rewrite(new ReferenceContainer<ReferenceType>(factory, this.wordHash, RowSet.importRowSet(b, payloadrow))); ReferenceContainer<ReferenceType> c = rewriter.reduce(new ReferenceContainer<ReferenceType>(factory, this.wordHash, RowSet.importRowSet(b, payloadrow)));
if (c == null) return null; if (c == null) return null;
return c.exportCollection(); byte bb[] = c.exportCollection();
assert bb.length <= b.length;
return bb;
} }
} }
public interface ContainerRewriter<ReferenceType extends Reference> { public interface ContainerReducer<ReferenceType extends Reference> {
public ReferenceContainer<ReferenceType> rewrite(ReferenceContainer<ReferenceType> container); public ReferenceContainer<ReferenceType> reduce(ReferenceContainer<ReferenceType> container);
} }
public int entries() { public int entries() {
return this.array.entries(); return this.array.entries();
} }

24
source/net/yacy/kelondro/table/Relations.java
Unescape View File

@ -30,7 +30,7 @@ import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.HashMap; import java.util.HashMap;
import net.yacy.kelondro.index.ObjectIndex; import net.yacy.kelondro.index.Index;
import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.Row;
import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
@ -40,7 +40,7 @@ import net.yacy.kelondro.order.NaturalOrder;
public class Relations { public class Relations {
private final File baseDir; private final File baseDir;
private HashMap<String, ObjectIndex> relations; private HashMap<String, Index> relations;
private final boolean useTailCache; private final boolean useTailCache;
private final boolean exceed134217727; private final boolean exceed134217727;
@ -81,7 +81,7 @@ public class Relations {
public void declareRelation(final String name, final int keysize, final int payloadsize) throws RowSpaceExceededException { public void declareRelation(final String name, final int keysize, final int payloadsize) throws RowSpaceExceededException {
// try to get the relation from the relation-cache // try to get the relation from the relation-cache
final ObjectIndex relation = relations.get(name); final Index relation = relations.get(name);
if (relation != null) return; if (relation != null) return;
// try to find the relation as stored on file // try to find the relation as stored on file
final String[] list = baseDir.list(); final String[] list = baseDir.list();
@ -91,7 +91,7 @@ public class Relations {
if (!list[i].equals(targetfilename)) continue; if (!list[i].equals(targetfilename)) continue;
final Row row = rowdef(list[i]); final Row row = rowdef(list[i]);
if (row.primaryKeyLength != keysize || row.column(1).cellwidth != payloadsize) continue; // a wrong table if (row.primaryKeyLength != keysize || row.column(1).cellwidth != payloadsize) continue; // a wrong table
ObjectIndex table; Index table;
try { try {
table = new Table(new File(baseDir, list[i]), row, 1024*1024, 0, this.useTailCache, this.exceed134217727); table = new Table(new File(baseDir, list[i]), row, 1024*1024, 0, this.useTailCache, this.exceed134217727);
} catch (RowSpaceExceededException e) { } catch (RowSpaceExceededException e) {
@ -103,7 +103,7 @@ public class Relations {
} }
// the relation does not exist, create it // the relation does not exist, create it
final Row row = rowdef(keysize, payloadsize); final Row row = rowdef(keysize, payloadsize);
ObjectIndex table; Index table;
try { try {
table = new Table(new File(baseDir, targetfilename), row, 1024*1024, 0, this.useTailCache, this.exceed134217727); table = new Table(new File(baseDir, targetfilename), row, 1024*1024, 0, this.useTailCache, this.exceed134217727);
} catch (RowSpaceExceededException e) { } catch (RowSpaceExceededException e) {
@ -112,16 +112,16 @@ public class Relations {
relations.put(name, table); relations.put(name, table);
} }
public ObjectIndex getRelation(final String name) throws RowSpaceExceededException { public Index getRelation(final String name) throws RowSpaceExceededException {
// try to get the relation from the relation-cache // try to get the relation from the relation-cache
final ObjectIndex relation = relations.get(name); final Index relation = relations.get(name);
if (relation != null) return relation; if (relation != null) return relation;
// try to find the relation as stored on file // try to find the relation as stored on file
final String[] list = baseDir.list(); final String[] list = baseDir.list();
for (int i = 0; i < list.length; i++) { for (int i = 0; i < list.length; i++) {
if (list[i].startsWith(name)) { if (list[i].startsWith(name)) {
final Row row = rowdef(list[i]); final Row row = rowdef(list[i]);
ObjectIndex table; Index table;
try { try {
table = new Table(new File(baseDir, list[i]), row, 1024*1024, 0, this.useTailCache, this.exceed134217727); table = new Table(new File(baseDir, list[i]), row, 1024*1024, 0, this.useTailCache, this.exceed134217727);
} catch (RowSpaceExceededException e) { } catch (RowSpaceExceededException e) {
@ -142,7 +142,7 @@ public class Relations {
} }
public byte[] putRelation(final String name, final byte[] key, final byte[] value) throws IOException, RowSpaceExceededException { public byte[] putRelation(final String name, final byte[] key, final byte[] value) throws IOException, RowSpaceExceededException {
final ObjectIndex table = getRelation(name); final Index table = getRelation(name);
if (table == null) return null; if (table == null) return null;
final Row.Entry entry = table.row().newEntry(); final Row.Entry entry = table.row().newEntry();
entry.setCol(0, key); entry.setCol(0, key);
@ -161,7 +161,7 @@ public class Relations {
} }
public byte[] getRelation(final String name, final byte[] key) throws IOException, RowSpaceExceededException { public byte[] getRelation(final String name, final byte[] key) throws IOException, RowSpaceExceededException {
final ObjectIndex table = getRelation(name); final Index table = getRelation(name);
if (table == null) return null; if (table == null) return null;
final Row.Entry entry = table.get(key); final Row.Entry entry = table.get(key);
if (entry == null) return null; if (entry == null) return null;
@ -169,13 +169,13 @@ public class Relations {
} }
public boolean hasRelation(final String name, final byte[] key) throws RowSpaceExceededException { public boolean hasRelation(final String name, final byte[] key) throws RowSpaceExceededException {
final ObjectIndex table = getRelation(name); final Index table = getRelation(name);
if (table == null) return false; if (table == null) return false;
return table.has(key); return table.has(key);
} }
public byte[] removeRelation(final String name, final byte[] key) throws IOException, RowSpaceExceededException { public byte[] removeRelation(final String name, final byte[] key) throws IOException, RowSpaceExceededException {
final ObjectIndex table = getRelation(name); final Index table = getRelation(name);
if (table == null) return null; if (table == null) return null;
final Row.Entry entry = table.remove(key); final Row.Entry entry = table.remove(key);
if (entry == null) return null; if (entry == null) return null;

4
source/net/yacy/kelondro/table/SQLTable.java
Unescape View File

@ -37,7 +37,7 @@ import java.util.Date;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import net.yacy.kelondro.index.ObjectIndex; import net.yacy.kelondro.index.Index;
import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.Row;
import net.yacy.kelondro.index.RowCollection; import net.yacy.kelondro.index.RowCollection;
import net.yacy.kelondro.index.Row.Entry; import net.yacy.kelondro.index.Row.Entry;
@ -58,7 +58,7 @@ import net.yacy.kelondro.order.NaturalOrder;
* grant ALL on yacy.* to yacy; * grant ALL on yacy.* to yacy;
*/ */
public class SQLTable implements ObjectIndex, Iterable<Row.Entry> { public class SQLTable implements Index, Iterable<Row.Entry> {
private static final String db_driver_str_mysql = "org.gjt.mm.mysql.Driver"; private static final String db_driver_str_mysql = "org.gjt.mm.mysql.Driver";
private static final String db_driver_str_pgsql = "org.postgresql.Driver"; private static final String db_driver_str_pgsql = "org.postgresql.Driver";

62
source/net/yacy/kelondro/table/SplitTable.java
Unescape View File

@ -44,7 +44,7 @@ import java.util.concurrent.TimeUnit;
import net.yacy.kelondro.blob.ArrayStack; import net.yacy.kelondro.blob.ArrayStack;
import net.yacy.kelondro.index.Cache; import net.yacy.kelondro.index.Cache;
import net.yacy.kelondro.index.HandleSet; import net.yacy.kelondro.index.HandleSet;
import net.yacy.kelondro.index.ObjectIndex; import net.yacy.kelondro.index.Index;
import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.Row;
import net.yacy.kelondro.index.RowCollection; import net.yacy.kelondro.index.RowCollection;
import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.index.RowSpaceExceededException;
@ -59,7 +59,7 @@ import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.NamePrefixThreadFactory; import net.yacy.kelondro.util.NamePrefixThreadFactory;
public class SplitTable implements ObjectIndex, Iterable<Row.Entry> { public class SplitTable implements Index, Iterable<Row.Entry> {
// this is a set of kelondro tables // this is a set of kelondro tables
// the set is divided into tables with different entry date // the set is divided into tables with different entry date
@ -70,7 +70,7 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
// the thread pool for the keeperOf executor service // the thread pool for the keeperOf executor service
private ExecutorService executor; private ExecutorService executor;
private Map<String, ObjectIndex> tables; // a map from a date string to a kelondroIndex object private Map<String, Index> tables; // a map from a date string to a kelondroIndex object
private final Row rowdef; private final Row rowdef;
private final File path; private final File path;
private final String prefix; private final String prefix;
@ -111,13 +111,13 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
public long mem() { public long mem() {
long m = 0; long m = 0;
for (ObjectIndex i: tables.values()) m += i.mem(); for (Index i: tables.values()) m += i.mem();
return m; return m;
} }
public final byte[] smallestKey() { public final byte[] smallestKey() {
HandleSet keysort = new HandleSet(this.rowdef.primaryKeyLength, this.rowdef.objectOrder, this.tables.size()); HandleSet keysort = new HandleSet(this.rowdef.primaryKeyLength, this.rowdef.objectOrder, this.tables.size());
for (ObjectIndex oi: this.tables.values()) try { for (Index oi: this.tables.values()) try {
keysort.put(oi.smallestKey()); keysort.put(oi.smallestKey());
} catch (RowSpaceExceededException e) { } catch (RowSpaceExceededException e) {
Log.logException(e); Log.logException(e);
@ -127,7 +127,7 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
public final byte[] largestKey() { public final byte[] largestKey() {
HandleSet keysort = new HandleSet(this.rowdef.primaryKeyLength, this.rowdef.objectOrder, this.tables.size()); HandleSet keysort = new HandleSet(this.rowdef.primaryKeyLength, this.rowdef.objectOrder, this.tables.size());
for (ObjectIndex oi: this.tables.values()) try { for (Index oi: this.tables.values()) try {
keysort.put(oi.largestKey()); keysort.put(oi.largestKey());
} catch (RowSpaceExceededException e) { } catch (RowSpaceExceededException e) {
Log.logException(e); Log.logException(e);
@ -143,7 +143,7 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
current = null; current = null;
// initialized tables map // initialized tables map
this.tables = new HashMap<String, ObjectIndex>(); this.tables = new HashMap<String, Index>();
if (!(path.exists())) path.mkdirs(); if (!(path.exists())) path.mkdirs();
String[] tablefile = path.list(); String[] tablefile = path.list();
@ -193,7 +193,7 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
Map.Entry<String, Long> entry; Map.Entry<String, Long> entry;
String maxf; String maxf;
long maxram; long maxram;
ObjectIndex table; Index table;
while (!t.isEmpty()) { while (!t.isEmpty()) {
// find maximum table // find maximum table
maxram = 0; maxram = 0;
@ -272,21 +272,21 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
} }
public int size() { public int size() {
final Iterator<ObjectIndex> i = tables.values().iterator(); final Iterator<Index> i = tables.values().iterator();
int s = 0; int s = 0;
while (i.hasNext()) s += i.next().size(); while (i.hasNext()) s += i.next().size();
return s; return s;
} }
public boolean isEmpty() { public boolean isEmpty() {
final Iterator<ObjectIndex> i = tables.values().iterator(); final Iterator<Index> i = tables.values().iterator();
while (i.hasNext()) if (!i.next().isEmpty()) return false; while (i.hasNext()) if (!i.next().isEmpty()) return false;
return true; return true;
} }
public int writeBufferSize() { public int writeBufferSize() {
int s = 0; int s = 0;
for (final ObjectIndex index : tables.values()) { for (final Index index : tables.values()) {
if (index instanceof Cache) s += ((Cache) index).writeBufferSize(); if (index instanceof Cache) s += ((Cache) index).writeBufferSize();
} }
return s; return s;
@ -301,12 +301,12 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
} }
public Row.Entry get(final byte[] key) throws IOException { public Row.Entry get(final byte[] key) throws IOException {
final ObjectIndex keeper = keeperOf(key); final Index keeper = keeperOf(key);
if (keeper == null) return null; if (keeper == null) return null;
return keeper.get(key); return keeper.get(key);
} }
private ObjectIndex newTable() { private Index newTable() {
this.current = newFilename(); this.current = newFilename();
final File f = new File(path, this.current); final File f = new File(path, this.current);
Table table = null; Table table = null;
@ -324,7 +324,7 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
return table; return table;
} }
private ObjectIndex checkTable(ObjectIndex table) { private Index checkTable(Index table) {
// check size and age of given table; in case it is too large or too old // check size and age of given table; in case it is too large or too old
// create a new table // create a new table
assert table != null; assert table != null;
@ -344,7 +344,7 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
public Row.Entry replace(final Row.Entry row) throws IOException, RowSpaceExceededException { public Row.Entry replace(final Row.Entry row) throws IOException, RowSpaceExceededException {
assert row.objectsize() <= this.rowdef.objectsize; assert row.objectsize() <= this.rowdef.objectsize;
ObjectIndex keeper = keeperOf(row.getColBytes(0, true)); Index keeper = keeperOf(row.getColBytes(0, true));
if (keeper != null) return keeper.replace(row); if (keeper != null) return keeper.replace(row);
synchronized (this.tables) { synchronized (this.tables) {
assert this.current == null || this.tables.get(this.current) != null : "this.current = " + this.current; assert this.current == null || this.tables.get(this.current) != null : "this.current = " + this.current;
@ -356,7 +356,7 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
public void put(final Row.Entry row) throws IOException, RowSpaceExceededException { public void put(final Row.Entry row) throws IOException, RowSpaceExceededException {
assert row.objectsize() <= this.rowdef.objectsize; assert row.objectsize() <= this.rowdef.objectsize;
ObjectIndex keeper = keeperOf(row.getColBytes(0, true)); Index keeper = keeperOf(row.getColBytes(0, true));
if (keeper != null) {keeper.put(row); return;} if (keeper != null) {keeper.put(row); return;}
synchronized (this.tables) { synchronized (this.tables) {
assert this.current == null || this.tables.get(this.current) != null : "this.current = " + this.current; assert this.current == null || this.tables.get(this.current) != null : "this.current = " + this.current;
@ -366,9 +366,9 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
} }
private ObjectIndex keeperOf(final byte[] key) { private Index keeperOf(final byte[] key) {
if (key == null) return null; if (key == null) return null;
for (ObjectIndex oi: tables.values()) { for (Index oi: tables.values()) {
if (oi.has(key)) return oi; if (oi.has(key)) return oi;
} }
return null; return null;
@ -376,7 +376,7 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
public void addUnique(final Row.Entry row) throws IOException, RowSpaceExceededException { public void addUnique(final Row.Entry row) throws IOException, RowSpaceExceededException {
assert row.objectsize() <= this.rowdef.objectsize; assert row.objectsize() <= this.rowdef.objectsize;
ObjectIndex table = (this.current == null) ? null : tables.get(this.current); Index table = (this.current == null) ? null : tables.get(this.current);
synchronized (this.tables) { synchronized (this.tables) {
assert this.current == null || this.tables.get(this.current) != null : "this.current = " + this.current; assert this.current == null || this.tables.get(this.current) != null : "this.current = " + this.current;
if (table == null) table = newTable(); else table = checkTable(table); if (table == null) table = newTable(); else table = checkTable(table);
@ -385,7 +385,7 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
} }
public ArrayList<RowCollection> removeDoubles() throws IOException, RowSpaceExceededException { public ArrayList<RowCollection> removeDoubles() throws IOException, RowSpaceExceededException {
final Iterator<ObjectIndex> i = tables.values().iterator(); final Iterator<Index> i = tables.values().iterator();
final ArrayList<RowCollection> report = new ArrayList<RowCollection>(); final ArrayList<RowCollection> report = new ArrayList<RowCollection>();
while (i.hasNext()) { while (i.hasNext()) {
report.addAll(i.next().removeDoubles()); report.addAll(i.next().removeDoubles());
@ -394,20 +394,20 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
} }
public boolean delete(final byte[] key) throws IOException { public boolean delete(final byte[] key) throws IOException {
final ObjectIndex table = keeperOf(key); final Index table = keeperOf(key);
if (table == null) return false; if (table == null) return false;
return table.delete(key); return table.delete(key);
} }
public Row.Entry remove(final byte[] key) throws IOException { public Row.Entry remove(final byte[] key) throws IOException {
final ObjectIndex table = keeperOf(key); final Index table = keeperOf(key);
if (table == null) return null; if (table == null) return null;
return table.remove(key); return table.remove(key);
} }
public Row.Entry removeOne() throws IOException { public Row.Entry removeOne() throws IOException {
final Iterator<ObjectIndex> i = tables.values().iterator(); final Iterator<Index> i = tables.values().iterator();
ObjectIndex table, maxtable = null; Index table, maxtable = null;
int maxcount = -1; int maxcount = -1;
while (i.hasNext()) { while (i.hasNext()) {
table = i.next(); table = i.next();
@ -423,8 +423,8 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
} }
public List<Row.Entry> top(int count) throws IOException { public List<Row.Entry> top(int count) throws IOException {
final Iterator<ObjectIndex> i = tables.values().iterator(); final Iterator<Index> i = tables.values().iterator();
ObjectIndex table, maxtable = null; Index table, maxtable = null;
int maxcount = -1; int maxcount = -1;
while (i.hasNext()) { while (i.hasNext()) {
table = i.next(); table = i.next();
@ -441,7 +441,7 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
public CloneableIterator<byte[]> keys(final boolean up, final byte[] firstKey) throws IOException { public CloneableIterator<byte[]> keys(final boolean up, final byte[] firstKey) throws IOException {
final List<CloneableIterator<byte[]>> c = new ArrayList<CloneableIterator<byte[]>>(tables.size()); final List<CloneableIterator<byte[]>> c = new ArrayList<CloneableIterator<byte[]>>(tables.size());
final Iterator<ObjectIndex> i = tables.values().iterator(); final Iterator<Index> i = tables.values().iterator();
CloneableIterator<byte[]> k; CloneableIterator<byte[]> k;
while (i.hasNext()) { while (i.hasNext()) {
k = i.next().keys(up, firstKey); k = i.next().keys(up, firstKey);
@ -452,7 +452,7 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
public CloneableIterator<Row.Entry> rows(final boolean up, final byte[] firstKey) throws IOException { public CloneableIterator<Row.Entry> rows(final boolean up, final byte[] firstKey) throws IOException {
final List<CloneableIterator<Row.Entry>> c = new ArrayList<CloneableIterator<Row.Entry>>(tables.size()); final List<CloneableIterator<Row.Entry>> c = new ArrayList<CloneableIterator<Row.Entry>>(tables.size());
final Iterator<ObjectIndex> i = tables.values().iterator(); final Iterator<Index> i = tables.values().iterator();
while (i.hasNext()) { while (i.hasNext()) {
c.add(i.next().rows(up, firstKey)); c.add(i.next().rows(up, firstKey));
} }
@ -470,7 +470,7 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public synchronized CloneableIterator<Row.Entry> rows() throws IOException { public synchronized CloneableIterator<Row.Entry> rows() throws IOException {
final CloneableIterator<Row.Entry>[] c = new CloneableIterator[tables.size()]; final CloneableIterator<Row.Entry>[] c = new CloneableIterator[tables.size()];
final Iterator<ObjectIndex> i = tables.values().iterator(); final Iterator<Index> i = tables.values().iterator();
int d = 0; int d = 0;
while (i.hasNext()) { while (i.hasNext()) {
c[d++] = i.next().rows(); c[d++] = i.next().rows();
@ -486,7 +486,7 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
} catch (final InterruptedException e) { } catch (final InterruptedException e) {
} }
this.executor = null; this.executor = null;
final Iterator<ObjectIndex> i = tables.values().iterator(); final Iterator<Index> i = tables.values().iterator();
while (i.hasNext()) { while (i.hasNext()) {
i.next().close(); i.next().close();
} }
@ -494,7 +494,7 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
} }
public void deleteOnExit() { public void deleteOnExit() {
for (ObjectIndex i: this.tables.values()) i.deleteOnExit(); for (Index i: this.tables.values()) i.deleteOnExit();
} }
} }

15
source/net/yacy/kelondro/table/Table.java
Unescape View File

@ -40,7 +40,7 @@ import java.util.TreeSet;
import net.yacy.kelondro.index.Column; import net.yacy.kelondro.index.Column;
import net.yacy.kelondro.index.HandleMap; import net.yacy.kelondro.index.HandleMap;
import net.yacy.kelondro.index.ObjectIndex; import net.yacy.kelondro.index.Index;
import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.Row;
import net.yacy.kelondro.index.RowCollection; import net.yacy.kelondro.index.RowCollection;
import net.yacy.kelondro.index.RowSet; import net.yacy.kelondro.index.RowSet;
@ -66,7 +66,7 @@ import net.yacy.kelondro.util.kelondroException;
* The content cache can also be deleted during run-time, if the available RAM gets too low. * The content cache can also be deleted during run-time, if the available RAM gets too low.
*/ */
public class Table implements ObjectIndex, Iterable<Row.Entry> { public class Table implements Index, Iterable<Row.Entry> {
// static tracker objects // static tracker objects
private final static TreeMap<String, Table> tableTracker = new TreeMap<String, Table>(); private final static TreeMap<String, Table> tableTracker = new TreeMap<String, Table>();
@ -660,12 +660,14 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
if (i == index.size() - 1) { if (i == index.size() - 1) {
// element is at last entry position // element is at last entry position
ix = (int) index.remove(key); ix = (int) index.remove(key);
assert index.size() < i + 1 : "index.size() = " + index.size() + ", i = " + i;
assert ix == i; assert ix == i;
file.cleanLast(b, 0); file.cleanLast(b, 0);
} else { } else {
// remove entry from index // remove entry from index
assert i < index.size() - 1; assert i < index.size() - 1 : "index.size() = " + index.size() + ", i = " + i;
ix = (int) index.remove(key); ix = (int) index.remove(key);
assert i < index.size() : "index.size() = " + index.size() + ", i = " + i;
assert ix == i; assert ix == i;
// read element that shall be removed // read element that shall be removed
@ -693,12 +695,14 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
if (i == index.size() - 1) { if (i == index.size() - 1) {
// special handling if the entry is the last entry in the file // special handling if the entry is the last entry in the file
ix = (int) index.remove(key); ix = (int) index.remove(key);
assert index.size() < i + 1 : "index.size() = " + index.size() + ", i = " + i;
assert ix == i; assert ix == i;
table.removeRow(i, false); table.removeRow(i, false);
file.cleanLast(); file.cleanLast();
} else { } else {
// remove entry from index // remove entry from index
ix = (int) index.remove(key); ix = (int) index.remove(key);
assert i < index.size() : "index.size() = " + index.size() + ", i = " + i;
assert ix == i; assert ix == i;
// switch values: // switch values:
@ -742,7 +746,12 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
file.cleanLast(le, 0); file.cleanLast(le, 0);
assert file.size() < fsb : "file.size() = " + file.size(); assert file.size() < fsb : "file.size() = " + file.size();
final Row.Entry lr = rowdef.newEntry(le); final Row.Entry lr = rowdef.newEntry(le);
assert lr != null;
assert lr.getPrimaryKeyBytes() != null;
final int is = index.size();
assert index.has(lr.getPrimaryKeyBytes());
final int i = (int) index.remove(lr.getPrimaryKeyBytes()); final int i = (int) index.remove(lr.getPrimaryKeyBytes());
assert i < 0 || index.size() < is : "index.size() = " + index.size() + ", is = " + is;
assert i >= 0; assert i >= 0;
if (table != null) table.removeOne(); if (table != null) table.removeOne();
assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size(); assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size();

Write Preview
Loading…
Cancel
Save