boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fix for http://forum.yacy-websuche.de/viewtopic.php?p=18500#p18500

Browse Source 
- catch uncatched OOM
- less wasting of memory

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6555 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent 32972139af
commit eeca2ded92
23 changed files with 142 additions and 79 deletions
Show Stats Download Patch File Download Diff File

2
source/de/anomic/crawler/ZURL.java
Unescape View File

@ -95,7 +95,7 @@ public class ZURL implements Iterable<ZURL.Entry> {
public ZURL() { public ZURL() {
// creates a new ZUR in RAM // creates a new ZUR in RAM
this.urlIndex = new RowSet(rowdef, 0); this.urlIndex = new RowSet(rowdef);
this.stack = new ConcurrentLinkedQueue<String>(); this.stack = new ConcurrentLinkedQueue<String>();
} }

2
source/de/anomic/data/URLAnalysis.java
Unescape View File

@ -416,7 +416,7 @@ public class URLAnalysis {
System.out.println("INDEX DIFF URL-COL startup"); System.out.println("INDEX DIFF URL-COL startup");
HandleMap idx = new HandleMap(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 4, new File(statisticFile), 0); HandleMap idx = new HandleMap(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 4, new File(statisticFile), 0);
MetadataRepository mr = new MetadataRepository(new File(metadataPath), "text.urlmd", false, false); MetadataRepository mr = new MetadataRepository(new File(metadataPath), "text.urlmd", false, false);
HandleSet hs = new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0, 1000000); HandleSet hs = new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 1000000, 0);
System.out.println("INDEX DIFF URL-COL loaded dump, starting diff"); System.out.println("INDEX DIFF URL-COL loaded dump, starting diff");
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
long update = start - 7000; long update = start - 7000;

2
source/de/anomic/yacy/dht/FlatWordPartitionScheme.java
Unescape View File

@ -111,7 +111,7 @@ public class FlatWordPartitionScheme implements PartitionScheme {
System.gc(); // for resource measurement System.gc(); // for resource measurement
long a = MemoryControl.available(); long a = MemoryControl.available();
HandleMap idx = new HandleMap(12, Base64Order.enhancedCoder, 4, 0, 150000); HandleMap idx = new HandleMap(12, Base64Order.enhancedCoder, 4, 150000);
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
try { try {
idx.inc(FlatWordPartitionScheme.positionToHash(r.nextInt(count))); idx.inc(FlatWordPartitionScheme.positionToHash(r.nextInt(count)));

7
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -538,7 +538,12 @@ public final class yacyClient {
assert words > 0 : "wordhashes = " + wordhashes; assert words > 0 : "wordhashes = " + wordhashes;
final ReferenceContainer<WordReference>[] container = new ReferenceContainer[words]; final ReferenceContainer<WordReference>[] container = new ReferenceContainer[words];
for (int i = 0; i < words; i++) { for (int i = 0; i < words; i++) {
container[i] = ReferenceContainer.emptyContainer(Segment.wordReferenceFactory, wordhashes.substring(i * Word.commonHashLength, (i + 1) * Word.commonHashLength).getBytes(), count); try {
container[i] = ReferenceContainer.emptyContainer(Segment.wordReferenceFactory, wordhashes.substring(i * Word.commonHashLength, (i + 1) * Word.commonHashLength).getBytes(), count);
} catch (RowSpaceExceededException e) {
Log.logException(e);
return null;
}
} }
// insert results to containers // insert results to containers

2
source/net/yacy/kelondro/blob/HeapReader.java
Unescape View File

@ -166,7 +166,7 @@ public class HeapReader {
Log.logInfo("HeapReader", "generating index for " + heapFile.toString() + ", " + (file.length() / 1024 / 1024) + " MB. Please wait."); Log.logInfo("HeapReader", "generating index for " + heapFile.toString() + ", " + (file.length() / 1024 / 1024) + " MB. Please wait.");
this.free = new Gap(); this.free = new Gap();
HandleMap.initDataConsumer indexready = HandleMap.asynchronusInitializer(keylength, this.ordering, 8, 0, Math.max(10, (int) (Runtime.getRuntime().freeMemory() / (10 * 1024 * 1024)))); HandleMap.initDataConsumer indexready = HandleMap.asynchronusInitializer(keylength, this.ordering, 8, Math.max(10, (int) (Runtime.getRuntime().freeMemory() / (10 * 1024 * 1024))));
byte[] key = new byte[keylength]; byte[] key = new byte[keylength];
int reclen; int reclen;
long seek = 0; long seek = 0;

2
source/net/yacy/kelondro/blob/HeapWriter.java
Unescape View File

@ -78,7 +78,7 @@ public final class HeapWriter {
this.heapFileTMP = temporaryHeapFile; this.heapFileTMP = temporaryHeapFile;
this.heapFileREADY = readyHeapFile; this.heapFileREADY = readyHeapFile;
this.keylength = keylength; this.keylength = keylength;
this.index = new HandleMap(keylength, ordering, 8, 10, 100000); this.index = new HandleMap(keylength, ordering, 8, 100000);
this.os = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(temporaryHeapFile), outBuffer)); this.os = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(temporaryHeapFile), outBuffer));
//this.doublecheck = new HashSet<String>(); //this.doublecheck = new HashSet<String>();
this.seek = 0; this.seek = 0;

4
source/net/yacy/kelondro/index/Cache.java
Unescape View File

@ -82,8 +82,8 @@ public final class Cache implements ObjectIndex, Iterable<Row.Entry> {
private void init() { private void init() {
this.keyrow = new Row(new Column[]{index.row().column(0)}, index.row().objectOrder); this.keyrow = new Row(new Column[]{index.row().column(0)}, index.row().objectOrder);
this.readHitCache = new RowSet(index.row(), 0); this.readHitCache = new RowSet(index.row());
this.readMissCache = new RowSet(this.keyrow, 0); this.readMissCache = new RowSet(this.keyrow);
this.readHit = 0; this.readHit = 0;
this.readMiss = 0; this.readMiss = 0;
this.writeUnique = 0; this.writeUnique = 0;

15
source/net/yacy/kelondro/index/HandleMap.java
Unescape View File

@ -62,9 +62,14 @@ public final class HandleMap implements Iterable<Row.Entry> {
* @param objectOrder * @param objectOrder
* @param space * @param space
*/ */
public HandleMap(final int keylength, final ByteOrder objectOrder, int idxbytes, final int initialspace, final int expectedspace) { public HandleMap(final int keylength, final ByteOrder objectOrder, int idxbytes, final int expectedspace) {
this.rowdef = new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, keylength, "key"), new Column("long c-" + idxbytes + " {b256}")}, objectOrder); this.rowdef = new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, keylength, "key"), new Column("long c-" + idxbytes + " {b256}")}, objectOrder);
this.index = new ObjectIndexCache(rowdef, initialspace, expectedspace); this.index = new ObjectIndexCache(rowdef, expectedspace);
}
public HandleMap(final int keylength, final ByteOrder objectOrder, int idxbytes, final int expectedspace, final int initialspace) throws RowSpaceExceededException {
this.rowdef = new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, keylength, "key"), new Column("long c-" + idxbytes + " {b256}")}, objectOrder);
this.index = new ObjectIndexCache(rowdef, expectedspace, initialspace);
} }
/** /**
@ -76,7 +81,7 @@ public final class HandleMap implements Iterable<Row.Entry> {
* @throws RowSpaceExceededException * @throws RowSpaceExceededException
*/ */
public HandleMap(final int keylength, final ByteOrder objectOrder, int idxbytes, final File file, final int expectedspace) throws IOException, RowSpaceExceededException { public HandleMap(final int keylength, final ByteOrder objectOrder, int idxbytes, final File file, final int expectedspace) throws IOException, RowSpaceExceededException {
this(keylength, objectOrder, idxbytes, (int) (file.length() / (keylength + idxbytes)), expectedspace); this(keylength, objectOrder, idxbytes, expectedspace, (int) (file.length() / (keylength + idxbytes)));
// read the index dump and fill the index // read the index dump and fill the index
InputStream is = new BufferedInputStream(new FileInputStream(file), 1024 * 1024); InputStream is = new BufferedInputStream(new FileInputStream(file), 1024 * 1024);
if (file.getName().endsWith(".gz")) is = new GZIPInputStream(is); if (file.getName().endsWith(".gz")) is = new GZIPInputStream(is);
@ -290,8 +295,8 @@ public final class HandleMap implements Iterable<Row.Entry> {
* @param bufferSize * @param bufferSize
* @return * @return
*/ */
public final static initDataConsumer asynchronusInitializer(final int keylength, final ByteOrder objectOrder, int idxbytes, final int space, final int expectedspace) { public final static initDataConsumer asynchronusInitializer(final int keylength, final ByteOrder objectOrder, int idxbytes, final int expectedspace) {
initDataConsumer initializer = new initDataConsumer(new HandleMap(keylength, objectOrder, idxbytes, space, expectedspace)); initDataConsumer initializer = new initDataConsumer(new HandleMap(keylength, objectOrder, idxbytes, expectedspace));
ExecutorService service = Executors.newSingleThreadExecutor(); ExecutorService service = Executors.newSingleThreadExecutor();
initializer.setResult(service.submit(initializer)); initializer.setResult(service.submit(initializer));
service.shutdown(); service.shutdown();

11
source/net/yacy/kelondro/index/HandleSet.java
Unescape View File

@ -44,9 +44,14 @@ public final class HandleSet implements Iterable<byte[]> {
private final Row rowdef; private final Row rowdef;
private ObjectIndex index; private ObjectIndex index;
public HandleSet(final int keylength, final ByteOrder objectOrder, final int initialspace, final int expectedspace) { public HandleSet(final int keylength, final ByteOrder objectOrder, final int expectedspace) {
this.rowdef = new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, keylength, "key")}, objectOrder); this.rowdef = new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, keylength, "key")}, objectOrder);
this.index = new ObjectIndexCache(rowdef, initialspace, expectedspace); this.index = new ObjectIndexCache(rowdef, expectedspace);
}
public HandleSet(final int keylength, final ByteOrder objectOrder, final int expectedspace, final int initialspace) throws RowSpaceExceededException {
this.rowdef = new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, keylength, "key")}, objectOrder);
this.index = new ObjectIndexCache(rowdef, expectedspace, initialspace);
} }
/** /**
@ -58,7 +63,7 @@ public final class HandleSet implements Iterable<byte[]> {
* @throws RowSpaceExceededException * @throws RowSpaceExceededException
*/ */
public HandleSet(final int keylength, final ByteOrder objectOrder, final File file, final int expectedspace) throws IOException, RowSpaceExceededException { public HandleSet(final int keylength, final ByteOrder objectOrder, final File file, final int expectedspace) throws IOException, RowSpaceExceededException {
this(keylength, objectOrder, (int) (file.length() / (keylength + 8)), expectedspace); this(keylength, objectOrder, expectedspace, (int) (file.length() / (keylength + 8)));
// read the index dump and fill the index // read the index dump and fill the index
InputStream is = new BufferedInputStream(new FileInputStream(file), 1024 * 1024); InputStream is = new BufferedInputStream(new FileInputStream(file), 1024 * 1024);
byte[] a = new byte[keylength]; byte[] a = new byte[keylength];

7
source/net/yacy/kelondro/index/IndexTest.java
Unescape View File

@ -112,7 +112,12 @@ public class IndexTest {
System.out.println("sorted map"); System.out.println("sorted map");
Runtime.getRuntime().gc(); Runtime.getRuntime().gc();
long freeStartKelondro = MemoryControl.available(); long freeStartKelondro = MemoryControl.available();
HandleMap ii = new HandleMap(12, Base64Order.enhancedCoder, 4, count, count); HandleMap ii = null;
try {
ii = new HandleMap(12, Base64Order.enhancedCoder, 4, count, count);
} catch (RowSpaceExceededException e1) {
e1.printStackTrace();
}
for (int i = 0; i < count; i++) for (int i = 0; i < count; i++)
try { try {
ii.putUnique(tests[i], 1); ii.putUnique(tests[i], 1);

23
source/net/yacy/kelondro/index/ObjectArrayCache.java
Unescape View File

@ -46,11 +46,16 @@ public final class ObjectArrayCache {
private RowSet index1; private RowSet index1;
//private final kelondroOrder<kelondroRow.Entry> entryOrder; //private final kelondroOrder<kelondroRow.Entry> entryOrder;
public ObjectArrayCache(final int payloadSize, final int initSize) { public ObjectArrayCache(final int payloadSize) {
this.rowdef = new Row("Cardinal key-4 {b256}, byte[] payload-" + payloadSize, NaturalOrder.naturalOrder); this.rowdef = new Row("Cardinal key-4 {b256}, byte[] payload-" + payloadSize, NaturalOrder.naturalOrder);
this.index0 = new RowSet(rowdef, initSize); this.index0 = new RowSet(rowdef);
this.index1 = null; this.index1 = null;
//this.entryOrder = new kelondroRow.EntryComparator(rowdef.objectOrder); }
public ObjectArrayCache(final int payloadSize, final int initSize) throws RowSpaceExceededException {
this.rowdef = new Row("Cardinal key-4 {b256}, byte[] payload-" + payloadSize, NaturalOrder.naturalOrder);
this.index0 = new RowSet(rowdef, initSize);
this.index1 = null;
} }
public final long memoryNeededForGrow() { public final long memoryNeededForGrow() {
@ -71,7 +76,7 @@ public final class ObjectArrayCache {
// finish initialization phase // finish initialization phase
index0.sort(); index0.sort();
index0.uniq(); index0.uniq();
index1 = new RowSet(rowdef, 0); index1 = new RowSet(rowdef);
} }
final Row.Entry indexentry = index0.get(key); final Row.Entry indexentry = index0.get(key);
if (indexentry != null) return indexentry.getColBytes(1); if (indexentry != null) return indexentry.getColBytes(1);
@ -133,7 +138,7 @@ public final class ObjectArrayCache {
// finish initialization phase // finish initialization phase
index0.sort(); index0.sort();
index0.uniq(); index0.uniq();
index1 = new RowSet(rowdef, 0); index1 = new RowSet(rowdef);
} }
final Row.Entry indexentry = index0.remove(key); final Row.Entry indexentry = index0.remove(key);
if (indexentry != null) { if (indexentry != null) {
@ -187,7 +192,7 @@ public final class ObjectArrayCache {
// finish initialization phase // finish initialization phase
index0.sort(); index0.sort();
index0.uniq(); index0.uniq();
index1 = new RowSet(rowdef, 0); index1 = new RowSet(rowdef);
} }
return index0.rows(); return index0.rows();
} else { } else {
@ -219,7 +224,7 @@ public final class ObjectArrayCache {
int p, rc = 0; int p, rc = 0;
final ArrayList<Long> ra = new ArrayList<Long>(); final ArrayList<Long> ra = new ArrayList<Long>();
final HashSet<Long> jcontrol = new HashSet<Long>(); final HashSet<Long> jcontrol = new HashSet<Long>();
final ObjectArrayCache kcontrol = new ObjectArrayCache(1, 0); final ObjectArrayCache kcontrol = new ObjectArrayCache(1);
for (int i = 0; i < 1000000; i++) { for (int i = 0; i < 1000000; i++) {
r = Math.abs(random.nextLong() % 10000); r = Math.abs(random.nextLong() % 10000);
//System.out.println("add " + r); //System.out.println("add " + r);

41
source/net/yacy/kelondro/index/ObjectIndexCache.java
Unescape View File

@ -42,22 +42,35 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable<Row.Entry>
private final Row.EntryComparator entryComparator; private final Row.EntryComparator entryComparator;
private final int spread; private final int spread;
public ObjectIndexCache(final Row rowdef, final int initialspace, final int expectedspace) { public ObjectIndexCache(final Row rowdef, final int expectedspace) {
this.rowdef = rowdef; this.rowdef = rowdef;
this.entryComparator = new Row.EntryComparator(rowdef.objectOrder); this.entryComparator = new Row.EntryComparator(rowdef.objectOrder);
this.spread = Math.max(10, expectedspace / 3000); this.spread = Math.max(10, expectedspace / 3000);
reset(initialspace); reset();
}
public ObjectIndexCache(final Row rowdef, final int expectedspace, final int initialspace) throws RowSpaceExceededException {
this.rowdef = rowdef;
this.entryComparator = new Row.EntryComparator(rowdef.objectOrder);
this.spread = Math.max(10, expectedspace / 3000);
reset(initialspace);
} }
public void clear() { public void clear() {
reset(0); reset();
} }
public final synchronized void reset(final int initialspace) { public final synchronized void reset() {
this.index0 = null; // first flush RAM to make room this.index0 = null; // first flush RAM to make room
this.index0 = new RowSet(rowdef, initialspace); this.index0 = new RowSet(rowdef);
this.index1 = null; // to show that this is the initialization phase this.index1 = null; // to show that this is the initialization phase
} }
public final synchronized void reset(final int initialspace) throws RowSpaceExceededException {
this.index0 = null; // first flush RAM to make room
this.index0 = new RowSet(rowdef, initialspace);
this.index1 = null; // to show that this is the initialization phase
}
public final Row row() { public final Row row() {
return index0.row(); return index0.row();
@ -69,7 +82,7 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable<Row.Entry>
index0.sort(); index0.sort();
index0.uniq(); index0.uniq();
index0.trim(false); index0.trim(false);
index1 = new RowSetArray(rowdef, 0, spread); index1 = new RowSetArray(rowdef, spread);
} }
} }
@ -230,7 +243,7 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable<Row.Entry>
// finish initialization phase // finish initialization phase
index0.sort(); index0.sort();
index0.uniq(); index0.uniq();
index1 = new RowSetArray(rowdef, 0, spread); index1 = new RowSetArray(rowdef, spread);
return index0.keys(up, firstKey); return index0.keys(up, firstKey);
} }
assert (index1 != null); assert (index1 != null);
@ -259,7 +272,7 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable<Row.Entry>
// finish initialization phase // finish initialization phase
index0.sort(); index0.sort();
index0.uniq(); index0.uniq();
index1 = new RowSetArray(rowdef, 0, spread); index1 = new RowSetArray(rowdef, spread);
return index0.rows(up, firstKey); return index0.rows(up, firstKey);
} }
assert (index1 != null); assert (index1 != null);
@ -293,7 +306,7 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable<Row.Entry>
// finish initialization phase // finish initialization phase
index0.sort(); index0.sort();
index0.uniq(); index0.uniq();
index1 = new RowSetArray(rowdef, 0, spread); index1 = new RowSetArray(rowdef, spread);
return index0.rows(); return index0.rows();
} }
assert (index1 != null); assert (index1 != null);

12
source/net/yacy/kelondro/index/RowCollection.java
Unescape View File

@ -79,12 +79,17 @@ public class RowCollection implements Iterable<Row.Entry> {
this.lastTimeWrote = rc.lastTimeWrote; this.lastTimeWrote = rc.lastTimeWrote;
} }
public RowCollection(final Row rowdef, final int objectCount) { public RowCollection(final Row rowdef) {
this.rowdef = rowdef; this.rowdef = rowdef;
this.chunkcache = new byte[objectCount * rowdef.objectsize];
this.chunkcount = 0;
this.sortBound = 0; this.sortBound = 0;
this.lastTimeWrote = System.currentTimeMillis(); this.lastTimeWrote = System.currentTimeMillis();
this.chunkcache = new byte[0];
this.chunkcount = 0;
}
public RowCollection(final Row rowdef, final int objectCount) throws RowSpaceExceededException {
this(rowdef);
ensureSize(objectCount);
} }
public RowCollection(final Row rowdef, final int objectCount, final byte[] cache, final int sortBound) { public RowCollection(final Row rowdef, final int objectCount, final byte[] cache, final int sortBound) {
@ -218,6 +223,7 @@ public class RowCollection implements Iterable<Row.Entry> {
} }
protected final void ensureSize(final int elements) throws RowSpaceExceededException { protected final void ensureSize(final int elements) throws RowSpaceExceededException {
if (elements == 0) return;
long allocram = neededSpaceForEnsuredSize(elements, true); long allocram = neededSpaceForEnsuredSize(elements, true);
if (allocram == 0) return; if (allocram == 0) return;
assert allocram > chunkcache.length : "wrong alloc computation: allocram = " + allocram + ", chunkcache.length = " + chunkcache.length; assert allocram > chunkcache.length : "wrong alloc computation: allocram = " + allocram + ", chunkcache.length = " + chunkcache.length;

19
source/net/yacy/kelondro/index/RowSet.java
Unescape View File

@ -46,11 +46,16 @@ public class RowSet extends RowCollection implements ObjectIndex, Iterable<Row.E
assert rowdef.objectOrder != null; assert rowdef.objectOrder != null;
} }
public RowSet(final Row rowdef, final int objectCount) { public RowSet(final Row rowdef, final int objectCount) throws RowSpaceExceededException {
super(rowdef, objectCount); super(rowdef, objectCount);
assert rowdef.objectOrder != null; assert rowdef.objectOrder != null;
} }
public RowSet(final Row rowdef) {
super(rowdef);
assert rowdef.objectOrder != null;
}
/** /**
* import an exported collection * import an exported collection
* @param rowdef * @param rowdef
@ -64,16 +69,16 @@ public class RowSet extends RowCollection implements ObjectIndex, Iterable<Row.E
public final static RowSet importRowSet(byte[] b, final Row rowdef) { public final static RowSet importRowSet(byte[] b, final Row rowdef) {
assert b.length >= exportOverheadSize : "b.length = " + b.length; assert b.length >= exportOverheadSize : "b.length = " + b.length;
if (b.length < exportOverheadSize) return new RowSet(rowdef, 0); if (b.length < exportOverheadSize) return new RowSet(rowdef);
final int size = (int) NaturalOrder.decodeLong(b, 0, 4); final int size = (int) NaturalOrder.decodeLong(b, 0, 4);
assert size >= 0 : "size = " + size; assert size >= 0 : "size = " + size;
if (size < 0) return new RowSet(rowdef, 0); if (size < 0) return new RowSet(rowdef);
final int orderbound = (int) NaturalOrder.decodeLong(b, 10, 4); final int orderbound = (int) NaturalOrder.decodeLong(b, 10, 4);
assert orderbound >= 0 : "orderbound = " + orderbound; assert orderbound >= 0 : "orderbound = " + orderbound;
if (orderbound < 0) return new RowSet(rowdef, 0); if (orderbound < 0) return new RowSet(rowdef);
final byte[] chunkcache = new byte[size * rowdef.objectsize]; final byte[] chunkcache = new byte[size * rowdef.objectsize];
assert b.length - exportOverheadSize == size * rowdef.objectsize; assert b.length - exportOverheadSize == size * rowdef.objectsize;
if (b.length - exportOverheadSize != size * rowdef.objectsize) return new RowSet(rowdef, 0); if (b.length - exportOverheadSize != size * rowdef.objectsize) return new RowSet(rowdef);
System.arraycopy(b, exportOverheadSize, chunkcache, 0, chunkcache.length); System.arraycopy(b, exportOverheadSize, chunkcache, 0, chunkcache.length);
return new RowSet(rowdef, size, chunkcache, orderbound); return new RowSet(rowdef, size, chunkcache, orderbound);
} }
@ -486,7 +491,7 @@ public class RowSet extends RowCollection implements ObjectIndex, Iterable<Row.E
"acht......xxxx", "acht......xxxx",
"neun......xxxx", "neun......xxxx",
"zehn......xxxx" }; "zehn......xxxx" };
final RowSet d = new RowSet(new Row("byte[] key-10, Cardinal x-4 {b256}", NaturalOrder.naturalOrder), 0); final RowSet d = new RowSet(new Row("byte[] key-10, Cardinal x-4 {b256}", NaturalOrder.naturalOrder));
for (int ii = 0; ii < test.length; ii++) for (int ii = 0; ii < test.length; ii++)
try { try {
d.add(test[ii].getBytes()); d.add(test[ii].getBytes());
@ -583,7 +588,7 @@ public class RowSet extends RowCollection implements ObjectIndex, Iterable<Row.E
// remove test // remove test
final long start = System.currentTimeMillis(); final long start = System.currentTimeMillis();
final RowSet c = new RowSet(new Row("byte[] a-12, byte[] b-12", Base64Order.enhancedCoder), 0); final RowSet c = new RowSet(new Row("byte[] a-12, byte[] b-12", Base64Order.enhancedCoder));
byte[] key; byte[] key;
final int testsize = 5000; final int testsize = 5000;
final byte[][] delkeys = new byte[testsize / 5][]; final byte[][] delkeys = new byte[testsize / 5][];

8
source/net/yacy/kelondro/index/RowSetArray.java
Unescape View File

@ -34,17 +34,15 @@ import net.yacy.kelondro.order.StackIterator;
public final class RowSetArray implements ObjectIndex, Iterable<Row.Entry> { public final class RowSetArray implements ObjectIndex, Iterable<Row.Entry> {
private final int objectCount;
private final Row rowdef; private final Row rowdef;
private final RowSet[] array; private final RowSet[] array;
public RowSetArray(final Row rowdef, final int objectCount, final int arraySize) { public RowSetArray(final Row rowdef, final int arraySize) {
this.array = new RowSet[arraySize]; this.array = new RowSet[arraySize];
this.rowdef = rowdef;
for (int i = 0; i < arraySize; i++) { for (int i = 0; i < arraySize; i++) {
this.array[i] = null; this.array[i] = null;
} }
this.rowdef = rowdef;
this.objectCount = objectCount / arraySize;
} }
private final int indexFor(byte[] key) { private final int indexFor(byte[] key) {
@ -58,7 +56,7 @@ public final class RowSetArray implements ObjectIndex, Iterable<Row.Entry> {
private final RowSet accessArray(int i) { private final RowSet accessArray(int i) {
RowSet r = this.array[i]; RowSet r = this.array[i];
if (r == null) synchronized (this.array) { if (r == null) synchronized (this.array) {
r = new RowSet(this.rowdef, this.objectCount); r = new RowSet(this.rowdef);
this.array[i] = r; this.array[i] = r;
} }
return r; return r;

2
source/net/yacy/kelondro/rwi/AbstractBufferedIndex.java
Unescape View File

@ -46,7 +46,7 @@ public abstract class AbstractBufferedIndex<ReferenceType extends Reference> ext
// this does not use the cache // this does not use the cache
final Order<ReferenceContainer<ReferenceType>> containerOrder = new ReferenceContainerOrder<ReferenceType>(factory, this.ordering().clone()); final Order<ReferenceContainer<ReferenceType>> containerOrder = new ReferenceContainerOrder<ReferenceType>(factory, this.ordering().clone());
if (startHash.length == 0) startHash = null; if (startHash.length == 0) startHash = null;
ReferenceContainer<ReferenceType> emptyContainer = ReferenceContainer.emptyContainer(factory, startHash, 0); ReferenceContainer<ReferenceType> emptyContainer = ReferenceContainer.emptyContainer(factory, startHash);
containerOrder.rotate(emptyContainer); containerOrder.rotate(emptyContainer);
final TreeSet<ReferenceContainer<ReferenceType>> containers = new TreeSet<ReferenceContainer<ReferenceType>>(containerOrder); final TreeSet<ReferenceContainer<ReferenceType>> containers = new TreeSet<ReferenceContainer<ReferenceType>>(containerOrder);
final Iterator<ReferenceContainer<ReferenceType>> i = references(startHash, rot, ram); final Iterator<ReferenceContainer<ReferenceType>> i = references(startHash, rot, ram);

2
source/net/yacy/kelondro/rwi/AbstractIndex.java
Unescape View File

@ -60,7 +60,7 @@ public abstract class AbstractIndex <ReferenceType extends Reference> implements
// creates a set of indexContainers // creates a set of indexContainers
// this does not use the cache // this does not use the cache
final Order<ReferenceContainer<ReferenceType>> containerOrder = new ReferenceContainerOrder<ReferenceType>(factory, this.ordering().clone()); final Order<ReferenceContainer<ReferenceType>> containerOrder = new ReferenceContainerOrder<ReferenceType>(factory, this.ordering().clone());
ReferenceContainer<ReferenceType> emptyContainer = ReferenceContainer.emptyContainer(factory, startHash, 0); ReferenceContainer<ReferenceType> emptyContainer = ReferenceContainer.emptyContainer(factory, startHash);
containerOrder.rotate(emptyContainer); containerOrder.rotate(emptyContainer);
final TreeSet<ReferenceContainer<ReferenceType>> containers = new TreeSet<ReferenceContainer<ReferenceType>>(containerOrder); final TreeSet<ReferenceContainer<ReferenceType>> containers = new TreeSet<ReferenceContainer<ReferenceType>>(containerOrder);
final Iterator<ReferenceContainer<ReferenceType>> i = references(startHash, rot); final Iterator<ReferenceContainer<ReferenceType>> i = references(startHash, rot);

4
source/net/yacy/kelondro/rwi/IndexCell.java
Unescape View File

@ -295,7 +295,7 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
public CloneableIterator<ReferenceContainer<ReferenceType>> references(byte[] starttermHash, boolean rot) { public CloneableIterator<ReferenceContainer<ReferenceType>> references(byte[] starttermHash, boolean rot) {
final Order<ReferenceContainer<ReferenceType>> containerOrder = new ReferenceContainerOrder<ReferenceType>(factory, this.ram.rowdef().getOrdering().clone()); final Order<ReferenceContainer<ReferenceType>> containerOrder = new ReferenceContainerOrder<ReferenceType>(factory, this.ram.rowdef().getOrdering().clone());
containerOrder.rotate(new ReferenceContainer<ReferenceType>(factory, starttermHash, 0)); containerOrder.rotate(new ReferenceContainer<ReferenceType>(factory, starttermHash));
return new MergeIterator<ReferenceContainer<ReferenceType>>( return new MergeIterator<ReferenceContainer<ReferenceType>>(
this.ram.references(starttermHash, rot), this.ram.references(starttermHash, rot),
new MergeIterator<ReferenceContainer<ReferenceType>>( new MergeIterator<ReferenceContainer<ReferenceType>>(
@ -311,7 +311,7 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
public CloneableIterator<ReferenceContainer<ReferenceType>> references(byte[] startTermHash, boolean rot, boolean ram) { public CloneableIterator<ReferenceContainer<ReferenceType>> references(byte[] startTermHash, boolean rot, boolean ram) {
final Order<ReferenceContainer<ReferenceType>> containerOrder = new ReferenceContainerOrder<ReferenceType>(factory, this.ram.rowdef().getOrdering().clone()); final Order<ReferenceContainer<ReferenceType>> containerOrder = new ReferenceContainerOrder<ReferenceType>(factory, this.ram.rowdef().getOrdering().clone());
containerOrder.rotate(new ReferenceContainer<ReferenceType>(factory, startTermHash, 0)); containerOrder.rotate(new ReferenceContainer<ReferenceType>(factory, startTermHash));
if (ram) { if (ram) {
return this.ram.references(startTermHash, rot); return this.ram.references(startTermHash, rot);
} }

19
source/net/yacy/kelondro/rwi/ReferenceContainer.java
Unescape View File

@ -62,7 +62,15 @@ public class ReferenceContainer<ReferenceType extends Reference> extends RowSet
this.termHash = termHash; this.termHash = termHash;
} }
public ReferenceContainer(final ReferenceFactory<ReferenceType> factory, final byte[] termHash, final int objectCount) { public ReferenceContainer(final ReferenceFactory<ReferenceType> factory, final byte[] termHash) {
super(factory.getRow());
assert termHash == null || (termHash[2] != '@' && termHash.length == this.rowdef.primaryKeyLength);
this.termHash = termHash;
this.factory = factory;
this.lastTimeWrote = 0;
}
public ReferenceContainer(final ReferenceFactory<ReferenceType> factory, final byte[] termHash, final int objectCount) throws RowSpaceExceededException {
super(factory.getRow(), objectCount); super(factory.getRow(), objectCount);
assert termHash == null || (termHash[2] != '@' && termHash.length == this.rowdef.primaryKeyLength); assert termHash == null || (termHash[2] != '@' && termHash.length == this.rowdef.primaryKeyLength);
this.termHash = termHash; this.termHash = termHash;
@ -76,8 +84,13 @@ public class ReferenceContainer<ReferenceType extends Reference> extends RowSet
return newContainer; return newContainer;
} }
public static <ReferenceType extends Reference> ReferenceContainer<ReferenceType> emptyContainer(final ReferenceFactory<ReferenceType> factory, final byte[] termHash, final int elementCount) { public static <ReferenceType extends Reference> ReferenceContainer<ReferenceType> emptyContainer(final ReferenceFactory<ReferenceType> factory, final byte[] termHash) {
assert termHash == null || (termHash[2] != '@' && termHash.length == factory.getRow().primaryKeyLength); assert termHash == null || (termHash[2] != '@' && termHash.length == factory.getRow().primaryKeyLength);
return new ReferenceContainer<ReferenceType>(factory, termHash);
}
public static <ReferenceType extends Reference> ReferenceContainer<ReferenceType> emptyContainer(final ReferenceFactory<ReferenceType> factory, final byte[] termHash, final int elementCount) throws RowSpaceExceededException {
assert termHash == null || (termHash[2] != '@' && termHash.length == factory.getRow().primaryKeyLength);
return new ReferenceContainer<ReferenceType>(factory, termHash, elementCount); return new ReferenceContainer<ReferenceType>(factory, termHash, elementCount);
} }

2
source/net/yacy/kelondro/rwi/ReferenceContainerArray.java
Unescape View File

@ -329,7 +329,7 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
final Row payloadrow) throws IOException, RowSpaceExceededException { final Row payloadrow) throws IOException, RowSpaceExceededException {
System.out.println("CELL REFERENCE COLLECTION startup"); System.out.println("CELL REFERENCE COLLECTION startup");
HandleMap references = new HandleMap(payloadrow.primaryKeyLength, termOrder, 4, 0, 1000000); HandleMap references = new HandleMap(payloadrow.primaryKeyLength, termOrder, 4, 1000000, 0);
String[] files = heapLocation.list(); String[] files = heapLocation.list();
for (String f: files) { for (String f: files) {
if (f.length() < 22 || !f.startsWith("text.index") || !f.endsWith(".blob")) continue; if (f.length() < 22 || !f.startsWith("text.index") || !f.endsWith(".blob")) continue;

25
source/net/yacy/kelondro/rwi/ReferenceContainerCache.java
Unescape View File

@ -282,25 +282,28 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
* get a indexContainer from a heap * get a indexContainer from a heap
* @param key * @param key
* @return the indexContainer if one exist, null otherwise * @return the indexContainer if one exist, null otherwise
* @throws
*/ */
public ReferenceContainer<ReferenceType> get(final byte[] key, Set<String> urlselection) { public ReferenceContainer<ReferenceType> get(final byte[] key, Set<String> urlselection) {
ReferenceContainer<ReferenceType> c = this.cache.get(new ByteArray(key)); ReferenceContainer<ReferenceType> c = this.cache.get(new ByteArray(key));
if (urlselection == null) return c; if (urlselection == null) return c;
if (c == null) return null; if (c == null) return null;
// because this is all in RAM, we must clone the entries (flat) // because this is all in RAM, we must clone the entries (flat)
ReferenceContainer<ReferenceType> c1 = new ReferenceContainer<ReferenceType>(factory, c.getTermHash(), c.size()); try {
Iterator<ReferenceType> e = c.entries(); ReferenceContainer<ReferenceType> c1 = new ReferenceContainer<ReferenceType>(factory, c.getTermHash(), c.size());
ReferenceType ee; Iterator<ReferenceType> e = c.entries();
while (e.hasNext()) { ReferenceType ee;
ee = e.next(); while (e.hasNext()) {
if (urlselection.contains(ee.metadataHash())) try { ee = e.next();
c1.add(ee); if (urlselection.contains(ee.metadataHash())) {
} catch (RowSpaceExceededException e1) { c1.add(ee);
Log.logException(e1); }
break;
} }
return c1;
} catch (RowSpaceExceededException e2) {
Log.logException(e2);
} }
return c1; return null;
} }
/** /**

2
source/net/yacy/kelondro/table/SplitTable.java
Unescape View File

@ -68,7 +68,7 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
// the table type can be either kelondroFlex or kelondroEco // the table type can be either kelondroFlex or kelondroEco
private static final int EcoFSBufferSize = 20; private static final int EcoFSBufferSize = 20;
static final ObjectIndex dummyIndex = new ObjectIndexCache(new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, 2, "key")}, NaturalOrder.naturalOrder), 0, 0); static final ObjectIndex dummyIndex = new ObjectIndexCache(new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, 2, "key")}, NaturalOrder.naturalOrder), 0);
// the thread pool for the keeperOf executor service // the thread pool for the keeperOf executor service
private ExecutorService executor; private ExecutorService executor;

8
source/net/yacy/kelondro/table/Table.java
Unescape View File

@ -135,8 +135,8 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
table = null; System.gc(); table = null; System.gc();
Log.logSevere("TABLE", tablefile.getName() + ": RAM after releasing the table: " + (MemoryControl.available() / 1024 / 1024) + "MB"); Log.logSevere("TABLE", tablefile.getName() + ": RAM after releasing the table: " + (MemoryControl.available() / 1024 / 1024) + "MB");
} }
index = new HandleMap(rowdef.primaryKeyLength, rowdef.objectOrder, 4, records, 100000); index = new HandleMap(rowdef.primaryKeyLength, rowdef.objectOrder, 4, records);
HandleMap errors = new HandleMap(rowdef.primaryKeyLength, NaturalOrder.naturalOrder, 4, records, 10); HandleMap errors = new HandleMap(rowdef.primaryKeyLength, NaturalOrder.naturalOrder, 4, records);
Log.logInfo("TABLE", tablefile + ": TABLE " + tablefile.toString() + " has table copy " + ((table == null) ? "DISABLED" : "ENABLED")); Log.logInfo("TABLE", tablefile + ": TABLE " + tablefile.toString() + " has table copy " + ((table == null) ? "DISABLED" : "ENABLED"));
// read all elements from the file into the copy table // read all elements from the file into the copy table
@ -695,8 +695,8 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
} }
// initialize index and copy table // initialize index and copy table
table = (table == null) ? null : new RowSet(taildef, 1); table = (table == null) ? null : new RowSet(taildef);
index = new HandleMap(rowdef.primaryKeyLength, rowdef.objectOrder, 4, 1, 100000); index = new HandleMap(rowdef.primaryKeyLength, rowdef.objectOrder, 4, 100000);
} }
public Row row() { public Row row() {

Write Preview
Loading…
Cancel
Save