From eeca2ded924774e94527dc9edebe70fc55c6f42b Mon Sep 17 00:00:00 2001 From: orbiter Date: Sat, 9 Jan 2010 00:08:16 +0000 Subject: [PATCH] fix for http://forum.yacy-websuche.de/viewtopic.php?p=18500#p18500 - catch uncatched OOM - less wasting of memory git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6555 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/de/anomic/crawler/ZURL.java | 2 +- source/de/anomic/data/URLAnalysis.java | 2 +- .../yacy/dht/FlatWordPartitionScheme.java | 2 +- source/de/anomic/yacy/yacyClient.java | 7 +++- source/net/yacy/kelondro/blob/HeapReader.java | 2 +- source/net/yacy/kelondro/blob/HeapWriter.java | 2 +- source/net/yacy/kelondro/index/Cache.java | 4 +- source/net/yacy/kelondro/index/HandleMap.java | 15 ++++--- source/net/yacy/kelondro/index/HandleSet.java | 11 +++-- source/net/yacy/kelondro/index/IndexTest.java | 7 +++- .../yacy/kelondro/index/ObjectArrayCache.java | 23 +++++++---- .../yacy/kelondro/index/ObjectIndexCache.java | 41 ++++++++++++------- .../yacy/kelondro/index/RowCollection.java | 12 ++++-- source/net/yacy/kelondro/index/RowSet.java | 19 +++++---- .../net/yacy/kelondro/index/RowSetArray.java | 8 ++-- .../kelondro/rwi/AbstractBufferedIndex.java | 2 +- .../net/yacy/kelondro/rwi/AbstractIndex.java | 2 +- source/net/yacy/kelondro/rwi/IndexCell.java | 4 +- .../yacy/kelondro/rwi/ReferenceContainer.java | 19 +++++++-- .../kelondro/rwi/ReferenceContainerArray.java | 2 +- .../kelondro/rwi/ReferenceContainerCache.java | 25 ++++++----- .../net/yacy/kelondro/table/SplitTable.java | 2 +- source/net/yacy/kelondro/table/Table.java | 8 ++-- 23 files changed, 142 insertions(+), 79 deletions(-) diff --git a/source/de/anomic/crawler/ZURL.java b/source/de/anomic/crawler/ZURL.java index 94b51767f..60e060e55 100755 --- a/source/de/anomic/crawler/ZURL.java +++ b/source/de/anomic/crawler/ZURL.java @@ -95,7 +95,7 @@ public class ZURL implements Iterable { public ZURL() { // creates a new ZUR in RAM - this.urlIndex = new RowSet(rowdef, 0); + this.urlIndex = new RowSet(rowdef); this.stack = new ConcurrentLinkedQueue(); } diff --git a/source/de/anomic/data/URLAnalysis.java b/source/de/anomic/data/URLAnalysis.java index 74dea3ce2..8143da41f 100644 --- a/source/de/anomic/data/URLAnalysis.java +++ b/source/de/anomic/data/URLAnalysis.java @@ -416,7 +416,7 @@ public class URLAnalysis { System.out.println("INDEX DIFF URL-COL startup"); HandleMap idx = new HandleMap(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 4, new File(statisticFile), 0); MetadataRepository mr = new MetadataRepository(new File(metadataPath), "text.urlmd", false, false); - HandleSet hs = new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0, 1000000); + HandleSet hs = new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 1000000, 0); System.out.println("INDEX DIFF URL-COL loaded dump, starting diff"); long start = System.currentTimeMillis(); long update = start - 7000; diff --git a/source/de/anomic/yacy/dht/FlatWordPartitionScheme.java b/source/de/anomic/yacy/dht/FlatWordPartitionScheme.java index 9552801ef..0d76ee975 100755 --- a/source/de/anomic/yacy/dht/FlatWordPartitionScheme.java +++ b/source/de/anomic/yacy/dht/FlatWordPartitionScheme.java @@ -111,7 +111,7 @@ public class FlatWordPartitionScheme implements PartitionScheme { System.gc(); // for resource measurement long a = MemoryControl.available(); - HandleMap idx = new HandleMap(12, Base64Order.enhancedCoder, 4, 0, 150000); + HandleMap idx = new HandleMap(12, Base64Order.enhancedCoder, 4, 150000); for (int i = 0; i < count; i++) { try { idx.inc(FlatWordPartitionScheme.positionToHash(r.nextInt(count))); diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index 5c4fdc9c2..67d4d26b2 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -538,7 +538,12 @@ public final class yacyClient { assert words > 0 : "wordhashes = " + wordhashes; final ReferenceContainer[] container = new ReferenceContainer[words]; for (int i = 0; i < words; i++) { - container[i] = ReferenceContainer.emptyContainer(Segment.wordReferenceFactory, wordhashes.substring(i * Word.commonHashLength, (i + 1) * Word.commonHashLength).getBytes(), count); + try { + container[i] = ReferenceContainer.emptyContainer(Segment.wordReferenceFactory, wordhashes.substring(i * Word.commonHashLength, (i + 1) * Word.commonHashLength).getBytes(), count); + } catch (RowSpaceExceededException e) { + Log.logException(e); + return null; + } } // insert results to containers diff --git a/source/net/yacy/kelondro/blob/HeapReader.java b/source/net/yacy/kelondro/blob/HeapReader.java index 2dc465098..bbd3bb34c 100644 --- a/source/net/yacy/kelondro/blob/HeapReader.java +++ b/source/net/yacy/kelondro/blob/HeapReader.java @@ -166,7 +166,7 @@ public class HeapReader { Log.logInfo("HeapReader", "generating index for " + heapFile.toString() + ", " + (file.length() / 1024 / 1024) + " MB. Please wait."); this.free = new Gap(); - HandleMap.initDataConsumer indexready = HandleMap.asynchronusInitializer(keylength, this.ordering, 8, 0, Math.max(10, (int) (Runtime.getRuntime().freeMemory() / (10 * 1024 * 1024)))); + HandleMap.initDataConsumer indexready = HandleMap.asynchronusInitializer(keylength, this.ordering, 8, Math.max(10, (int) (Runtime.getRuntime().freeMemory() / (10 * 1024 * 1024)))); byte[] key = new byte[keylength]; int reclen; long seek = 0; diff --git a/source/net/yacy/kelondro/blob/HeapWriter.java b/source/net/yacy/kelondro/blob/HeapWriter.java index 9d5964a47..8bdd8bc89 100644 --- a/source/net/yacy/kelondro/blob/HeapWriter.java +++ b/source/net/yacy/kelondro/blob/HeapWriter.java @@ -78,7 +78,7 @@ public final class HeapWriter { this.heapFileTMP = temporaryHeapFile; this.heapFileREADY = readyHeapFile; this.keylength = keylength; - this.index = new HandleMap(keylength, ordering, 8, 10, 100000); + this.index = new HandleMap(keylength, ordering, 8, 100000); this.os = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(temporaryHeapFile), outBuffer)); //this.doublecheck = new HashSet(); this.seek = 0; diff --git a/source/net/yacy/kelondro/index/Cache.java b/source/net/yacy/kelondro/index/Cache.java index 483400eeb..af8fc413b 100644 --- a/source/net/yacy/kelondro/index/Cache.java +++ b/source/net/yacy/kelondro/index/Cache.java @@ -82,8 +82,8 @@ public final class Cache implements ObjectIndex, Iterable { private void init() { this.keyrow = new Row(new Column[]{index.row().column(0)}, index.row().objectOrder); - this.readHitCache = new RowSet(index.row(), 0); - this.readMissCache = new RowSet(this.keyrow, 0); + this.readHitCache = new RowSet(index.row()); + this.readMissCache = new RowSet(this.keyrow); this.readHit = 0; this.readMiss = 0; this.writeUnique = 0; diff --git a/source/net/yacy/kelondro/index/HandleMap.java b/source/net/yacy/kelondro/index/HandleMap.java index d9e256fe9..7d129fe5c 100644 --- a/source/net/yacy/kelondro/index/HandleMap.java +++ b/source/net/yacy/kelondro/index/HandleMap.java @@ -62,9 +62,14 @@ public final class HandleMap implements Iterable { * @param objectOrder * @param space */ - public HandleMap(final int keylength, final ByteOrder objectOrder, int idxbytes, final int initialspace, final int expectedspace) { + public HandleMap(final int keylength, final ByteOrder objectOrder, int idxbytes, final int expectedspace) { this.rowdef = new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, keylength, "key"), new Column("long c-" + idxbytes + " {b256}")}, objectOrder); - this.index = new ObjectIndexCache(rowdef, initialspace, expectedspace); + this.index = new ObjectIndexCache(rowdef, expectedspace); + } + + public HandleMap(final int keylength, final ByteOrder objectOrder, int idxbytes, final int expectedspace, final int initialspace) throws RowSpaceExceededException { + this.rowdef = new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, keylength, "key"), new Column("long c-" + idxbytes + " {b256}")}, objectOrder); + this.index = new ObjectIndexCache(rowdef, expectedspace, initialspace); } /** @@ -76,7 +81,7 @@ public final class HandleMap implements Iterable { * @throws RowSpaceExceededException */ public HandleMap(final int keylength, final ByteOrder objectOrder, int idxbytes, final File file, final int expectedspace) throws IOException, RowSpaceExceededException { - this(keylength, objectOrder, idxbytes, (int) (file.length() / (keylength + idxbytes)), expectedspace); + this(keylength, objectOrder, idxbytes, expectedspace, (int) (file.length() / (keylength + idxbytes))); // read the index dump and fill the index InputStream is = new BufferedInputStream(new FileInputStream(file), 1024 * 1024); if (file.getName().endsWith(".gz")) is = new GZIPInputStream(is); @@ -290,8 +295,8 @@ public final class HandleMap implements Iterable { * @param bufferSize * @return */ - public final static initDataConsumer asynchronusInitializer(final int keylength, final ByteOrder objectOrder, int idxbytes, final int space, final int expectedspace) { - initDataConsumer initializer = new initDataConsumer(new HandleMap(keylength, objectOrder, idxbytes, space, expectedspace)); + public final static initDataConsumer asynchronusInitializer(final int keylength, final ByteOrder objectOrder, int idxbytes, final int expectedspace) { + initDataConsumer initializer = new initDataConsumer(new HandleMap(keylength, objectOrder, idxbytes, expectedspace)); ExecutorService service = Executors.newSingleThreadExecutor(); initializer.setResult(service.submit(initializer)); service.shutdown(); diff --git a/source/net/yacy/kelondro/index/HandleSet.java b/source/net/yacy/kelondro/index/HandleSet.java index 35afa69da..51ade462a 100644 --- a/source/net/yacy/kelondro/index/HandleSet.java +++ b/source/net/yacy/kelondro/index/HandleSet.java @@ -44,9 +44,14 @@ public final class HandleSet implements Iterable { private final Row rowdef; private ObjectIndex index; - public HandleSet(final int keylength, final ByteOrder objectOrder, final int initialspace, final int expectedspace) { + public HandleSet(final int keylength, final ByteOrder objectOrder, final int expectedspace) { this.rowdef = new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, keylength, "key")}, objectOrder); - this.index = new ObjectIndexCache(rowdef, initialspace, expectedspace); + this.index = new ObjectIndexCache(rowdef, expectedspace); + } + + public HandleSet(final int keylength, final ByteOrder objectOrder, final int expectedspace, final int initialspace) throws RowSpaceExceededException { + this.rowdef = new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, keylength, "key")}, objectOrder); + this.index = new ObjectIndexCache(rowdef, expectedspace, initialspace); } /** @@ -58,7 +63,7 @@ public final class HandleSet implements Iterable { * @throws RowSpaceExceededException */ public HandleSet(final int keylength, final ByteOrder objectOrder, final File file, final int expectedspace) throws IOException, RowSpaceExceededException { - this(keylength, objectOrder, (int) (file.length() / (keylength + 8)), expectedspace); + this(keylength, objectOrder, expectedspace, (int) (file.length() / (keylength + 8))); // read the index dump and fill the index InputStream is = new BufferedInputStream(new FileInputStream(file), 1024 * 1024); byte[] a = new byte[keylength]; diff --git a/source/net/yacy/kelondro/index/IndexTest.java b/source/net/yacy/kelondro/index/IndexTest.java index e0c65aba7..b96a08125 100644 --- a/source/net/yacy/kelondro/index/IndexTest.java +++ b/source/net/yacy/kelondro/index/IndexTest.java @@ -112,7 +112,12 @@ public class IndexTest { System.out.println("sorted map"); Runtime.getRuntime().gc(); long freeStartKelondro = MemoryControl.available(); - HandleMap ii = new HandleMap(12, Base64Order.enhancedCoder, 4, count, count); + HandleMap ii = null; + try { + ii = new HandleMap(12, Base64Order.enhancedCoder, 4, count, count); + } catch (RowSpaceExceededException e1) { + e1.printStackTrace(); + } for (int i = 0; i < count; i++) try { ii.putUnique(tests[i], 1); diff --git a/source/net/yacy/kelondro/index/ObjectArrayCache.java b/source/net/yacy/kelondro/index/ObjectArrayCache.java index b65f38c75..637e2f15f 100644 --- a/source/net/yacy/kelondro/index/ObjectArrayCache.java +++ b/source/net/yacy/kelondro/index/ObjectArrayCache.java @@ -46,11 +46,16 @@ public final class ObjectArrayCache { private RowSet index1; //private final kelondroOrder entryOrder; - public ObjectArrayCache(final int payloadSize, final int initSize) { - this.rowdef = new Row("Cardinal key-4 {b256}, byte[] payload-" + payloadSize, NaturalOrder.naturalOrder); - this.index0 = new RowSet(rowdef, initSize); - this.index1 = null; - //this.entryOrder = new kelondroRow.EntryComparator(rowdef.objectOrder); + public ObjectArrayCache(final int payloadSize) { + this.rowdef = new Row("Cardinal key-4 {b256}, byte[] payload-" + payloadSize, NaturalOrder.naturalOrder); + this.index0 = new RowSet(rowdef); + this.index1 = null; + } + + public ObjectArrayCache(final int payloadSize, final int initSize) throws RowSpaceExceededException { + this.rowdef = new Row("Cardinal key-4 {b256}, byte[] payload-" + payloadSize, NaturalOrder.naturalOrder); + this.index0 = new RowSet(rowdef, initSize); + this.index1 = null; } public final long memoryNeededForGrow() { @@ -71,7 +76,7 @@ public final class ObjectArrayCache { // finish initialization phase index0.sort(); index0.uniq(); - index1 = new RowSet(rowdef, 0); + index1 = new RowSet(rowdef); } final Row.Entry indexentry = index0.get(key); if (indexentry != null) return indexentry.getColBytes(1); @@ -133,7 +138,7 @@ public final class ObjectArrayCache { // finish initialization phase index0.sort(); index0.uniq(); - index1 = new RowSet(rowdef, 0); + index1 = new RowSet(rowdef); } final Row.Entry indexentry = index0.remove(key); if (indexentry != null) { @@ -187,7 +192,7 @@ public final class ObjectArrayCache { // finish initialization phase index0.sort(); index0.uniq(); - index1 = new RowSet(rowdef, 0); + index1 = new RowSet(rowdef); } return index0.rows(); } else { @@ -219,7 +224,7 @@ public final class ObjectArrayCache { int p, rc = 0; final ArrayList ra = new ArrayList(); final HashSet jcontrol = new HashSet(); - final ObjectArrayCache kcontrol = new ObjectArrayCache(1, 0); + final ObjectArrayCache kcontrol = new ObjectArrayCache(1); for (int i = 0; i < 1000000; i++) { r = Math.abs(random.nextLong() % 10000); //System.out.println("add " + r); diff --git a/source/net/yacy/kelondro/index/ObjectIndexCache.java b/source/net/yacy/kelondro/index/ObjectIndexCache.java index 578919488..4a882a385 100644 --- a/source/net/yacy/kelondro/index/ObjectIndexCache.java +++ b/source/net/yacy/kelondro/index/ObjectIndexCache.java @@ -42,22 +42,35 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable private final Row.EntryComparator entryComparator; private final int spread; - public ObjectIndexCache(final Row rowdef, final int initialspace, final int expectedspace) { - this.rowdef = rowdef; - this.entryComparator = new Row.EntryComparator(rowdef.objectOrder); - this.spread = Math.max(10, expectedspace / 3000); - reset(initialspace); + public ObjectIndexCache(final Row rowdef, final int expectedspace) { + this.rowdef = rowdef; + this.entryComparator = new Row.EntryComparator(rowdef.objectOrder); + this.spread = Math.max(10, expectedspace / 3000); + reset(); + } + + public ObjectIndexCache(final Row rowdef, final int expectedspace, final int initialspace) throws RowSpaceExceededException { + this.rowdef = rowdef; + this.entryComparator = new Row.EntryComparator(rowdef.objectOrder); + this.spread = Math.max(10, expectedspace / 3000); + reset(initialspace); } public void clear() { - reset(0); + reset(); } - public final synchronized void reset(final int initialspace) { - this.index0 = null; // first flush RAM to make room - this.index0 = new RowSet(rowdef, initialspace); + public final synchronized void reset() { + this.index0 = null; // first flush RAM to make room + this.index0 = new RowSet(rowdef); this.index1 = null; // to show that this is the initialization phase - } + } + + public final synchronized void reset(final int initialspace) throws RowSpaceExceededException { + this.index0 = null; // first flush RAM to make room + this.index0 = new RowSet(rowdef, initialspace); + this.index1 = null; // to show that this is the initialization phase + } public final Row row() { return index0.row(); @@ -69,7 +82,7 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable index0.sort(); index0.uniq(); index0.trim(false); - index1 = new RowSetArray(rowdef, 0, spread); + index1 = new RowSetArray(rowdef, spread); } } @@ -230,7 +243,7 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable // finish initialization phase index0.sort(); index0.uniq(); - index1 = new RowSetArray(rowdef, 0, spread); + index1 = new RowSetArray(rowdef, spread); return index0.keys(up, firstKey); } assert (index1 != null); @@ -259,7 +272,7 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable // finish initialization phase index0.sort(); index0.uniq(); - index1 = new RowSetArray(rowdef, 0, spread); + index1 = new RowSetArray(rowdef, spread); return index0.rows(up, firstKey); } assert (index1 != null); @@ -293,7 +306,7 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable // finish initialization phase index0.sort(); index0.uniq(); - index1 = new RowSetArray(rowdef, 0, spread); + index1 = new RowSetArray(rowdef, spread); return index0.rows(); } assert (index1 != null); diff --git a/source/net/yacy/kelondro/index/RowCollection.java b/source/net/yacy/kelondro/index/RowCollection.java index bc5faeb8a..18767d4ce 100644 --- a/source/net/yacy/kelondro/index/RowCollection.java +++ b/source/net/yacy/kelondro/index/RowCollection.java @@ -79,12 +79,17 @@ public class RowCollection implements Iterable { this.lastTimeWrote = rc.lastTimeWrote; } - public RowCollection(final Row rowdef, final int objectCount) { + public RowCollection(final Row rowdef) { this.rowdef = rowdef; - this.chunkcache = new byte[objectCount * rowdef.objectsize]; - this.chunkcount = 0; this.sortBound = 0; this.lastTimeWrote = System.currentTimeMillis(); + this.chunkcache = new byte[0]; + this.chunkcount = 0; + } + + public RowCollection(final Row rowdef, final int objectCount) throws RowSpaceExceededException { + this(rowdef); + ensureSize(objectCount); } public RowCollection(final Row rowdef, final int objectCount, final byte[] cache, final int sortBound) { @@ -218,6 +223,7 @@ public class RowCollection implements Iterable { } protected final void ensureSize(final int elements) throws RowSpaceExceededException { + if (elements == 0) return; long allocram = neededSpaceForEnsuredSize(elements, true); if (allocram == 0) return; assert allocram > chunkcache.length : "wrong alloc computation: allocram = " + allocram + ", chunkcache.length = " + chunkcache.length; diff --git a/source/net/yacy/kelondro/index/RowSet.java b/source/net/yacy/kelondro/index/RowSet.java index b18b9feab..0a81fff93 100644 --- a/source/net/yacy/kelondro/index/RowSet.java +++ b/source/net/yacy/kelondro/index/RowSet.java @@ -46,11 +46,16 @@ public class RowSet extends RowCollection implements ObjectIndex, Iterable= exportOverheadSize : "b.length = " + b.length; - if (b.length < exportOverheadSize) return new RowSet(rowdef, 0); + if (b.length < exportOverheadSize) return new RowSet(rowdef); final int size = (int) NaturalOrder.decodeLong(b, 0, 4); assert size >= 0 : "size = " + size; - if (size < 0) return new RowSet(rowdef, 0); + if (size < 0) return new RowSet(rowdef); final int orderbound = (int) NaturalOrder.decodeLong(b, 10, 4); assert orderbound >= 0 : "orderbound = " + orderbound; - if (orderbound < 0) return new RowSet(rowdef, 0); + if (orderbound < 0) return new RowSet(rowdef); final byte[] chunkcache = new byte[size * rowdef.objectsize]; assert b.length - exportOverheadSize == size * rowdef.objectsize; - if (b.length - exportOverheadSize != size * rowdef.objectsize) return new RowSet(rowdef, 0); + if (b.length - exportOverheadSize != size * rowdef.objectsize) return new RowSet(rowdef); System.arraycopy(b, exportOverheadSize, chunkcache, 0, chunkcache.length); return new RowSet(rowdef, size, chunkcache, orderbound); } @@ -486,7 +491,7 @@ public class RowSet extends RowCollection implements ObjectIndex, Iterable { - private final int objectCount; private final Row rowdef; private final RowSet[] array; - public RowSetArray(final Row rowdef, final int objectCount, final int arraySize) { + public RowSetArray(final Row rowdef, final int arraySize) { this.array = new RowSet[arraySize]; + this.rowdef = rowdef; for (int i = 0; i < arraySize; i++) { this.array[i] = null; } - this.rowdef = rowdef; - this.objectCount = objectCount / arraySize; } private final int indexFor(byte[] key) { @@ -58,7 +56,7 @@ public final class RowSetArray implements ObjectIndex, Iterable { private final RowSet accessArray(int i) { RowSet r = this.array[i]; if (r == null) synchronized (this.array) { - r = new RowSet(this.rowdef, this.objectCount); + r = new RowSet(this.rowdef); this.array[i] = r; } return r; diff --git a/source/net/yacy/kelondro/rwi/AbstractBufferedIndex.java b/source/net/yacy/kelondro/rwi/AbstractBufferedIndex.java index b5ee47a62..f249f8b74 100644 --- a/source/net/yacy/kelondro/rwi/AbstractBufferedIndex.java +++ b/source/net/yacy/kelondro/rwi/AbstractBufferedIndex.java @@ -46,7 +46,7 @@ public abstract class AbstractBufferedIndex ext // this does not use the cache final Order> containerOrder = new ReferenceContainerOrder(factory, this.ordering().clone()); if (startHash.length == 0) startHash = null; - ReferenceContainer emptyContainer = ReferenceContainer.emptyContainer(factory, startHash, 0); + ReferenceContainer emptyContainer = ReferenceContainer.emptyContainer(factory, startHash); containerOrder.rotate(emptyContainer); final TreeSet> containers = new TreeSet>(containerOrder); final Iterator> i = references(startHash, rot, ram); diff --git a/source/net/yacy/kelondro/rwi/AbstractIndex.java b/source/net/yacy/kelondro/rwi/AbstractIndex.java index f9baa27c7..903b2c5a3 100644 --- a/source/net/yacy/kelondro/rwi/AbstractIndex.java +++ b/source/net/yacy/kelondro/rwi/AbstractIndex.java @@ -60,7 +60,7 @@ public abstract class AbstractIndex implements // creates a set of indexContainers // this does not use the cache final Order> containerOrder = new ReferenceContainerOrder(factory, this.ordering().clone()); - ReferenceContainer emptyContainer = ReferenceContainer.emptyContainer(factory, startHash, 0); + ReferenceContainer emptyContainer = ReferenceContainer.emptyContainer(factory, startHash); containerOrder.rotate(emptyContainer); final TreeSet> containers = new TreeSet>(containerOrder); final Iterator> i = references(startHash, rot); diff --git a/source/net/yacy/kelondro/rwi/IndexCell.java b/source/net/yacy/kelondro/rwi/IndexCell.java index b7385de26..8a1d4d039 100644 --- a/source/net/yacy/kelondro/rwi/IndexCell.java +++ b/source/net/yacy/kelondro/rwi/IndexCell.java @@ -295,7 +295,7 @@ public final class IndexCell extends AbstractBu public CloneableIterator> references(byte[] starttermHash, boolean rot) { final Order> containerOrder = new ReferenceContainerOrder(factory, this.ram.rowdef().getOrdering().clone()); - containerOrder.rotate(new ReferenceContainer(factory, starttermHash, 0)); + containerOrder.rotate(new ReferenceContainer(factory, starttermHash)); return new MergeIterator>( this.ram.references(starttermHash, rot), new MergeIterator>( @@ -311,7 +311,7 @@ public final class IndexCell extends AbstractBu public CloneableIterator> references(byte[] startTermHash, boolean rot, boolean ram) { final Order> containerOrder = new ReferenceContainerOrder(factory, this.ram.rowdef().getOrdering().clone()); - containerOrder.rotate(new ReferenceContainer(factory, startTermHash, 0)); + containerOrder.rotate(new ReferenceContainer(factory, startTermHash)); if (ram) { return this.ram.references(startTermHash, rot); } diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainer.java b/source/net/yacy/kelondro/rwi/ReferenceContainer.java index 0ce7da89d..464a94c1b 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainer.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainer.java @@ -62,7 +62,15 @@ public class ReferenceContainer extends RowSet this.termHash = termHash; } - public ReferenceContainer(final ReferenceFactory factory, final byte[] termHash, final int objectCount) { + public ReferenceContainer(final ReferenceFactory factory, final byte[] termHash) { + super(factory.getRow()); + assert termHash == null || (termHash[2] != '@' && termHash.length == this.rowdef.primaryKeyLength); + this.termHash = termHash; + this.factory = factory; + this.lastTimeWrote = 0; + } + + public ReferenceContainer(final ReferenceFactory factory, final byte[] termHash, final int objectCount) throws RowSpaceExceededException { super(factory.getRow(), objectCount); assert termHash == null || (termHash[2] != '@' && termHash.length == this.rowdef.primaryKeyLength); this.termHash = termHash; @@ -76,8 +84,13 @@ public class ReferenceContainer extends RowSet return newContainer; } - public static ReferenceContainer emptyContainer(final ReferenceFactory factory, final byte[] termHash, final int elementCount) { - assert termHash == null || (termHash[2] != '@' && termHash.length == factory.getRow().primaryKeyLength); + public static ReferenceContainer emptyContainer(final ReferenceFactory factory, final byte[] termHash) { + assert termHash == null || (termHash[2] != '@' && termHash.length == factory.getRow().primaryKeyLength); + return new ReferenceContainer(factory, termHash); + } + + public static ReferenceContainer emptyContainer(final ReferenceFactory factory, final byte[] termHash, final int elementCount) throws RowSpaceExceededException { + assert termHash == null || (termHash[2] != '@' && termHash.length == factory.getRow().primaryKeyLength); return new ReferenceContainer(factory, termHash, elementCount); } diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java index 45b3a0292..e3f1c4c67 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java @@ -329,7 +329,7 @@ public final class ReferenceContainerArray { final Row payloadrow) throws IOException, RowSpaceExceededException { System.out.println("CELL REFERENCE COLLECTION startup"); - HandleMap references = new HandleMap(payloadrow.primaryKeyLength, termOrder, 4, 0, 1000000); + HandleMap references = new HandleMap(payloadrow.primaryKeyLength, termOrder, 4, 1000000, 0); String[] files = heapLocation.list(); for (String f: files) { if (f.length() < 22 || !f.startsWith("text.index") || !f.endsWith(".blob")) continue; diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java b/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java index a64a653e1..2048388f4 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java @@ -282,25 +282,28 @@ public final class ReferenceContainerCache exte * get a indexContainer from a heap * @param key * @return the indexContainer if one exist, null otherwise + * @throws */ public ReferenceContainer get(final byte[] key, Set urlselection) { ReferenceContainer c = this.cache.get(new ByteArray(key)); if (urlselection == null) return c; if (c == null) return null; // because this is all in RAM, we must clone the entries (flat) - ReferenceContainer c1 = new ReferenceContainer(factory, c.getTermHash(), c.size()); - Iterator e = c.entries(); - ReferenceType ee; - while (e.hasNext()) { - ee = e.next(); - if (urlselection.contains(ee.metadataHash())) try { - c1.add(ee); - } catch (RowSpaceExceededException e1) { - Log.logException(e1); - break; + try { + ReferenceContainer c1 = new ReferenceContainer(factory, c.getTermHash(), c.size()); + Iterator e = c.entries(); + ReferenceType ee; + while (e.hasNext()) { + ee = e.next(); + if (urlselection.contains(ee.metadataHash())) { + c1.add(ee); + } } + return c1; + } catch (RowSpaceExceededException e2) { + Log.logException(e2); } - return c1; + return null; } /** diff --git a/source/net/yacy/kelondro/table/SplitTable.java b/source/net/yacy/kelondro/table/SplitTable.java index ee3ce883e..d56cb8724 100644 --- a/source/net/yacy/kelondro/table/SplitTable.java +++ b/source/net/yacy/kelondro/table/SplitTable.java @@ -68,7 +68,7 @@ public class SplitTable implements ObjectIndex, Iterable { // the table type can be either kelondroFlex or kelondroEco private static final int EcoFSBufferSize = 20; - static final ObjectIndex dummyIndex = new ObjectIndexCache(new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, 2, "key")}, NaturalOrder.naturalOrder), 0, 0); + static final ObjectIndex dummyIndex = new ObjectIndexCache(new Row(new Column[]{new Column("key", Column.celltype_binary, Column.encoder_bytes, 2, "key")}, NaturalOrder.naturalOrder), 0); // the thread pool for the keeperOf executor service private ExecutorService executor; diff --git a/source/net/yacy/kelondro/table/Table.java b/source/net/yacy/kelondro/table/Table.java index 86085aaa0..d96b4218d 100644 --- a/source/net/yacy/kelondro/table/Table.java +++ b/source/net/yacy/kelondro/table/Table.java @@ -135,8 +135,8 @@ public class Table implements ObjectIndex, Iterable { table = null; System.gc(); Log.logSevere("TABLE", tablefile.getName() + ": RAM after releasing the table: " + (MemoryControl.available() / 1024 / 1024) + "MB"); } - index = new HandleMap(rowdef.primaryKeyLength, rowdef.objectOrder, 4, records, 100000); - HandleMap errors = new HandleMap(rowdef.primaryKeyLength, NaturalOrder.naturalOrder, 4, records, 10); + index = new HandleMap(rowdef.primaryKeyLength, rowdef.objectOrder, 4, records); + HandleMap errors = new HandleMap(rowdef.primaryKeyLength, NaturalOrder.naturalOrder, 4, records); Log.logInfo("TABLE", tablefile + ": TABLE " + tablefile.toString() + " has table copy " + ((table == null) ? "DISABLED" : "ENABLED")); // read all elements from the file into the copy table @@ -695,8 +695,8 @@ public class Table implements ObjectIndex, Iterable { } // initialize index and copy table - table = (table == null) ? null : new RowSet(taildef, 1); - index = new HandleMap(rowdef.primaryKeyLength, rowdef.objectOrder, 4, 1, 100000); + table = (table == null) ? null : new RowSet(taildef); + index = new HandleMap(rowdef.primaryKeyLength, rowdef.objectOrder, 4, 100000); } public Row row() {