diff --git a/source/net/yacy/kelondro/index/RowSet.java b/source/net/yacy/kelondro/index/RowSet.java index 2a6d2fa31..c89184056 100644 --- a/source/net/yacy/kelondro/index/RowSet.java +++ b/source/net/yacy/kelondro/index/RowSet.java @@ -76,29 +76,29 @@ public class RowSet extends RowCollection implements Index, Iterable public final static RowSet importRowSet(final byte[] b, final Row rowdef) throws RowSpaceExceededException { assert b.length >= exportOverheadSize : "b.length = " + b.length; - if (b.length < exportOverheadSize) return new RowSet(rowdef); + if (b.length < exportOverheadSize) return new RowSet(rowdef, 0); final int size = (int) NaturalOrder.decodeLong(b, 0, 4); assert size >= 0 : "size = " + size; - if (size < 0) return new RowSet(rowdef); + if (size < 0) return new RowSet(rowdef, 0); final int orderbound = (int) NaturalOrder.decodeLong(b, 10, 4); assert orderbound >= 0 : "orderbound = " + orderbound; - if (orderbound < 0) return new RowSet(rowdef); // error + if (orderbound < 0) return new RowSet(rowdef, 0); // error final long alloc = ((long) size) * ((long) rowdef.objectsize); assert alloc <= Integer.MAX_VALUE : "alloc = " + alloc; - if (alloc > Integer.MAX_VALUE) return null; + if (alloc > Integer.MAX_VALUE) throw new RowSpaceExceededException((int) alloc, "importRowSet: alloc > Integer.MAX_VALUE"); assert alloc == b.length - exportOverheadSize; - if (alloc != b.length - exportOverheadSize) return null; + if (alloc != b.length - exportOverheadSize) throw new RowSpaceExceededException((int) alloc, "importRowSet: alloc != b.length - exportOverheadSize"); MemoryControl.request((int) alloc, true); final byte[] chunkcache; try { chunkcache = new byte[(int) alloc]; } catch (final OutOfMemoryError e) { - throw new RowSpaceExceededException((int) alloc, "importRowSet"); + throw new RowSpaceExceededException((int) alloc, "importRowSet: OutOfMemoryError"); } //assert b.length - exportOverheadSize == size * rowdef.objectsize : "b.length = " + b.length + ", size * rowdef.objectsize = " + size * rowdef.objectsize; if (b.length - exportOverheadSize != alloc) { Log.logSevere("RowSet", "exportOverheadSize wrong: b.length = " + b.length + ", size * rowdef.objectsize = " + size * rowdef.objectsize); - return new RowSet(rowdef); + return new RowSet(rowdef, 0); } System.arraycopy(b, (int) exportOverheadSize, chunkcache, 0, chunkcache.length); return new RowSet(rowdef, size, chunkcache, orderbound); diff --git a/source/net/yacy/kelondro/rwi/ReferenceIterator.java b/source/net/yacy/kelondro/rwi/ReferenceIterator.java index 94da81a82..6089cc9bd 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceIterator.java +++ b/source/net/yacy/kelondro/rwi/ReferenceIterator.java @@ -9,7 +9,7 @@ // $LastChangedBy$ // // LICENSE -// +// // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or @@ -28,7 +28,6 @@ package net.yacy.kelondro.rwi; import java.io.File; import java.io.IOException; -import java.util.Iterator; import java.util.Map; import net.yacy.kelondro.blob.HeapReader; @@ -36,62 +35,59 @@ import net.yacy.kelondro.index.RowSet; import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.CloneableIterator; +import net.yacy.kelondro.util.LookAheadIterator; /** * iterator of BLOBHeap files: is used to import heap dumps into a write-enabled index heap */ -public class ReferenceIterator implements CloneableIterator>, Iterable> { +public class ReferenceIterator extends LookAheadIterator> implements CloneableIterator>, Iterable> { HeapReader.entries blobs; File blobFile; ReferenceFactory factory; - - public ReferenceIterator(final File blobFile, ReferenceFactory factory) throws IOException { + + public ReferenceIterator(final File blobFile, final ReferenceFactory factory) throws IOException { this.blobs = new HeapReader.entries(blobFile, factory.getRow().primaryKeyLength); this.blobFile = blobFile; this.factory = factory; } - - public boolean hasNext() { - if (blobs == null) return false; - if (blobs.hasNext()) return true; - close(); - return false; - } /** * return an index container * because they may get very large, it is wise to deallocate some memory before calling next() */ - public ReferenceContainer next() { - Map.Entry entry = blobs.next(); - byte[] payload = entry.getValue(); - try { - return new ReferenceContainer(factory, entry.getKey(), RowSet.importRowSet(payload, factory.getRow())); - } catch (RowSpaceExceededException e) { - Log.logSevere("ReferenceIterator", "lost entry '" + entry.getKey() + "' because of too low memory: " + e.toString()); - return null; + public ReferenceContainer next0() { + if (this.blobs == null) return null; + RowSet row; + Map.Entry entry; + while (this.blobs.hasNext()) { + entry = this.blobs.next(); + try { + row = RowSet.importRowSet(entry.getValue(), this.factory.getRow()); + if (row == null) { + Log.logSevere("ReferenceIterator", "lost entry '" + entry.getKey() + "' because importRowSet returned null"); + continue; // thats a fail but not as REALLY bad if the whole method would crash here + } + return new ReferenceContainer(this.factory, entry.getKey(), row); + } catch (final RowSpaceExceededException e) { + Log.logSevere("ReferenceIterator", "lost entry '" + entry.getKey() + "' because of too low memory: " + e.toString()); + continue; + } } - } - - public void remove() { - throw new UnsupportedOperationException("heap dumps are read-only"); + close(); + return null; } - public Iterator> iterator() { - return this; - } - public void close() { - if (blobs != null) this.blobs.close(); - blobs = null; + if (this.blobs != null) this.blobs.close(); + this.blobs = null; } - public CloneableIterator> clone(Object modifier) { - if (blobs != null) this.blobs.close(); - blobs = null; + public CloneableIterator> clone(final Object modifier) { + if (this.blobs != null) this.blobs.close(); + this.blobs = null; try { - return new ReferenceIterator(this.blobFile, factory); - } catch (IOException e) { + return new ReferenceIterator(this.blobFile, this.factory); + } catch (final IOException e) { Log.logException(e); return null; } diff --git a/source/net/yacy/kelondro/util/LookAheadIterator.java b/source/net/yacy/kelondro/util/LookAheadIterator.java index acae69cb6..e0d376fd1 100644 --- a/source/net/yacy/kelondro/util/LookAheadIterator.java +++ b/source/net/yacy/kelondro/util/LookAheadIterator.java @@ -9,7 +9,7 @@ // $LastChangedBy$ // // LICENSE -// +// // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or @@ -39,44 +39,44 @@ public abstract class LookAheadIterator implements Iterator, Iterable { private boolean fresh = true; private A next = null; - + public LookAheadIterator() { } - public Iterator iterator() { + public final Iterator iterator() { return this; } - + /** * the internal next-method * @return a value of type A if available or null if no more value are available */ protected abstract A next0() ; - + private final void checkInit() { - if (fresh) { - next = next0(); - fresh = false; + if (this.fresh) { + this.next = next0(); + this.fresh = false; } } - + public final boolean hasNext() { checkInit(); - return next != null; + return this.next != null; } public final A next() { checkInit(); - A n = next; - next = next0(); + final A n = this.next; + this.next = next0(); return n; } - + /** * a remove is not possible with this implementation */ public final void remove() { throw new UnsupportedOperationException(); } - + }