From cb4de9ceee5cd09edc99ee21befef472516bfe18 Mon Sep 17 00:00:00 2001 From: orbiter Date: Mon, 12 Oct 2009 08:06:35 +0000 Subject: [PATCH] fixed a bug in table iterator (did not recognize elements in write buffer) git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6401 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/net/yacy/kelondro/index/Cache.java | 11 ++++- .../net/yacy/kelondro/index/ObjectIndex.java | 2 + .../yacy/kelondro/index/ObjectIndexCache.java | 6 ++- .../net/yacy/kelondro/io/BufferedRecords.java | 23 ++++++---- source/net/yacy/kelondro/io/Records.java | 3 +- .../yacy/kelondro/table/ChunkIterator.java | 6 +++ source/net/yacy/kelondro/table/SQLTable.java | 12 ++++- .../net/yacy/kelondro/table/SplitTable.java | 11 ++++- source/net/yacy/kelondro/table/Table.java | 46 +++++++++++-------- 9 files changed, 86 insertions(+), 34 deletions(-) diff --git a/source/net/yacy/kelondro/index/Cache.java b/source/net/yacy/kelondro/index/Cache.java index 21632656c..2e528fae5 100644 --- a/source/net/yacy/kelondro/index/Cache.java +++ b/source/net/yacy/kelondro/index/Cache.java @@ -36,11 +36,12 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; +import net.yacy.kelondro.index.Row.Entry; import net.yacy.kelondro.order.CloneableIterator; import net.yacy.kelondro.util.MemoryControl; -public class Cache implements ObjectIndex { +public class Cache implements ObjectIndex, Iterable { // this is a combined read cache and write buffer // we maintain four tables: @@ -436,6 +437,14 @@ public class Cache implements ObjectIndex { return index.rows(up, firstKey); } + public Iterator iterator() { + try { + return rows(); + } catch (IOException e) { + return null; + } + } + public synchronized CloneableIterator rows() throws IOException { return index.rows(); } diff --git a/source/net/yacy/kelondro/index/ObjectIndex.java b/source/net/yacy/kelondro/index/ObjectIndex.java index 319a9cce4..664874f4c 100644 --- a/source/net/yacy/kelondro/index/ObjectIndex.java +++ b/source/net/yacy/kelondro/index/ObjectIndex.java @@ -33,6 +33,7 @@ package net.yacy.kelondro.index; import java.io.IOException; import java.util.ArrayList; +import java.util.Iterator; import net.yacy.kelondro.order.CloneableIterator; @@ -53,6 +54,7 @@ public interface ObjectIndex { public CloneableIterator keys(boolean up, byte[] firstKey) throws IOException; // iterates only the key public CloneableIterator rows(boolean up, byte[] firstKey) throws IOException; // iterates the whole row using the order of the keys public CloneableIterator rows() throws IOException; // iterates the whole row without any order + public Iterator iterator(); public void deleteOnExit(); public void clear() throws IOException; public void close(); diff --git a/source/net/yacy/kelondro/index/ObjectIndexCache.java b/source/net/yacy/kelondro/index/ObjectIndexCache.java index 35a9e4c0e..a601373f5 100644 --- a/source/net/yacy/kelondro/index/ObjectIndexCache.java +++ b/source/net/yacy/kelondro/index/ObjectIndexCache.java @@ -34,7 +34,7 @@ import net.yacy.kelondro.order.MergeIterator; import net.yacy.kelondro.order.StackIterator; -public class ObjectIndexCache implements ObjectIndex { +public class ObjectIndexCache implements ObjectIndex, Iterable { private final Row rowdef; private RowSet index0; @@ -269,6 +269,10 @@ public class ObjectIndexCache implements ObjectIndex { true); } + public Iterator iterator() { + return rows(); + } + public synchronized CloneableIterator rows() { // returns the row-iterator of the underlying kelondroIndex if (index1 == null) { diff --git a/source/net/yacy/kelondro/io/BufferedRecords.java b/source/net/yacy/kelondro/io/BufferedRecords.java index 35bc3e284..620c148e2 100644 --- a/source/net/yacy/kelondro/io/BufferedRecords.java +++ b/source/net/yacy/kelondro/io/BufferedRecords.java @@ -26,7 +26,6 @@ package net.yacy.kelondro.io; import java.io.File; import java.io.IOException; -import java.util.Iterator; import java.util.Map; import java.util.TreeMap; @@ -52,15 +51,23 @@ public class BufferedRecords { this.buffer = new TreeMap(); } - private void flushBuffer() throws IOException { + /** + * flush the buffer: this shall be called before any file-based iterations + * on data structures on rescords are made + * @throws IOException + */ + public synchronized void flushBuffer() throws IOException { + this.flushBuffer0(); + efs.flushBuffer(); + } + + private void flushBuffer0() throws IOException { if (efs == null) return; - final Iterator> i = buffer.entrySet().iterator(); - Map.Entry entry; - while (i.hasNext()) { - entry = i.next(); + for (Map.Entry entry: buffer.entrySet()) { efs.put(entry.getKey().intValue(), entry.getValue(), 0); } buffer.clear(); + efs.flushBuffer(); } public synchronized long size() throws IOException { @@ -73,7 +80,7 @@ public class BufferedRecords { public synchronized void close() { try { - flushBuffer(); + flushBuffer0(); } catch (final IOException e) { e.printStackTrace(); } @@ -106,7 +113,7 @@ public class BufferedRecords { final byte[] bb = new byte[efs.recordsize]; System.arraycopy(b, start, bb, 0, efs.recordsize); buffer.put(Long.valueOf(index), bb); - if (buffer.size() > this.maxEntries) flushBuffer(); + if (buffer.size() > this.maxEntries) flushBuffer0(); } } diff --git a/source/net/yacy/kelondro/io/Records.java b/source/net/yacy/kelondro/io/Records.java index 5532cd475..33a1bea7b 100644 --- a/source/net/yacy/kelondro/io/Records.java +++ b/source/net/yacy/kelondro/io/Records.java @@ -221,12 +221,11 @@ public class Records { /** * write buffer to end of file */ - private void flushBuffer() { + public void flushBuffer() { try { raf.seek(raf.length()); raf.write(this.buffer, 0, this.recordsize * this.buffercount); } catch (final IOException e) { - // TODO Auto-generated catch block e.printStackTrace(); } this.buffercount = 0; diff --git a/source/net/yacy/kelondro/table/ChunkIterator.java b/source/net/yacy/kelondro/table/ChunkIterator.java index 1aaa1b86b..64342e434 100644 --- a/source/net/yacy/kelondro/table/ChunkIterator.java +++ b/source/net/yacy/kelondro/table/ChunkIterator.java @@ -26,6 +26,7 @@ package net.yacy.kelondro.table; import java.io.BufferedInputStream; import java.io.DataInputStream; +import java.io.EOFException; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -40,6 +41,7 @@ public class ChunkIterator implements Iterator { * create a ChunkIterator * a ChunkIterator uses a BufferedInputStream to iterate through the file * and is therefore a fast option to get all elements in the file as a sequence + * ATTENTION: before calling this class ensure that all file buffers are flushed * @param file: the file * @param recordsize: the size of the elements in the file * @param chunksize: the size of the chunks that are returned by next(). remaining bytes until the lenght of recordsize are skipped @@ -79,7 +81,11 @@ public class ChunkIterator implements Iterator { r += s; } return chunk; + } catch (final EOFException e) { + // no real exception, this is the normal termination + return null; } catch (final IOException e) { + e.printStackTrace(); return null; } } diff --git a/source/net/yacy/kelondro/table/SQLTable.java b/source/net/yacy/kelondro/table/SQLTable.java index 1c693388d..4a34edfd7 100644 --- a/source/net/yacy/kelondro/table/SQLTable.java +++ b/source/net/yacy/kelondro/table/SQLTable.java @@ -34,11 +34,13 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.Date; +import java.util.Iterator; import java.util.List; import net.yacy.kelondro.index.ObjectIndex; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.RowCollection; +import net.yacy.kelondro.index.Row.Entry; import net.yacy.kelondro.order.ByteOrder; import net.yacy.kelondro.order.CloneableIterator; import net.yacy.kelondro.order.NaturalOrder; @@ -55,7 +57,7 @@ import net.yacy.kelondro.order.NaturalOrder; * grant ALL on yacy.* to yacy; */ -public class SQLTable implements ObjectIndex { +public class SQLTable implements ObjectIndex, Iterable { private static final String db_driver_str_mysql = "org.gjt.mm.mysql.Driver"; private static final String db_driver_str_pgsql = "org.postgresql.Driver"; @@ -266,6 +268,14 @@ public class SQLTable implements ObjectIndex { // Objects are of type kelondroRow.Entry return null; } + + public Iterator iterator() { + try { + return rows(); + } catch (IOException e) { + return null; + } + } public CloneableIterator rows() throws IOException { return null; diff --git a/source/net/yacy/kelondro/table/SplitTable.java b/source/net/yacy/kelondro/table/SplitTable.java index 0fc50048a..cb1343f84 100644 --- a/source/net/yacy/kelondro/table/SplitTable.java +++ b/source/net/yacy/kelondro/table/SplitTable.java @@ -50,6 +50,7 @@ import net.yacy.kelondro.index.ObjectIndex; import net.yacy.kelondro.index.ObjectIndexCache; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.RowCollection; +import net.yacy.kelondro.index.Row.Entry; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.CloneableIterator; import net.yacy.kelondro.order.MergeIterator; @@ -61,7 +62,7 @@ import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.NamePrefixThreadFactory; -public class SplitTable implements ObjectIndex { +public class SplitTable implements ObjectIndex, Iterable { // this is a set of kelondro tables // the set is divided into tables with different entry date @@ -594,6 +595,14 @@ public class SplitTable implements ObjectIndex { return MergeIterator.cascade(c, entryOrder, MergeIterator.simpleMerge, up); } + public Iterator iterator() { + try { + return rows(); + } catch (IOException e) { + return null; + } + } + public synchronized CloneableIterator rows() throws IOException { final List> c = new ArrayList>(tables.size()); final Iterator i = tables.values().iterator(); diff --git a/source/net/yacy/kelondro/table/Table.java b/source/net/yacy/kelondro/table/Table.java index 65399de66..ec3123a34 100644 --- a/source/net/yacy/kelondro/table/Table.java +++ b/source/net/yacy/kelondro/table/Table.java @@ -65,7 +65,7 @@ import net.yacy.kelondro.util.kelondroException; * The content cache can also be deleted during run-time, if the available RAM gets too low. */ -public class Table implements ObjectIndex { +public class Table implements ObjectIndex, Iterable { // static tracker objects private static TreeMap tableTracker = new TreeMap(); @@ -628,8 +628,17 @@ public class Table implements ObjectIndex { public synchronized int size() { return index.size(); } + + public Iterator iterator() { + try { + return rows(); + } catch (IOException e) { + return null; + } + } public synchronized CloneableIterator rows() throws IOException { + this.file.flushBuffer(); return new rowIteratorNoOrder(); } @@ -747,10 +756,10 @@ public class Table implements ObjectIndex { return result; } - private static ObjectIndex testTable(final File f, final String testentities, final boolean useTailCache, final boolean exceed134217727) throws IOException { + private static Table testTable(final File f, final String testentities, final boolean useTailCache, final boolean exceed134217727) throws IOException { if (f.exists()) FileUtils.deletedelete(f); final Row rowdef = new Row("byte[] a-4, byte[] b-4", NaturalOrder.naturalOrder); - final ObjectIndex tt = new Table(f, rowdef, 100, 0, useTailCache, exceed134217727); + final Table tt = new Table(f, rowdef, 100, 0, useTailCache, exceed134217727); byte[] b; final Row.Entry row = rowdef.newEntry(); for (int i = 0; i < testentities.length(); i++) { @@ -762,19 +771,13 @@ public class Table implements ObjectIndex { return tt; } - private static int countElements(final ObjectIndex t) { + private static int countElements(final Table t) { int count = 0; - try { - final Iterator iter = t.rows(); - Row.Entry row; - while (iter.hasNext()) { - count++; - row = iter.next(); - if (row == null) System.out.println("ERROR! null element found"); - // else System.out.println("counted element: " + new - // String(n.getKey())); - } - } catch (final IOException e) { + for (Row.Entry row: t) { + count++; + if (row == null) System.out.println("ERROR! null element found"); + // else System.out.println("counted element: " + new + // String(n.getKey())); } return count; } @@ -783,14 +786,16 @@ public class Table implements ObjectIndex { System.out.println("starting big test with " + elements + " elements:"); final long start = System.currentTimeMillis(); final String[] s = permutations(elements); - ObjectIndex tt; + Table tt; + int count; try { for (int i = 0; i < s.length; i++) { System.out.println("*** probing tree " + i + " for permutation " + s[i]); // generate tree and delete elements tt = testTable(testFile, s[i], useTailCache, exceed134217727); - if (countElements(tt) != tt.size()) { - System.out.println("wrong size for " + s[i]); + count = countElements(tt); + if (count != tt.size()) { + System.out.println("wrong size for " + s[i] + ": count = " + count + ", size() = " + tt.size()); } tt.close(); for (int j = 0; j < s.length; j++) { @@ -798,8 +803,9 @@ public class Table implements ObjectIndex { // delete by permutation j for (int elt = 0; elt < s[j].length(); elt++) { tt.remove(testWord(s[j].charAt(elt))); - if (countElements(tt) != tt.size()) { - System.out.println("ERROR! wrong size for probe tree " + s[i] + "; probe delete " + s[j] + "; position " + elt); + count = countElements(tt); + if (count != tt.size()) { + System.out.println("ERROR! wrong size for probe tree " + s[i] + "; probe delete " + s[j] + "; position " + elt + "; count = " + count + ", size() = " + tt.size()); } } tt.close();