boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fixed a bug in table iterator (did not recognize elements in write buffer)

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6401 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent 5841ee83d3
commit cb4de9ceee
9 changed files with 86 additions and 34 deletions
Show Stats Download Patch File Download Diff File

11
source/net/yacy/kelondro/index/Cache.java
Unescape View File

@ -36,11 +36,12 @@ import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import net.yacy.kelondro.index.Row.Entry;
import net.yacy.kelondro.order.CloneableIterator;
import net.yacy.kelondro.util.MemoryControl;
public class Cache implements ObjectIndex {
public class Cache implements ObjectIndex, Iterable<Row.Entry> {
// this is a combined read cache and write buffer
// we maintain four tables:
@ -436,6 +437,14 @@ public class Cache implements ObjectIndex {
return index.rows(up, firstKey);
}
public Iterator<Entry> iterator() {
try {
return rows();
} catch (IOException e) {
return null;
}
}
public synchronized CloneableIterator<Row.Entry> rows() throws IOException {
return index.rows();
}

2
source/net/yacy/kelondro/index/ObjectIndex.java
Unescape View File

@ -33,6 +33,7 @@ package net.yacy.kelondro.index;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import net.yacy.kelondro.order.CloneableIterator;
@ -53,6 +54,7 @@ public interface ObjectIndex {
public CloneableIterator<byte[]> keys(boolean up, byte[] firstKey) throws IOException; // iterates only the key
public CloneableIterator<Row.Entry> rows(boolean up, byte[] firstKey) throws IOException; // iterates the whole row using the order of the keys
public CloneableIterator<Row.Entry> rows() throws IOException; // iterates the whole row without any order
public Iterator<Row.Entry> iterator();
public void deleteOnExit();
public void clear() throws IOException;
public void close();

6
source/net/yacy/kelondro/index/ObjectIndexCache.java
Unescape View File

@ -34,7 +34,7 @@ import net.yacy.kelondro.order.MergeIterator;
import net.yacy.kelondro.order.StackIterator;
public class ObjectIndexCache implements ObjectIndex {
public class ObjectIndexCache implements ObjectIndex, Iterable<Row.Entry> {
private final Row rowdef;
private RowSet index0;
@ -269,6 +269,10 @@ public class ObjectIndexCache implements ObjectIndex {
true);
}
public Iterator<Entry> iterator() {
return rows();
}
public synchronized CloneableIterator<Row.Entry> rows() {
// returns the row-iterator of the underlying kelondroIndex
if (index1 == null) {

23
source/net/yacy/kelondro/io/BufferedRecords.java
Unescape View File

@ -26,7 +26,6 @@ package net.yacy.kelondro.io;
import java.io.File;
import java.io.IOException;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
@ -52,15 +51,23 @@ public class BufferedRecords {
this.buffer = new TreeMap<Long, byte[]>();
}
private void flushBuffer() throws IOException {
/**
* flush the buffer: this shall be called before any file-based iterations
* on data structures on rescords are made
* @throws IOException
*/
public synchronized void flushBuffer() throws IOException {
this.flushBuffer0();
efs.flushBuffer();
}
private void flushBuffer0() throws IOException {
if (efs == null) return;
final Iterator<Map.Entry<Long, byte[]>> i = buffer.entrySet().iterator();
Map.Entry<Long, byte[]> entry;
while (i.hasNext()) {
entry = i.next();
for (Map.Entry<Long, byte[]> entry: buffer.entrySet()) {
efs.put(entry.getKey().intValue(), entry.getValue(), 0);
}
buffer.clear();
efs.flushBuffer();
}
public synchronized long size() throws IOException {
@ -73,7 +80,7 @@ public class BufferedRecords {
public synchronized void close() {
try {
flushBuffer();
flushBuffer0();
} catch (final IOException e) {
e.printStackTrace();
}
@ -106,7 +113,7 @@ public class BufferedRecords {
final byte[] bb = new byte[efs.recordsize];
System.arraycopy(b, start, bb, 0, efs.recordsize);
buffer.put(Long.valueOf(index), bb);
if (buffer.size() > this.maxEntries) flushBuffer();
if (buffer.size() > this.maxEntries) flushBuffer0();
}
}

3
source/net/yacy/kelondro/io/Records.java
Unescape View File

@ -221,12 +221,11 @@ public class Records {
/**
* write buffer to end of file
*/
private void flushBuffer() {
public void flushBuffer() {
try {
raf.seek(raf.length());
raf.write(this.buffer, 0, this.recordsize * this.buffercount);
} catch (final IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
this.buffercount = 0;

6
source/net/yacy/kelondro/table/ChunkIterator.java
Unescape View File

@ -26,6 +26,7 @@ package net.yacy.kelondro.table;
import java.io.BufferedInputStream;
import java.io.DataInputStream;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
@ -40,6 +41,7 @@ public class ChunkIterator implements Iterator<byte[]> {
* create a ChunkIterator
* a ChunkIterator uses a BufferedInputStream to iterate through the file
* and is therefore a fast option to get all elements in the file as a sequence
* ATTENTION: before calling this class ensure that all file buffers are flushed
* @param file: the file
* @param recordsize: the size of the elements in the file
* @param chunksize: the size of the chunks that are returned by next(). remaining bytes until the lenght of recordsize are skipped
@ -79,7 +81,11 @@ public class ChunkIterator implements Iterator<byte[]> {
r += s;
}
return chunk;
} catch (final EOFException e) {
// no real exception, this is the normal termination
return null;
} catch (final IOException e) {
e.printStackTrace();
return null;
}
}

12
source/net/yacy/kelondro/table/SQLTable.java
Unescape View File

@ -34,11 +34,13 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import net.yacy.kelondro.index.ObjectIndex;
import net.yacy.kelondro.index.Row;
import net.yacy.kelondro.index.RowCollection;
import net.yacy.kelondro.index.Row.Entry;
import net.yacy.kelondro.order.ByteOrder;
import net.yacy.kelondro.order.CloneableIterator;
import net.yacy.kelondro.order.NaturalOrder;
@ -55,7 +57,7 @@ import net.yacy.kelondro.order.NaturalOrder;
* grant ALL on yacy.* to yacy;
*/
public class SQLTable implements ObjectIndex {
public class SQLTable implements ObjectIndex, Iterable<Row.Entry> {
private static final String db_driver_str_mysql = "org.gjt.mm.mysql.Driver";
private static final String db_driver_str_pgsql = "org.postgresql.Driver";
@ -266,6 +268,14 @@ public class SQLTable implements ObjectIndex {
// Objects are of type kelondroRow.Entry
return null;
}
public Iterator<Entry> iterator() {
try {
return rows();
} catch (IOException e) {
return null;
}
}
public CloneableIterator<Row.Entry> rows() throws IOException {
return null;

11
source/net/yacy/kelondro/table/SplitTable.java
Unescape View File

@ -50,6 +50,7 @@ import net.yacy.kelondro.index.ObjectIndex;
import net.yacy.kelondro.index.ObjectIndexCache;
import net.yacy.kelondro.index.Row;
import net.yacy.kelondro.index.RowCollection;
import net.yacy.kelondro.index.Row.Entry;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.CloneableIterator;
import net.yacy.kelondro.order.MergeIterator;
@ -61,7 +62,7 @@ import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.NamePrefixThreadFactory;
public class SplitTable implements ObjectIndex {
public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
// this is a set of kelondro tables
// the set is divided into tables with different entry date
@ -594,6 +595,14 @@ public class SplitTable implements ObjectIndex {
return MergeIterator.cascade(c, entryOrder, MergeIterator.simpleMerge, up);
}
public Iterator<Entry> iterator() {
try {
return rows();
} catch (IOException e) {
return null;
}
}
public synchronized CloneableIterator<Row.Entry> rows() throws IOException {
final List<CloneableIterator<Row.Entry>> c = new ArrayList<CloneableIterator<Row.Entry>>(tables.size());
final Iterator<ObjectIndex> i = tables.values().iterator();

46
source/net/yacy/kelondro/table/Table.java
Unescape View File

@ -65,7 +65,7 @@ import net.yacy.kelondro.util.kelondroException;
* The content cache can also be deleted during run-time, if the available RAM gets too low.
*/
public class Table implements ObjectIndex {
public class Table implements ObjectIndex, Iterable<Row.Entry> {
// static tracker objects
private static TreeMap<String, Table> tableTracker = new TreeMap<String, Table>();
@ -628,8 +628,17 @@ public class Table implements ObjectIndex {
public synchronized int size() {
return index.size();
}
public Iterator<Entry> iterator() {
try {
return rows();
} catch (IOException e) {
return null;
}
}
public synchronized CloneableIterator<Entry> rows() throws IOException {
this.file.flushBuffer();
return new rowIteratorNoOrder();
}
@ -747,10 +756,10 @@ public class Table implements ObjectIndex {
return result;
}
private static ObjectIndex testTable(final File f, final String testentities, final boolean useTailCache, final boolean exceed134217727) throws IOException {
private static Table testTable(final File f, final String testentities, final boolean useTailCache, final boolean exceed134217727) throws IOException {
if (f.exists()) FileUtils.deletedelete(f);
final Row rowdef = new Row("byte[] a-4, byte[] b-4", NaturalOrder.naturalOrder);
final ObjectIndex tt = new Table(f, rowdef, 100, 0, useTailCache, exceed134217727);
final Table tt = new Table(f, rowdef, 100, 0, useTailCache, exceed134217727);
byte[] b;
final Row.Entry row = rowdef.newEntry();
for (int i = 0; i < testentities.length(); i++) {
@ -762,19 +771,13 @@ public class Table implements ObjectIndex {
return tt;
}
private static int countElements(final ObjectIndex t) {
private static int countElements(final Table t) {
int count = 0;
try {
final Iterator<Row.Entry> iter = t.rows();
Row.Entry row;
while (iter.hasNext()) {
count++;
row = iter.next();
if (row == null) System.out.println("ERROR! null element found");
// else System.out.println("counted element: " + new
// String(n.getKey()));
}
} catch (final IOException e) {
for (Row.Entry row: t) {
count++;
if (row == null) System.out.println("ERROR! null element found");
// else System.out.println("counted element: " + new
// String(n.getKey()));
}
return count;
}
@ -783,14 +786,16 @@ public class Table implements ObjectIndex {
System.out.println("starting big test with " + elements + " elements:");
final long start = System.currentTimeMillis();
final String[] s = permutations(elements);
ObjectIndex tt;
Table tt;
int count;
try {
for (int i = 0; i < s.length; i++) {
System.out.println("*** probing tree " + i + " for permutation " + s[i]);
// generate tree and delete elements
tt = testTable(testFile, s[i], useTailCache, exceed134217727);
if (countElements(tt) != tt.size()) {
System.out.println("wrong size for " + s[i]);
count = countElements(tt);
if (count != tt.size()) {
System.out.println("wrong size for " + s[i] + ": count = " + count + ", size() = " + tt.size());
}
tt.close();
for (int j = 0; j < s.length; j++) {
@ -798,8 +803,9 @@ public class Table implements ObjectIndex {
// delete by permutation j
for (int elt = 0; elt < s[j].length(); elt++) {
tt.remove(testWord(s[j].charAt(elt)));
if (countElements(tt) != tt.size()) {
System.out.println("ERROR! wrong size for probe tree " + s[i] + "; probe delete " + s[j] + "; position " + elt);
count = countElements(tt);
if (count != tt.size()) {
System.out.println("ERROR! wrong size for probe tree " + s[i] + "; probe delete " + s[j] + "; position " + elt + "; count = " + count + ", size() = " + tt.size());
}
}
tt.close();

Write Preview
Loading…
Cancel
Save