boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fix for bad table iteration

Browse Source
pull/1/head
Michael Peter Christen 10 years ago
parent e05b7332b9
commit 1db476c67e
7 changed files with 150 additions and 38 deletions
Show Stats Download Patch File Download Diff File

12
htroot/NetworkHistory.java
Unescape View File

@ -21,9 +21,9 @@
import java.io.IOException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@ -33,7 +33,6 @@ import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.kelondro.blob.Tables;
import net.yacy.kelondro.blob.Tables.Row;
import net.yacy.search.Switchboard;
import net.yacy.server.serverObjects;
@ -70,16 +69,13 @@ public class NetworkHistory {
long now = System.currentTimeMillis();
long timelimit = now - maxtime * 3600000L;
try {
// BEncodedHeap statTable = sb.tables.getHeap("stats");
// Iterator<byte[]> i = statTable.keys(false, false);
Collection<Row> rowi = Tables.orderByPK(sb.tables.iterator("stats"), 10000, true); // this is bad, fix the statTable.keys(false, false) method
Iterator<Row> rowi = sb.tables.iterator("stats", false);
Map <String, Long> statrow;
for (Row row: rowi) {
while (rowi.hasNext()) {
Row row = rowi.next();
String d = ASCII.String(row.getPK());
Date date = GenericFormatter.SHORT_MINUTE_FORMATTER.parse(d);
//System.out.println(date);
if (date.getTime() < timelimit) break;
//Map<String, byte[]> row = statTable.get(pk);
statrow = new HashMap<>();
for (String key: columns) {
byte[] x = row.get(key);

2
htroot/Table_YMark_p.java
Unescape View File

@ -243,7 +243,7 @@ public class Table_YMark_p {
final String tagsString = YMarkUtil.cleanTagsString(post.get(YMarkEntry.BOOKMARK.TAGS.key()));
mapIterator = sb.tables.bookmarks.getBookmarksByTag(bmk_user, tagsString);
} else {
mapIterator = Tables.orderByPK(sb.tables.iterator(table, matcher), maxcount, false).iterator();
mapIterator = sb.tables.iterator(table, matcher, true);
}
Tables.Row row;

10
htroot/Tables_p.java
Unescape View File

@ -166,14 +166,12 @@ public class Tables_p {
}
count = 0;
try {
final Iterator<Tables.Row> plainIterator = sb.tables.iterator(table, matcher);
final Iterator<Tables.Row> mapIterator = Tables.orderByPK(plainIterator, maxcount, reverse).iterator();
Tables.Row row;
final Iterator<Tables.Row> plainIterator = sb.tables.iterator(table, matcher, !reverse);
boolean dark = true;
byte[] cell;
prop.putXML("showtable_" + count + "_table", table); // only used in XML
while (mapIterator.hasNext() && count < maxcount) {
row = mapIterator.next();
byte[] cell;
while (plainIterator.hasNext() && count < maxcount) {
Tables.Row row = plainIterator.next();
if (row == null) continue;
// write table content

3
htroot/api/table_p.java
Unescape View File

@ -154,8 +154,7 @@ public class table_p {
}
int count = 0;
try {
final Iterator<Tables.Row> plainIterator = sb.tables.iterator(table, matcher);
final Iterator<Tables.Row> mapIterator = Tables.orderByPK(plainIterator, maxcount, false).iterator();
final Iterator<Tables.Row> mapIterator = sb.tables.iterator(table, matcher, true);
Tables.Row trow;
boolean dark = true;
String cellName, cellValue;

2
source/net/yacy/kelondro/blob/BEncodedHeap.java
Unescape View File

@ -564,7 +564,7 @@ public class BEncodedHeap implements MapStore {
}
/**
* iteratore all keys of the table
* iterate all keys of the table
*
* @param up
* @param rotating

154
source/net/yacy/kelondro/blob/Tables.java
Unescape View File

@ -437,33 +437,44 @@ public class Tables implements Iterable<String> {
final BEncodedHeap heap = getHeap(table);
return heap.keys();
}
public Iterator<byte[]> keys(final String table, final boolean up, final boolean rotating) throws IOException {
final BEncodedHeap heap = getHeap(table);
return heap.keys(up, rotating);
}
public Iterator<Row> iterator(final String table) throws IOException {
return new RowIterator(table);
return new HeapRowIterator(table);
}
public Iterator<Row> iterator(final String table, final String whereColumn, final byte[] whereValue) throws IOException {
return new RowIterator(table, whereColumn, whereValue);
return new HeapRowIterator(table, whereColumn, whereValue);
}
public Iterator<Row> iterator(final String table, final String whereColumn, final Pattern wherePattern) throws IOException {
return new RowIterator(table, whereColumn, wherePattern);
return new HeapRowIterator(table, whereColumn, wherePattern);
}
public Iterator<Row> iterator(final String table, final Pattern wherePattern) throws IOException {
return new RowIterator(table, wherePattern);
return new HeapRowIterator(table, wherePattern);
}
public static Collection<Row> orderByPK(final Iterator<Row> rowIterator, int maxcount, boolean reverse) {
final TreeMap<String, Row> sortTree = new TreeMap<String, Row>();
Row row;
while ((maxcount < 0 || maxcount-- > 0) && rowIterator.hasNext()) {
row = rowIterator.next();
sortTree.put(UTF8.String(row.pk), row);
}
return reverse ? sortTree.descendingMap().values() : sortTree.values();
public Iterator<Row> iterator(final String table, final boolean up) throws IOException {
return new OrderedRowIterator(table, up);
}
public Iterator<Row> iterator(final String table, final String whereColumn, final byte[] whereValue, final boolean up) throws IOException {
return new OrderedRowIterator(table, whereColumn, whereValue, up);
}
public Iterator<Row> iterator(final String table, final String whereColumn, final Pattern wherePattern, final boolean up) throws IOException {
return new OrderedRowIterator(table, whereColumn, wherePattern, up);
}
public Iterator<Row> iterator(final String table, final Pattern wherePattern, final boolean up) throws IOException {
return new OrderedRowIterator(table, wherePattern, up);
}
public static Collection<Row> orderBy(final Iterator<Row> rowIterator, int maxcount, final String sortColumn) {
final TreeMap<String, Row> sortTree = new TreeMap<String, Row>();
Row row;
@ -485,7 +496,7 @@ public class Tables implements Iterable<String> {
return heap.columns();
}
public class RowIterator extends LookAheadIterator<Row> implements Iterator<Row> {
public class HeapRowIterator extends LookAheadIterator<Row> implements Iterator<Row> {
private final String whereColumn;
private final byte[] whereValue;
@ -497,7 +508,7 @@ public class Tables implements Iterable<String> {
* @param table
* @throws IOException
*/
public RowIterator(final String table) throws IOException {
public HeapRowIterator(final String table) throws IOException {
this.whereColumn = null;
this.whereValue = null;
this.wherePattern = null;
@ -513,7 +524,7 @@ public class Tables implements Iterable<String> {
* @param whereValue
* @throws IOException
*/
public RowIterator(final String table, final String whereColumn, final byte[] whereValue) throws IOException {
public HeapRowIterator(final String table, final String whereColumn, final byte[] whereValue) throws IOException {
assert whereColumn != null || whereValue == null;
this.whereColumn = whereColumn;
this.whereValue = whereValue;
@ -530,7 +541,7 @@ public class Tables implements Iterable<String> {
* @param wherePattern
* @throws IOException
*/
public RowIterator(final String table, final String whereColumn, final Pattern wherePattern) throws IOException {
public HeapRowIterator(final String table, final String whereColumn, final Pattern wherePattern) throws IOException {
this.whereColumn = whereColumn;
this.whereValue = null;
this.wherePattern = wherePattern == null || wherePattern.toString().isEmpty() ? null : wherePattern;
@ -545,7 +556,7 @@ public class Tables implements Iterable<String> {
* @param pattern
* @throws IOException
*/
public RowIterator(final String table, final Pattern pattern) throws IOException {
public HeapRowIterator(final String table, final Pattern pattern) throws IOException {
this.whereColumn = null;
this.whereValue = null;
this.wherePattern = pattern == null || pattern.toString().isEmpty() ? null : pattern;
@ -577,9 +588,116 @@ public class Tables implements Iterable<String> {
}
return null;
}
}
public class OrderedRowIterator extends LookAheadIterator<Row> implements Iterator<Row> {
private final String whereColumn;
private final byte[] whereValue;
private final Pattern wherePattern;
private final Iterator<byte[]> i;
private final BEncodedHeap heap;
/**
* iterator that iterates all elements in the given table
* @param table
* @param up
* @throws IOException
*/
public OrderedRowIterator(final String table, final boolean up) throws IOException {
this.whereColumn = null;
this.whereValue = null;
this.wherePattern = null;
this.heap = getHeap(table);
this.i = heap.keys(up, false);
}
/**
* iterator that iterates all elements in the given table
* where a given column is equal to a given value
* @param table
* @param whereColumn
* @param whereValue
* @param up
* @throws IOException
*/
public OrderedRowIterator(final String table, final String whereColumn, final byte[] whereValue, final boolean up) throws IOException {
assert whereColumn != null || whereValue == null;
this.whereColumn = whereColumn;
this.whereValue = whereValue;
this.wherePattern = null;
this.heap = getHeap(table);
this.i = heap.keys(up, false);
}
/**
* iterator that iterates all elements in the given table
* where a given column matches with a given value
* @param table
* @param whereColumn
* @param wherePattern
* @param up
* @throws IOException
*/
public OrderedRowIterator(final String table, final String whereColumn, final Pattern wherePattern, final boolean up) throws IOException {
this.whereColumn = whereColumn;
this.whereValue = null;
this.wherePattern = wherePattern == null || wherePattern.toString().isEmpty() ? null : wherePattern;
this.heap = getHeap(table);
this.i = heap.keys(up, false);
}
/**
* iterator that iterates all elements in the given table
* where a given column matches with a given value
* @param table
* @param wherePattern
* @param up
* @throws IOException
*/
public OrderedRowIterator(final String table, final Pattern wherePattern, final boolean up) throws IOException {
this.whereColumn = null;
this.whereValue = null;
this.wherePattern = wherePattern == null || wherePattern.toString().isEmpty() ? null : wherePattern;
this.heap = getHeap(table);
this.i = heap.keys(up, false);
}
@Override
protected Row next0() {
if (this.i == null) return null;
Row r;
while (this.i.hasNext()) {
byte[] pk = this.i.next();
Map<String, byte[]> map;
try {
map = this.heap.get(pk);
if (map == null) continue;
r = new Row(pk, map);
if (this.whereValue != null) {
if (ByteBuffer.equals(r.get(this.whereColumn), this.whereValue)) return r;
} else if (this.wherePattern != null) {
if (this.whereColumn == null) {
// shall match any column
for (final byte[] b: r.values()) {
if (this.wherePattern.matcher(UTF8.String(b)).matches()) return r;
}
} else {
// must match the given column
if (this.wherePattern.matcher(UTF8.String(r.get(this.whereColumn))).matches()) return r;
}
} else {
return r;
}
} catch (IOException | SpaceExceededException e) {
continue;
}
}
return null;
}
}
public static class Data extends LinkedHashMap<String, byte[]> {
private static final long serialVersionUID = 978426054043749337L;

5
source/net/yacy/kelondro/index/RowSet.java
Unescape View File

@ -360,6 +360,7 @@ public class RowSet extends RowCollection implements Index, Iterable<Row.Entry>,
@Override
public final synchronized CloneableIterator<byte[]> keys(final boolean up, final byte[] firstKey) {
this.sort();
return new keyIterator(up, firstKey);
}
@ -378,10 +379,10 @@ public class RowSet extends RowCollection implements Index, Iterable<Row.Entry>,
this.first = firstKey;
this.bound = RowSet.this.sortBound;
if (this.first == null) {
this.p = 0;
this.p = up ? 0 : this.bound - 1;
} else {
assert this.first.length == RowSet.this.rowdef.primaryKeyLength : "first.length = " + this.first.length + ", rowdef.primaryKeyLength = " + RowSet.this.rowdef.primaryKeyLength;
this.p = binaryPosition(this.first, 0); // check this to find bug in DHT selection enumeration
this.p = up ? binaryPosition(this.first, 0) : this.bound - 1; // check this to find bug in DHT selection enumeration
}
}

Write Preview
Loading…
Cancel
Save