diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index 74a95d67c..bef856467 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -490,7 +490,9 @@ public final class yacyClient { } // insert results to containers + int term = count; for (final URIMetadataRow urlEntry: result.links) { + if (term-- <= 0) break; // do not process more that requested (in case that evil peers fill us up with rubbish) // get one single search result if (urlEntry == null) continue; assert (urlEntry.hash().length == 12) : "urlEntry.hash() = " + ASCII.String(urlEntry.hash()); diff --git a/source/net/yacy/kelondro/index/Row.java b/source/net/yacy/kelondro/index/Row.java index 2fc4e435f..2f2c3f960 100644 --- a/source/net/yacy/kelondro/index/Row.java +++ b/source/net/yacy/kelondro/index/Row.java @@ -379,7 +379,7 @@ public final class Row { } public final byte[] bytes() { - if ((this.offset == 0) && (this.rowinstance.length == Row.this.objectsize)) { + if (this.offset == 0 && this.rowinstance.length == Row.this.objectsize) { return this.rowinstance; } final byte[] tmp = new byte[Row.this.objectsize]; @@ -518,6 +518,10 @@ public final class Row { } public final byte[] getPrimaryKeyBytes() { + if (Row.this.columns() == 1 && this.offset == 0 && this.rowinstance.length == Row.this.primaryKeyLength) { + // avoid memory allocation in case that the row consists in only the primary key + return this.rowinstance; + } final byte[] c = new byte[Row.this.primaryKeyLength]; System.arraycopy(this.rowinstance, this.offset, c, 0, Row.this.primaryKeyLength); return c; diff --git a/source/net/yacy/kelondro/index/RowSet.java b/source/net/yacy/kelondro/index/RowSet.java index 2e59ab419..47be42021 100644 --- a/source/net/yacy/kelondro/index/RowSet.java +++ b/source/net/yacy/kelondro/index/RowSet.java @@ -158,10 +158,12 @@ public class RowSet extends RowCollection implements Index, Iterable */ public final boolean put(final Row.Entry entry) throws RowSpaceExceededException { assert (entry != null); - assert (entry.getPrimaryKeyBytes() != null); + final byte[] key = entry.getPrimaryKeyBytes(); + assert (key != null); + final byte[] entrybytes = entry.bytes(); + assert entrybytes.length >= this.rowdef.primaryKeyLength; synchronized (this) { - assert entry.bytes().length >= this.rowdef.primaryKeyLength; - final int index = find(entry.bytes(), 0); + final int index = find(key, 0); if (index < 0) { super.addUnique(entry); return true; @@ -174,26 +176,30 @@ public class RowSet extends RowCollection implements Index, Iterable } } - public final synchronized Row.Entry replace(final Row.Entry entry) throws RowSpaceExceededException { + public final Row.Entry replace(final Row.Entry entry) throws RowSpaceExceededException { assert (entry != null); - assert (entry.getPrimaryKeyBytes() != null); - int index = -1; - Row.Entry oldentry = null; - // when reaching a specific amount of un-sorted entries, re-sort all - if ((this.chunkcount - this.sortBound) > collectionReSortLimit) { - sort(); - } - assert entry.bytes().length >= this.rowdef.primaryKeyLength; - index = find(entry.bytes(), 0); - if (index < 0) { - super.addUnique(entry); - } else { - oldentry = get(index, true); - final int sb = this.sortBound; // save the sortBound, because it is not altered (we replace at the same place) - set(index, entry); // this may alter the sortBound, which we will revert in the next step - this.sortBound = sb; // revert a sortBound altering + final byte[] key = entry.getPrimaryKeyBytes(); + assert (key != null); + final byte[] entrybytes = entry.bytes(); + assert entrybytes.length >= this.rowdef.primaryKeyLength; + synchronized (this) { + int index = -1; + Row.Entry oldentry = null; + // when reaching a specific amount of un-sorted entries, re-sort all + if ((this.chunkcount - this.sortBound) > collectionReSortLimit) { + sort(); + } + index = find(key, 0); + if (index < 0) { + super.addUnique(entry); + } else { + oldentry = get(index, true); + final int sb = this.sortBound; // save the sortBound, because it is not altered (we replace at the same place) + set(index, entry); // this may alter the sortBound, which we will revert in the next step + this.sortBound = sb; // revert a sortBound altering + } + return oldentry; } - return oldentry; } public final synchronized long inc(final byte[] key, final int col, final long add, final Row.Entry initrow) throws RowSpaceExceededException { diff --git a/source/net/yacy/kelondro/table/SplitTable.java b/source/net/yacy/kelondro/table/SplitTable.java index 8fc687803..587f3c703 100644 --- a/source/net/yacy/kelondro/table/SplitTable.java +++ b/source/net/yacy/kelondro/table/SplitTable.java @@ -388,8 +388,13 @@ public class SplitTable implements Index, Iterable { assert row.objectsize() <= this.rowdef.objectsize; final byte[] key = row.getPrimaryKeyBytes(); if (this.tables == null) return true; + Index keeper = null; synchronized (this.tables) { - Index keeper = keeperOf(key); + keeper = keeperOf(key); + } + if (keeper != null) return keeper.put(row); + synchronized (this.tables) { + keeper = keeperOf(key); // we must check that again because it could have changed in between if (keeper != null) return keeper.put(row); assert this.current == null || this.tables.get(this.current) != null : "this.current = " + this.current; keeper = (this.current == null) ? newTable() : checkTable(this.tables.get(this.current)); diff --git a/source/net/yacy/kelondro/table/Table.java b/source/net/yacy/kelondro/table/Table.java index 1ae57a373..0f77eb8b6 100644 --- a/source/net/yacy/kelondro/table/Table.java +++ b/source/net/yacy/kelondro/table/Table.java @@ -355,16 +355,17 @@ public class Table implements Index, Iterable { // try again with less memory this.index.putUnique(row.getPrimaryKeyBytes(), i); } + final byte[] rowbytes = row.bytes(); if (this.table != null) { assert this.table.size() == i; try { - this.table.addUnique(this.taildef.newEntry(row.bytes(), this.rowdef.primaryKeyLength, true)); + this.table.addUnique(this.taildef.newEntry(rowbytes, this.rowdef.primaryKeyLength, true)); } catch (final RowSpaceExceededException e) { this.table = null; } if (abandonTable()) this.table = null; } - this.file.add(row.bytes(), 0); + this.file.add(rowbytes, 0); assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size(); } @@ -511,49 +512,54 @@ public class Table implements Index, Iterable { return this.index.keys(up, firstKey); } - public synchronized Entry replace(final Entry row) throws IOException, RowSpaceExceededException { - assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size(); - assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size(); + public Entry replace(final Entry row) throws IOException, RowSpaceExceededException { assert row != null; - assert row.bytes() != null; - if (row == null || row.bytes() == null) return null; - final int i = (int) this.index.get(row.getPrimaryKeyBytes()); - if (i == -1) { - try { - addUnique(row); - } catch (final RowSpaceExceededException e) { - if (this.table == null) throw e; - this.table = null; - addUnique(row); + if (this.file == null || row == null) return null; + final byte[] rowb = row.bytes(); + assert rowb != null; + if (rowb == null) return null; + final byte[] key = row.getPrimaryKeyBytes(); + synchronized (this) { + assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size(); + assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size(); + final int i = (int) this.index.get(key); + if (i == -1) { + try { + addUnique(row); + } catch (final RowSpaceExceededException e) { + if (this.table == null) throw e; + this.table = null; + addUnique(row); + } + return null; } - return null; - } - final byte[] b = new byte[this.rowdef.objectsize]; - Row.Entry cacherow; - if (this.table == null || (cacherow = this.table.get(i, false)) == null) { - // read old value - this.file.get(i, b, 0); - // write new value - this.file.put(i, row.bytes(), 0); - } else { - // read old value - assert cacherow != null; - System.arraycopy(row.getPrimaryKeyBytes(), 0, b, 0, this.rowdef.primaryKeyLength); - System.arraycopy(cacherow.bytes(), 0, b, this.rowdef.primaryKeyLength, this.rowdef.objectsize - this.rowdef.primaryKeyLength); - // write new value - try { - this.table.set(i, this.taildef.newEntry(row.bytes(), this.rowdef.primaryKeyLength, true)); - } catch (final RowSpaceExceededException e) { - this.table = null; + final byte[] b = new byte[this.rowdef.objectsize]; + Row.Entry cacherow; + if (this.table == null || (cacherow = this.table.get(i, false)) == null) { + // read old value + this.file.get(i, b, 0); + // write new value + this.file.put(i, rowb, 0); + } else { + // read old value + assert cacherow != null; + System.arraycopy(key, 0, b, 0, this.rowdef.primaryKeyLength); + System.arraycopy(cacherow.bytes(), 0, b, this.rowdef.primaryKeyLength, this.rowdef.objectsize - this.rowdef.primaryKeyLength); + // write new value + try { + this.table.set(i, this.taildef.newEntry(rowb, this.rowdef.primaryKeyLength, true)); + } catch (final RowSpaceExceededException e) { + this.table = null; + } + if (abandonTable()) this.table = null; + this.file.put(i, rowb, 0); } - if (abandonTable()) this.table = null; - this.file.put(i, row.bytes(), 0); + assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size(); + assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size(); + // return old value + return this.rowdef.newEntry(b); } - assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size(); - assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size(); - // return old value - return this.rowdef.newEntry(b); } /** @@ -563,39 +569,44 @@ public class Table implements Index, Iterable { * @throws IOException * @throws RowSpaceExceededException */ - public synchronized boolean put(final Entry row) throws IOException, RowSpaceExceededException { - assert this.file == null || this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size() + ", file = " + filename(); - assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size() + ", file = " + filename(); + public boolean put(final Entry row) throws IOException, RowSpaceExceededException { assert row != null; - assert row.bytes() != null; - if (this.file == null || row == null || row.bytes() == null) return true; - final int i = (int) this.index.get(row.getPrimaryKeyBytes()); - if (i == -1) { - try { - addUnique(row); - } catch (final RowSpaceExceededException e) { - if (this.table == null) throw e; - this.table = null; - addUnique(row); + if (this.file == null || row == null) return true; + final byte[] rowb = row.bytes(); + assert rowb != null; + if (rowb == null) return true; + final byte[] key = row.getPrimaryKeyBytes(); + synchronized (this) { + assert this.file == null || this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size() + ", file = " + filename(); + assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size() + ", file = " + filename(); + final int i = (int) this.index.get(key); + if (i == -1) { + try { + addUnique(row); + } catch (final RowSpaceExceededException e) { + if (this.table == null) throw e; + this.table = null; + addUnique(row); + } + return true; } - return true; - } - if (this.table == null) { - // write new value - this.file.put(i, row.bytes(), 0); - } else { - // write new value - this.file.put(i, row.bytes(), 0); - if (abandonTable()) this.table = null; else try { - this.table.set(i, this.taildef.newEntry(row.bytes(), this.rowdef.primaryKeyLength, true)); - } catch (final RowSpaceExceededException e) { - this.table = null; + if (this.table == null) { + // write new value + this.file.put(i, rowb, 0); + } else { + // write new value + this.file.put(i, rowb, 0); + if (abandonTable()) this.table = null; else try { + this.table.set(i, this.taildef.newEntry(rowb, this.rowdef.primaryKeyLength, true)); + } catch (final RowSpaceExceededException e) { + this.table = null; + } } + assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size(); + assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size(); + return false; } - assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size(); - assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size(); - return false; } public Entry put(final Entry row, final Date entryDate) throws IOException, RowSpaceExceededException {