From 13c666adefb6d952698e2df0b001eed9b43c3d64 Mon Sep 17 00:00:00 2001 From: orbiter Date: Wed, 11 Mar 2009 20:23:19 +0000 Subject: [PATCH] performance hack to ObjectIndex put() method: Java standard classes provide a Map Interface, that has a put() method that returns the object that was replaced by the object that was the argument of the put call. The kelondro ObjectIndex defined a put method in the same way, that means it also returned the previous value of the Entry object before the put call. However, this value was not used by the calling code in the most cases. Omitting a return of the previous value would cause some performance benefit. This change implements a put method that does not return the previous value to reflect the common use. Omitting the return of previous values will cause some benefit in performance. The functionality to get the previous value is still maintained, and provided with a new 'replace' method. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5700 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/de/anomic/kelondro/blob/Cache.java | 58 ++++++++++++++----- .../de/anomic/kelondro/index/HandleSet.java | 2 +- .../kelondro/index/IntegerHandleIndex.java | 2 +- .../kelondro/index/LongHandleIndex.java | 2 +- .../kelondro/index/ObjectArrayCache.java | 2 +- .../de/anomic/kelondro/index/ObjectIndex.java | 7 ++- .../kelondro/index/ObjectIndexCache.java | 35 ++++++----- .../kelondro/index/ObjectIndexDaemon.java | 16 +++-- .../anomic/kelondro/index/RowCollection.java | 2 +- source/de/anomic/kelondro/index/RowSet.java | 24 ++++++-- source/de/anomic/kelondro/table/EcoTable.java | 36 +++++++++--- .../de/anomic/kelondro/table/FlexTable.java | 29 ++++++++-- .../de/anomic/kelondro/table/Relations.java | 2 +- source/de/anomic/kelondro/table/SQLTable.java | 34 +++++++---- .../de/anomic/kelondro/table/SplitTable.java | 40 ++++++++++--- source/de/anomic/kelondro/table/Tree.java | 17 ++---- .../kelondro/text/ReferenceContainer.java | 6 +- source/de/anomic/yacy/yacyNewsDB.java | 4 +- 18 files changed, 224 insertions(+), 94 deletions(-) diff --git a/source/de/anomic/kelondro/blob/Cache.java b/source/de/anomic/kelondro/blob/Cache.java index 841f6f1c9..169791646 100644 --- a/source/de/anomic/kelondro/blob/Cache.java +++ b/source/de/anomic/kelondro/blob/Cache.java @@ -255,25 +255,55 @@ public class Cache implements ObjectIndex { // learn from result if (entry == null) { if ((checkMissSpace()) && (readMissCache != null)) { - final Row.Entry dummy = readMissCache.put(readMissCache.row().newEntry(key)); + final Row.Entry dummy = readMissCache.replace(readMissCache.row().newEntry(key)); if (dummy == null) this.hasnotUnique++; else this.hasnotDouble++; } return null; } if ((checkHitSpace()) && (readHitCache != null)) { - final Row.Entry dummy = readHitCache.put(entry); + final Row.Entry dummy = readHitCache.replace(entry); if (dummy == null) this.writeUnique++; else this.writeDouble++; } return entry; } - public synchronized void putMultiple(final List rows) throws IOException { + public synchronized void put(final List rows) throws IOException { final Iterator i = rows.iterator(); while (i.hasNext()) put(i.next()); } + + public synchronized void put(final Entry row) throws IOException { + assert (row != null); + assert (row.columns() == row().columns()); + //assert (!(serverLog.allZero(row.getColBytes(index.primarykey())))); + + final byte[] key = row.getPrimaryKeyBytes(); + checkHitSpace(); + + // remove entry from miss- and hit-cache + if (readMissCache != null) { + if (readMissCache.remove(key) != null) { + this.hasnotHit++; + // the entry does not exist before + index.put(row); // write to backend + if (readHitCache != null) { + final Row.Entry dummy = readHitCache.replace(row); // learn that entry + if (dummy == null) this.writeUnique++; else this.writeDouble++; + } + return; + } + } + + index.put(row); + if (readHitCache != null) { + // learn from situation + final Row.Entry dummy = readHitCache.replace(row); // overwrite old entry + if (dummy == null) this.writeUnique++; else this.writeDouble++; + } + } - public synchronized Entry put(final Entry row) throws IOException { + public synchronized Entry replace(final Entry row) throws IOException { assert (row != null); assert (row.columns() == row().columns()); //assert (!(serverLog.allZero(row.getColBytes(index.primarykey())))); @@ -288,7 +318,7 @@ public class Cache implements ObjectIndex { // the entry does not exist before index.put(row); // write to backend if (readHitCache != null) { - final Row.Entry dummy = readHitCache.put(row); // learn that entry + final Row.Entry dummy = readHitCache.replace(row); // learn that entry if (dummy == null) this.writeUnique++; else this.writeDouble++; } return null; @@ -304,16 +334,16 @@ public class Cache implements ObjectIndex { // write directly to backend index index.put(row); // learn from situation - final Row.Entry dummy = readHitCache.put(row); // overwrite old entry + final Row.Entry dummy = readHitCache.replace(row); // overwrite old entry if (dummy == null) this.writeUnique++; else this.writeDouble++; return entry; } } // the worst case: we must write to the back-end directly - entry = index.put(row); + entry = index.replace(row); if (readHitCache != null) { - final Row.Entry dummy = readHitCache.put(row); // learn that entry + final Row.Entry dummy = readHitCache.replace(row); // learn that entry if (dummy == null) this.writeUnique++; else this.writeDouble++; } return entry; @@ -334,7 +364,7 @@ public class Cache implements ObjectIndex { // the entry does not exist before index.addUnique(row); // write to backend if (readHitCache != null) { - final Row.Entry dummy = readHitCache.put(row); // learn that entry + final Row.Entry dummy = readHitCache.replace(row); // learn that entry if (dummy == null) this.writeUnique++; else this.writeDouble++; } return; @@ -343,7 +373,7 @@ public class Cache implements ObjectIndex { // the worst case: we must write to the back-end directly index.addUnique(row); if (readHitCache != null) { - final Row.Entry dummy = readHitCache.put(row); // learn that entry + final Row.Entry dummy = readHitCache.replace(row); // learn that entry if (dummy == null) this.writeUnique++; else this.writeDouble++; } } @@ -369,12 +399,12 @@ public class Cache implements ObjectIndex { // the worst case: we must write to the backend directly index.addUnique(row); if (readHitCache != null) { - final Row.Entry dummy = readHitCache.put(row); // learn that entry + final Row.Entry dummy = readHitCache.replace(row); // learn that entry if (dummy == null) this.writeUnique++; else this.writeDouble++; } } - public synchronized void addUniqueMultiple(final List rows) throws IOException { + public synchronized void addUnique(final List rows) throws IOException { final Iterator i = rows.iterator(); while (i.hasNext()) addUnique(i.next()); } @@ -390,7 +420,7 @@ public class Cache implements ObjectIndex { // add entry to miss-cache if (readMissCache != null) { // set the miss cache; if there was already an entry we know that the return value must be null - final Row.Entry dummy = readMissCache.put(readMissCache.row().newEntry(key)); + final Row.Entry dummy = readMissCache.replace(readMissCache.row().newEntry(key)); if (dummy == null) { this.hasnotUnique++; } else { @@ -421,7 +451,7 @@ public class Cache implements ObjectIndex { if (entry == null) return null; final byte[] key = entry.getPrimaryKeyBytes(); if (readMissCache != null) { - final Row.Entry dummy = readMissCache.put(readMissCache.row().newEntry(key)); + final Row.Entry dummy = readMissCache.replace(readMissCache.row().newEntry(key)); if (dummy == null) this.hasnotUnique++; else this.hasnotDouble++; } if (readHitCache != null) { diff --git a/source/de/anomic/kelondro/index/HandleSet.java b/source/de/anomic/kelondro/index/HandleSet.java index 3bcf74676..f753b5537 100644 --- a/source/de/anomic/kelondro/index/HandleSet.java +++ b/source/de/anomic/kelondro/index/HandleSet.java @@ -109,7 +109,7 @@ public class HandleSet implements Iterable { assert (key != null); final Row.Entry newentry = index.row().newEntry(); newentry.setCol(0, key); - final Row.Entry oldentry = index.put(newentry); + final Row.Entry oldentry = index.replace(newentry); if (oldentry == null) return -1; return (int) oldentry.getColLong(1); } diff --git a/source/de/anomic/kelondro/index/IntegerHandleIndex.java b/source/de/anomic/kelondro/index/IntegerHandleIndex.java index 208a6be49..39253a0df 100644 --- a/source/de/anomic/kelondro/index/IntegerHandleIndex.java +++ b/source/de/anomic/kelondro/index/IntegerHandleIndex.java @@ -129,7 +129,7 @@ public class IntegerHandleIndex { final Row.Entry newentry = index.row().newEntry(); newentry.setCol(0, key); newentry.setCol(1, i); - final Row.Entry oldentry = index.put(newentry); + final Row.Entry oldentry = index.replace(newentry); if (oldentry == null) return -1; return (int) oldentry.getColLong(1); } diff --git a/source/de/anomic/kelondro/index/LongHandleIndex.java b/source/de/anomic/kelondro/index/LongHandleIndex.java index d51e98736..ac49aca1c 100644 --- a/source/de/anomic/kelondro/index/LongHandleIndex.java +++ b/source/de/anomic/kelondro/index/LongHandleIndex.java @@ -129,7 +129,7 @@ public class LongHandleIndex { final Row.Entry newentry = index.row().newEntry(); newentry.setCol(0, key); newentry.setCol(1, l); - final Row.Entry oldentry = index.put(newentry); + final Row.Entry oldentry = index.replace(newentry); if (oldentry == null) return -1; return oldentry.getColLong(1); } diff --git a/source/de/anomic/kelondro/index/ObjectArrayCache.java b/source/de/anomic/kelondro/index/ObjectArrayCache.java index 265734236..bd9ee4210 100644 --- a/source/de/anomic/kelondro/index/ObjectArrayCache.java +++ b/source/de/anomic/kelondro/index/ObjectArrayCache.java @@ -108,7 +108,7 @@ public class ObjectArrayCache { final Row.Entry newentry = rowdef.newEntry(); newentry.setCol(0, ii); newentry.setCol(1, value); - final Row.Entry oldentry = index1.put(newentry); + final Row.Entry oldentry = index1.replace(newentry); if (oldentry == null) return null; return oldentry.getColBytes(1); } diff --git a/source/de/anomic/kelondro/index/ObjectIndex.java b/source/de/anomic/kelondro/index/ObjectIndex.java index 28fc75dce..1b51d58e8 100644 --- a/source/de/anomic/kelondro/index/ObjectIndex.java +++ b/source/de/anomic/kelondro/index/ObjectIndex.java @@ -44,10 +44,11 @@ public interface ObjectIndex { public Row row(); public boolean has(byte[] key); // use this only if there is no get in case that has returns true public Row.Entry get(byte[] key) throws IOException; - public Row.Entry put(Row.Entry row) throws IOException; - public void putMultiple(List rows) throws IOException; // for R/W head path optimization + public Row.Entry replace(Row.Entry row) throws IOException; + public void put(Row.Entry row) throws IOException; + public void put(List rows) throws IOException; // for R/W head path optimization public void addUnique(Row.Entry row) throws IOException; // no double-check - public void addUniqueMultiple(List rows) throws IOException; // no double-check + public void addUnique(List rows) throws IOException; // no double-check //public long inc(final byte[] key, int col, long add, Row.Entry initrow); // replace a column with a recomputed value public ArrayList removeDoubles() throws IOException; // removes all elements that are double (to be used after all addUnique) public Row.Entry remove(byte[] key) throws IOException; diff --git a/source/de/anomic/kelondro/index/ObjectIndexCache.java b/source/de/anomic/kelondro/index/ObjectIndexCache.java index 8d878ef7f..df40caeab 100644 --- a/source/de/anomic/kelondro/index/ObjectIndexCache.java +++ b/source/de/anomic/kelondro/index/ObjectIndexCache.java @@ -26,7 +26,6 @@ package de.anomic.kelondro.index; import java.io.IOException; import java.util.ArrayList; -import java.util.Date; import java.util.Iterator; import java.util.List; @@ -102,7 +101,7 @@ public class ObjectIndexCache implements ObjectIndex { return index1.put(entry); } */ - public synchronized Row.Entry put(final Row.Entry entry) { + public synchronized Row.Entry replace(final Row.Entry entry) { assert (entry != null); finishInitialization(); // if the new entry is within the initialization part, just overwrite it @@ -110,21 +109,29 @@ public class ObjectIndexCache implements ObjectIndex { byte[] key = entry.getPrimaryKeyBytes(); if (index0.has(key)) { // replace the entry - return index0.put(entry); + return index0.replace(entry); } // else place it in the index1 - return index1.put(entry); + return index1.replace(entry); } - - public Entry put(final Entry row, final Date entryDate) { - return put(row); - } - - public void putMultiple(final List rows) { + + public synchronized void put(final Row.Entry entry) { + assert (entry != null); + finishInitialization(); + // if the new entry is within the initialization part, just overwrite it + assert index0.isSorted(); + byte[] key = entry.getPrimaryKeyBytes(); + if (index0.has(key)) { + // replace the entry + index0.put(entry); + } + // else place it in the index1 + index1.put(entry); + } + + public void put(final List rows) { final Iterator i = rows.iterator(); - while (i.hasNext()) { - put(i.next()); - } + while (i.hasNext()) put(i.next()); } public synchronized void addUnique(final Row.Entry entry) { @@ -138,7 +145,7 @@ public class ObjectIndexCache implements ObjectIndex { index1.addUnique(entry); } - public void addUniqueMultiple(final List rows) { + public void addUnique(final List rows) { final Iterator i = rows.iterator(); while (i.hasNext()) addUnique(i.next()); } diff --git a/source/de/anomic/kelondro/index/ObjectIndexDaemon.java b/source/de/anomic/kelondro/index/ObjectIndexDaemon.java index 39d8cffd0..2e5311234 100644 --- a/source/de/anomic/kelondro/index/ObjectIndexDaemon.java +++ b/source/de/anomic/kelondro/index/ObjectIndexDaemon.java @@ -75,8 +75,8 @@ public class ObjectIndexDaemon implements ObjectIndex { index.addUnique(row); } - public void addUniqueMultiple(List rows) throws IOException { - index.addUniqueMultiple(rows); + public void addUnique(List rows) throws IOException { + index.addUnique(rows); } public void clear() throws IOException { @@ -112,7 +112,7 @@ public class ObjectIndexDaemon implements ObjectIndex { return this.index.has(key); } - public Entry put(Entry row) throws IOException { + public Entry replace(Entry row) throws IOException { Entry entry = get(row.getPrimaryKeyBytes()); try { this.queue.put(row); @@ -122,7 +122,15 @@ public class ObjectIndexDaemon implements ObjectIndex { return entry; } - public void putMultiple(List rows) throws IOException { + public void put(Entry row) throws IOException { + try { + this.queue.put(row); + } catch (InterruptedException e) { + this.index.put(row); + } + } + + public void put(List rows) throws IOException { for (Entry entry: rows) try { this.queue.put(entry); } catch (InterruptedException e) { diff --git a/source/de/anomic/kelondro/index/RowCollection.java b/source/de/anomic/kelondro/index/RowCollection.java index 737343957..c7572a866 100644 --- a/source/de/anomic/kelondro/index/RowCollection.java +++ b/source/de/anomic/kelondro/index/RowCollection.java @@ -288,7 +288,7 @@ public class RowCollection implements Iterable { addUnique(r, 0, r.length); } - public synchronized void addUniqueMultiple(final List rows) { + public synchronized void addUnique(final List rows) { assert this.sortBound == 0 : "sortBound = " + this.sortBound + ", chunkcount = " + this.chunkcount; final Iterator i = rows.iterator(); while (i.hasNext()) addUnique(i.next()); diff --git a/source/de/anomic/kelondro/index/RowSet.java b/source/de/anomic/kelondro/index/RowSet.java index 33a004bd2..2e9a3ed41 100644 --- a/source/de/anomic/kelondro/index/RowSet.java +++ b/source/de/anomic/kelondro/index/RowSet.java @@ -26,7 +26,6 @@ package de.anomic.kelondro.index; import java.io.DataInput; import java.io.IOException; -import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.Random; @@ -105,16 +104,29 @@ public class RowSet extends RowCollection implements ObjectIndex, Iterable rows) { + public synchronized void put(final List rows) { final Iterator i = rows.iterator(); while (i.hasNext()) put(i.next()); } - public Row.Entry put(final Row.Entry row, final Date entryDate) { - return put(row); + public synchronized void put(final Row.Entry entry) { + assert (entry != null); + assert (entry.getPrimaryKeyBytes() != null); + // when reaching a specific amount of un-sorted entries, re-sort all + if ((this.chunkcount - this.sortBound) > collectionReSortLimit) { + sort(); + } + int index = find(entry.bytes(), (rowdef.primaryKeyIndex < 0) ? 0 :super.rowdef.colstart[rowdef.primaryKeyIndex], super.rowdef.primaryKeyLength); + if (index < 0) { + super.addUnique(entry); + } else { + int sb = this.sortBound; // save the sortBound, because it is not altered (we replace at the same place) + set(index, entry); // this may alter the sortBound, which we will revert in the next step + this.sortBound = sb; // revert a sortBound altering + } } - - public synchronized Row.Entry put(final Row.Entry entry) { + + public synchronized Row.Entry replace(final Row.Entry entry) { assert (entry != null); assert (entry.getPrimaryKeyBytes() != null); int index = -1; diff --git a/source/de/anomic/kelondro/table/EcoTable.java b/source/de/anomic/kelondro/table/EcoTable.java index 014b2076a..59edcfd9f 100644 --- a/source/de/anomic/kelondro/table/EcoTable.java +++ b/source/de/anomic/kelondro/table/EcoTable.java @@ -274,7 +274,7 @@ public class EcoTable implements ObjectIndex { assert file.size() == index.size() + fail : "file.size() = " + file.size() + ", index.size() = " + index.size(); } - public synchronized void addUniqueMultiple(final List rows) throws IOException { + public synchronized void addUnique(final List rows) throws IOException { assert file.size() == index.size() + fail : "file.size() = " + file.size() + ", index.size() = " + index.size(); final Iterator i = rows.iterator(); while (i.hasNext()) { @@ -382,7 +382,7 @@ public class EcoTable implements ObjectIndex { return index.keys(up, firstKey); } - public synchronized Entry put(final Entry row) throws IOException { + public synchronized Entry replace(final Entry row) throws IOException { assert file.size() == index.size() + fail : "file.size() = " + file.size() + ", index.size() = " + index.size(); assert ((table == null) || (table.size() == index.size())); assert row != null; @@ -415,17 +415,39 @@ public class EcoTable implements ObjectIndex { // return old value return rowdef.newEntry(b); } + + public synchronized void put(final Entry row) throws IOException { + assert file.size() == index.size() + fail : "file.size() = " + file.size() + ", index.size() = " + index.size(); + assert ((table == null) || (table.size() == index.size())); + assert row != null; + assert row.bytes() != null; + if ((row == null) || (row.bytes() == null)) return; + final int i = index.get(row.getPrimaryKeyBytes()); + if (i == -1) { + addUnique(row); + return; + } + + if (table == null) { + // write new value + file.put(i, row.bytes(), 0); + } else { + // write new value + table.set(i, taildef.newEntry(row.bytes(), rowdef.primaryKeyLength, true)); + file.put(i, row.bytes(), 0); + } + assert file.size() == index.size() + fail : "file.size() = " + file.size() + ", index.size() = " + index.size(); + assert ((table == null) || (table.size() == index.size())); + } public synchronized Entry put(final Entry row, final Date entryDate) throws IOException { - return put(row); + return replace(row); } - public synchronized void putMultiple(final List rows) throws IOException { + public synchronized void put(final List rows) throws IOException { assert file.size() == index.size() + fail : "file.size() = " + file.size() + ", index.size() = " + index.size(); final Iterator i = rows.iterator(); - while (i.hasNext()) { - put(i.next()); - } + while (i.hasNext()) put(i.next()); assert file.size() == index.size() + fail : "file.size() = " + file.size() + ", index.size() = " + index.size(); } diff --git a/source/de/anomic/kelondro/table/FlexTable.java b/source/de/anomic/kelondro/table/FlexTable.java index 2279649df..fae3cbce3 100644 --- a/source/de/anomic/kelondro/table/FlexTable.java +++ b/source/de/anomic/kelondro/table/FlexTable.java @@ -198,7 +198,7 @@ public class FlexTable extends FlexWidthArray implements ObjectIndex { //} } - public synchronized void putMultiple(final List rows) throws IOException { + public synchronized void put(final List rows) throws IOException { // put a list of entries in a ordered way. // this should save R/W head positioning time final Iterator i = rows.iterator(); @@ -222,16 +222,16 @@ public class FlexTable extends FlexWidthArray implements ObjectIndex { super.setMultiple(old_rows_ordered); // write new entries to index - addUniqueMultiple(new_rows_sequential); + addUnique(new_rows_sequential); assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); } public synchronized Row.Entry put(final Row.Entry row, final Date entryDate) throws IOException { assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); - return put(row); + return replace(row); } - public synchronized Row.Entry put(final Row.Entry row) throws IOException { + public synchronized Row.Entry replace(final Row.Entry row) throws IOException { assert (row != null); assert (!(Log.allZero(row.getColBytes(0)))); assert row.objectsize() <= this.rowdef.objectsize; @@ -265,13 +265,32 @@ public class FlexTable extends FlexWidthArray implements ObjectIndex { return oldentry; } + public synchronized void put(final Row.Entry row) throws IOException { + assert (row != null); + assert (!(Log.allZero(row.getColBytes(0)))); + assert row.objectsize() <= this.rowdef.objectsize; + final byte[] key = row.getColBytes(0); + if (index == null) return; // case may appear during shutdown + int pos = index.get(key); + if (pos < 0) { + pos = super.add(row); + index.put(key, pos); + assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); + return; + } + //System.out.println("row.key=" + serverLog.arrayList(row.bytes(), 0, row.objectsize())); + assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); + super.set(pos, row); + assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); + } + public synchronized void addUnique(final Row.Entry row) throws IOException { assert row.objectsize() == this.rowdef.objectsize; assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); index.putUnique(row.getColBytes(0), super.add(row)); } - public synchronized void addUniqueMultiple(final List rows) throws IOException { + public synchronized void addUnique(final List rows) throws IOException { // add a list of entries in a ordered way. // this should save R/W head positioning time final TreeMap indexed_result = super.addMultiple(rows); diff --git a/source/de/anomic/kelondro/table/Relations.java b/source/de/anomic/kelondro/table/Relations.java index 40fe7260f..ab2ad1b2d 100755 --- a/source/de/anomic/kelondro/table/Relations.java +++ b/source/de/anomic/kelondro/table/Relations.java @@ -124,7 +124,7 @@ public class Relations { entry.setCol(1, System.currentTimeMillis()); entry.setCol(2, 1000000); entry.setCol(3, value); - final Row.Entry oldentry = table.put(entry); + final Row.Entry oldentry = table.replace(entry); if (oldentry == null) return null; return oldentry.getColBytes(3); } diff --git a/source/de/anomic/kelondro/table/SQLTable.java b/source/de/anomic/kelondro/table/SQLTable.java index 649b9a773..dd2f2deb0 100644 --- a/source/de/anomic/kelondro/table/SQLTable.java +++ b/source/de/anomic/kelondro/table/SQLTable.java @@ -175,26 +175,40 @@ public class SQLTable implements ObjectIndex { } } - public synchronized void putMultiple(final List rows) throws IOException { + public synchronized void put(final List rows) throws IOException { final Iterator i = rows.iterator(); while (i.hasNext()) put(i.next()); } - public Row.Entry put(final Row.Entry row, final Date entryDate) throws IOException { - return put(row); - } - - public Row.Entry put(final Row.Entry row) throws IOException { + public Row.Entry replace(final Row.Entry row) throws IOException { try { + final Row.Entry oldEntry = remove(row.getColBytes(0)); + final String sqlQuery = "INSERT INTO test (" + + "hash, " + + "value) " + + "VALUES (?,?)"; - final Row.Entry oldEntry = remove(row.getColBytes(0)); + final PreparedStatement sqlStatement = this.theDBConnection.prepareStatement(sqlQuery); + sqlStatement.setString(1, row.getColString(0, null)); + sqlStatement.setBytes(2, row.bytes()); + sqlStatement.execute(); + + sqlStatement.close(); + + return oldEntry; + } catch (final Exception e) { + throw new IOException(e.getMessage()); + } + } + + public void put(final Row.Entry row) throws IOException { + try { final String sqlQuery = "INSERT INTO test (" + "hash, " + "value) " + "VALUES (?,?)"; - final PreparedStatement sqlStatement = this.theDBConnection.prepareStatement(sqlQuery); sqlStatement.setString(1, row.getColString(0, null)); @@ -203,7 +217,7 @@ public class SQLTable implements ObjectIndex { sqlStatement.close(); - return oldEntry; + return; } catch (final Exception e) { throw new IOException(e.getMessage()); } @@ -217,7 +231,7 @@ public class SQLTable implements ObjectIndex { throw new UnsupportedOperationException(); } - public synchronized void addUniqueMultiple(final List rows) throws IOException { + public synchronized void addUnique(final List rows) throws IOException { throw new UnsupportedOperationException(); } diff --git a/source/de/anomic/kelondro/table/SplitTable.java b/source/de/anomic/kelondro/table/SplitTable.java index 0390c360b..fdc072cec 100644 --- a/source/de/anomic/kelondro/table/SplitTable.java +++ b/source/de/anomic/kelondro/table/SplitTable.java @@ -222,19 +222,15 @@ public class SplitTable implements ObjectIndex { return keeper.get(key); } - public synchronized void putMultiple(final List rows) throws IOException { + public synchronized void put(final List rows) throws IOException { throw new UnsupportedOperationException("not yet implemented"); } - public synchronized Row.Entry put(final Row.Entry row) throws IOException { - return put(row, null); // entry for current date - } - - public synchronized Row.Entry put(final Row.Entry row, Date entryDate) throws IOException { + public synchronized Row.Entry replace(final Row.Entry row) throws IOException { assert row.objectsize() <= this.rowdef.objectsize; final ObjectIndex keeper = keeperOf(row.getColBytes(0)); - if (keeper != null) return keeper.put(row); - if ((entryDate == null) || (entryDate.after(new Date()))) entryDate = new Date(); // fix date + if (keeper != null) return keeper.replace(row); + Date entryDate = new Date(); final String suffix = dateSuffix(entryDate); if (suffix == null) return null; ObjectIndex table = tables.get(suffix); @@ -257,6 +253,32 @@ public class SplitTable implements ObjectIndex { return null; } + public synchronized void put(final Row.Entry row) throws IOException { + assert row.objectsize() <= this.rowdef.objectsize; + final ObjectIndex keeper = keeperOf(row.getColBytes(0)); + if (keeper != null) {keeper.put(row); return;} + Date entryDate = new Date(); + final String suffix = dateSuffix(entryDate); + if (suffix == null) return; + ObjectIndex table = tables.get(suffix); + if (table == null) { + // open table + final File f = new File(path, tablename + "." + suffix); + if (f.exists()) { + if (f.isDirectory()) { + FlexTable.delete(path, tablename + "." + suffix); + } + // open a eco table + table = new EcoTable(f, rowdef, EcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0); + } else { + // make new table + table = new EcoTable(f, rowdef, EcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0); + } + tables.put(suffix, table); + } + table.put(row); + } + public synchronized ObjectIndex keeperOf(final byte[] key) { // because the index is stored only in one table, // and the index is completely in RAM, a concurrency will create @@ -322,7 +344,7 @@ public class SplitTable implements ObjectIndex { table.addUnique(row); } - public synchronized void addUniqueMultiple(final List rows) throws IOException { + public synchronized void addUnique(final List rows) throws IOException { final Iterator i = rows.iterator(); while (i.hasNext()) addUnique(i.next()); } diff --git a/source/de/anomic/kelondro/table/Tree.java b/source/de/anomic/kelondro/table/Tree.java index e8ee32db6..9592a3a0b 100644 --- a/source/de/anomic/kelondro/table/Tree.java +++ b/source/de/anomic/kelondro/table/Tree.java @@ -33,7 +33,6 @@ import java.io.IOException; import java.io.RandomAccessFile; import java.util.ArrayList; import java.util.Comparator; -import java.util.Date; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; @@ -313,16 +312,16 @@ public class Tree extends CachedRecords implements ObjectIndex { return (lc.equals(childn.handle())); } - public synchronized void putMultiple(final List rows) throws IOException { + public synchronized void put(final List rows) throws IOException { final Iterator i = rows.iterator(); while (i.hasNext()) put(i.next()); } - public Row.Entry put(final Row.Entry row, final Date entryDate) throws IOException { - return put(row); + public void put(final Row.Entry newrow) throws IOException { + replace(newrow); } - public Row.Entry put(final Row.Entry newrow) throws IOException { + public Row.Entry replace(final Row.Entry newrow) throws IOException { assert (newrow != null); assert (newrow.columns() == row().columns()); assert (!(Log.allZero(newrow.getPrimaryKeyBytes()))); @@ -468,11 +467,7 @@ public class Tree extends CachedRecords implements ObjectIndex { this.put(row); } - public synchronized void addUnique(final Row.Entry row, final Date entryDate) throws IOException { - this.put(row, entryDate); - } - - public synchronized void addUniqueMultiple(final List rows) throws IOException { + public synchronized void addUnique(final List rows) throws IOException { final Iterator i = rows.iterator(); while (i.hasNext()) addUnique(i.next()); } @@ -596,7 +591,7 @@ public class Tree extends CachedRecords implements ObjectIndex { // Associates the specified value with the specified key in this map public byte[] put(final byte[] key, final byte[] value) throws IOException { final Row.Entry row = row().newEntry(new byte[][]{key, value}); - final Row.Entry ret = put(row); + final Row.Entry ret = replace(row); if (ret == null) return null; return ret.getColBytes(0); } diff --git a/source/de/anomic/kelondro/text/ReferenceContainer.java b/source/de/anomic/kelondro/text/ReferenceContainer.java index a4913a541..875f47f5c 100644 --- a/source/de/anomic/kelondro/text/ReferenceContainer.java +++ b/source/de/anomic/kelondro/text/ReferenceContainer.java @@ -111,7 +111,7 @@ public class ReferenceContainer extends RowSet { public Reference put(final ReferenceRow entry) { assert entry.toKelondroEntry().objectsize() == super.rowdef.objectsize; - final Row.Entry r = super.put(entry.toKelondroEntry()); + final Row.Entry r = super.replace(entry.toKelondroEntry()); if (r == null) return null; return new ReferenceRow(r); } @@ -119,13 +119,13 @@ public class ReferenceContainer extends RowSet { public boolean putRecent(final ReferenceRow entry) { assert entry.toKelondroEntry().objectsize() == super.rowdef.objectsize; // returns true if the new entry was added, false if it already existed - final Row.Entry oldEntryRow = this.put(entry.toKelondroEntry()); + final Row.Entry oldEntryRow = this.replace(entry.toKelondroEntry()); if (oldEntryRow == null) { return true; } final ReferenceRow oldEntry = new ReferenceRow(oldEntryRow); if (entry.isOlder(oldEntry)) { // A more recent Entry is already in this container - this.put(oldEntry.toKelondroEntry()); // put it back + this.replace(oldEntry.toKelondroEntry()); // put it back return false; } return true; diff --git a/source/de/anomic/yacy/yacyNewsDB.java b/source/de/anomic/yacy/yacyNewsDB.java index 6ef23444e..056e23b27 100644 --- a/source/de/anomic/yacy/yacyNewsDB.java +++ b/source/de/anomic/yacy/yacyNewsDB.java @@ -93,10 +93,10 @@ public class yacyNewsDB { public synchronized yacyNewsRecord put(final yacyNewsRecord record) throws IOException { try { - return b2r(news.put(r2b(record))); + return b2r(news.replace(r2b(record))); } catch (final kelondroException e) { resetDB(); - return b2r(news.put(r2b(record))); + return b2r(news.replace(r2b(record))); } }