From 1f1be1518c287cf9dd00609db2f37d38cf9d26ef Mon Sep 17 00:00:00 2001 From: orbiter Date: Wed, 11 Mar 2009 15:52:03 +0000 Subject: [PATCH] added stub for another performance hack: concurrent indexes git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5699 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/de/anomic/kelondro/blob/Cache.java | 11 - .../de/anomic/kelondro/index/ObjectIndex.java | 2 - .../kelondro/index/ObjectIndexDaemon.java | 218 ++++++++++++++++++ source/de/anomic/kelondro/index/Row.java | 4 + .../kelondro/text/MetadataRepository.java | 3 +- 5 files changed, 223 insertions(+), 15 deletions(-) create mode 100644 source/de/anomic/kelondro/index/ObjectIndexDaemon.java diff --git a/source/de/anomic/kelondro/blob/Cache.java b/source/de/anomic/kelondro/blob/Cache.java index cccd4ad4b..841f6f1c9 100644 --- a/source/de/anomic/kelondro/blob/Cache.java +++ b/source/de/anomic/kelondro/blob/Cache.java @@ -273,11 +273,6 @@ public class Cache implements ObjectIndex { while (i.hasNext()) put(i.next()); } - public synchronized void putMultiple(final List rows, final Date entryDate) throws IOException { - final Iterator i = rows.iterator(); - while (i.hasNext()) put(i.next(), entryDate); - } - public synchronized Entry put(final Entry row) throws IOException { assert (row != null); assert (row.columns() == row().columns()); @@ -324,12 +319,6 @@ public class Cache implements ObjectIndex { return entry; } - public synchronized Entry put(final Entry row, final Date entryDate) throws IOException { - // a put with a date is bad for the cache: the date cannot be handled - // we omit the date here and use the current Date everywhere - return this.put(row); - } - public synchronized void addUnique(final Entry row) throws IOException { assert (row != null); assert (row.columns() == row().columns()); diff --git a/source/de/anomic/kelondro/index/ObjectIndex.java b/source/de/anomic/kelondro/index/ObjectIndex.java index b1ec8b0ba..28fc75dce 100644 --- a/source/de/anomic/kelondro/index/ObjectIndex.java +++ b/source/de/anomic/kelondro/index/ObjectIndex.java @@ -33,7 +33,6 @@ package de.anomic.kelondro.index; import java.io.IOException; import java.util.ArrayList; -import java.util.Date; import java.util.List; import de.anomic.kelondro.order.CloneableIterator; @@ -46,7 +45,6 @@ public interface ObjectIndex { public boolean has(byte[] key); // use this only if there is no get in case that has returns true public Row.Entry get(byte[] key) throws IOException; public Row.Entry put(Row.Entry row) throws IOException; - public Row.Entry put(Row.Entry row, Date entryDate) throws IOException; public void putMultiple(List rows) throws IOException; // for R/W head path optimization public void addUnique(Row.Entry row) throws IOException; // no double-check public void addUniqueMultiple(List rows) throws IOException; // no double-check diff --git a/source/de/anomic/kelondro/index/ObjectIndexDaemon.java b/source/de/anomic/kelondro/index/ObjectIndexDaemon.java new file mode 100644 index 000000000..39d8cffd0 --- /dev/null +++ b/source/de/anomic/kelondro/index/ObjectIndexDaemon.java @@ -0,0 +1,218 @@ +// ObjectIndexDaemon.java +// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany +// first published 11.03.2009 on http://yacy.net +// +// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ +// $LastChangedRevision: 1986 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package de.anomic.kelondro.index; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import de.anomic.kelondro.index.Row.Entry; +import de.anomic.kelondro.index.Row.Queue; +import de.anomic.kelondro.order.CloneableIterator; + +public class ObjectIndexDaemon implements ObjectIndex { + + private Row.Entry poison; + private PutScheduler putScheduler; + private RowSet index; + private Queue queue; + private int queueCount; + + public ObjectIndexDaemon(final Row rowdef, final int objectCount, final int queueCount) { + this.index = new RowSet(rowdef, objectCount); + assert rowdef.objectOrder != null; + this.poison = rowdef.newEntry(); + this.queueCount = queueCount; + this.queue = index.rowdef.newQueue(queueCount); + this.putScheduler = new PutScheduler(); + this.putScheduler.start(); + } + + public class PutScheduler extends Thread { + public PutScheduler() {} + public void run() { + Row.Entry next; + try { + while ((next = queue.take()) != poison) index.put(next); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + public void close() { + try { + this.queue.put(poison); + this.putScheduler.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + public void addUnique(Entry row) throws IOException { + index.addUnique(row); + } + + public void addUniqueMultiple(List rows) throws IOException { + index.addUniqueMultiple(rows); + } + + public void clear() throws IOException { + try { + this.queue.put(poison); + this.putScheduler.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + this.index.clear(); + this.queue = index.rowdef.newQueue(queueCount); + this.putScheduler = new PutScheduler(); + this.putScheduler.start(); + } + + public void deleteOnExit() { + this.index.deleteOnExit(); + } + + public String filename() { + return this.index.filename(); + } + + public Entry get(byte[] key) throws IOException { + Entry entry = this.queue.get(key); + if (entry != null) return entry; + return this.index.get(key); + } + + public boolean has(byte[] key) { + Entry entry = this.queue.get(key); + if (entry != null) return true; + return this.index.has(key); + } + + public Entry put(Entry row) throws IOException { + Entry entry = get(row.getPrimaryKeyBytes()); + try { + this.queue.put(row); + } catch (InterruptedException e) { + this.index.put(row); + } + return entry; + } + + public void putMultiple(List rows) throws IOException { + for (Entry entry: rows) try { + this.queue.put(entry); + } catch (InterruptedException e) { + this.index.put(entry); + } + } + + public Entry remove(byte[] key) throws IOException { + Entry entry = this.queue.delete(key); + if (entry == null) return this.index.remove(key); + this.index.remove(key); + return entry; + } + + public synchronized ArrayList removeDoubles() throws IOException { + try { + this.queue.put(poison); + this.putScheduler.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + ArrayList d = index.removeDoubles(); + this.queue = index.rowdef.newQueue(queueCount); + this.putScheduler = new PutScheduler(); + this.putScheduler.start(); + return d; + } + + public synchronized Entry removeOne() throws IOException { + try { + this.queue.put(poison); + this.putScheduler.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + Entry d = index.removeOne(); + this.queue = index.rowdef.newQueue(queueCount); + this.putScheduler = new PutScheduler(); + this.putScheduler.start(); + return d; + } + + public Row row() { + return this.index.row(); + } + + public synchronized CloneableIterator keys(boolean up, byte[] firstKey) throws IOException { + try { + this.queue.put(poison); + this.putScheduler.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + CloneableIterator keys = index.keys(up, firstKey); + this.queue = index.rowdef.newQueue(queueCount); + this.putScheduler = new PutScheduler(); + this.putScheduler.start(); + return keys; + } + + public synchronized CloneableIterator rows(boolean up, byte[] firstKey) throws IOException { + try { + this.queue.put(poison); + this.putScheduler.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + CloneableIterator rows = index.rows(up, firstKey); + this.queue = index.rowdef.newQueue(queueCount); + this.putScheduler = new PutScheduler(); + this.putScheduler.start(); + return rows; + } + + public CloneableIterator rows() throws IOException { + return rows(true, null); + } + + public synchronized int size() { + try { + this.queue.put(poison); + this.putScheduler.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + int size = index.size(); + this.queue = index.rowdef.newQueue(queueCount); + this.putScheduler = new PutScheduler(); + this.putScheduler.start(); + return size; + } + +} diff --git a/source/de/anomic/kelondro/index/Row.java b/source/de/anomic/kelondro/index/Row.java index 28b8d704b..a5f230746 100644 --- a/source/de/anomic/kelondro/index/Row.java +++ b/source/de/anomic/kelondro/index/Row.java @@ -636,6 +636,10 @@ public final class Row { } } + public Queue newQueue(int maxsize) { + return new Queue(maxsize); + } + public final class Queue { private final ArrayBlockingQueue queue; diff --git a/source/de/anomic/kelondro/text/MetadataRepository.java b/source/de/anomic/kelondro/text/MetadataRepository.java index 75b22c19a..fc8baa9a4 100644 --- a/source/de/anomic/kelondro/text/MetadataRepository.java +++ b/source/de/anomic/kelondro/text/MetadataRepository.java @@ -32,7 +32,6 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintWriter; import java.util.ArrayList; -import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -134,7 +133,7 @@ public final class MetadataRepository implements Iterable { return; // this did not need to be stored, but is updated } - urlIndexFile.put(entry.toRowEntry(), new Date() /*entry.loaddate()*/); + urlIndexFile.put(entry.toRowEntry()); statsDump = null; }