boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added stub for another performance hack: concurrent indexes

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5699 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent 3e4c28e188
commit 1f1be1518c
5 changed files with 223 additions and 15 deletions
Show Stats Download Patch File Download Diff File

11
source/de/anomic/kelondro/blob/Cache.java
Unescape View File

@ -273,11 +273,6 @@ public class Cache implements ObjectIndex {
while (i.hasNext()) put(i.next());
}
public synchronized void putMultiple(final List<Entry> rows, final Date entryDate) throws IOException {
final Iterator<Entry> i = rows.iterator();
while (i.hasNext()) put(i.next(), entryDate);
}
public synchronized Entry put(final Entry row) throws IOException {
assert (row != null);
assert (row.columns() == row().columns());
@ -324,12 +319,6 @@ public class Cache implements ObjectIndex {
return entry;
}
public synchronized Entry put(final Entry row, final Date entryDate) throws IOException {
// a put with a date is bad for the cache: the date cannot be handled
// we omit the date here and use the current Date everywhere
return this.put(row);
}
public synchronized void addUnique(final Entry row) throws IOException {
assert (row != null);
assert (row.columns() == row().columns());

2
source/de/anomic/kelondro/index/ObjectIndex.java
Unescape View File

@ -33,7 +33,6 @@ package de.anomic.kelondro.index;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import de.anomic.kelondro.order.CloneableIterator;
@ -46,7 +45,6 @@ public interface ObjectIndex {
public boolean has(byte[] key); // use this only if there is no get in case that has returns true
public Row.Entry get(byte[] key) throws IOException;
public Row.Entry put(Row.Entry row) throws IOException;
public Row.Entry put(Row.Entry row, Date entryDate) throws IOException;
public void putMultiple(List<Row.Entry> rows) throws IOException; // for R/W head path optimization
public void addUnique(Row.Entry row) throws IOException; // no double-check
public void addUniqueMultiple(List<Row.Entry> rows) throws IOException; // no double-check

218
source/de/anomic/kelondro/index/ObjectIndexDaemon.java
Unescape View File

@ -0,0 +1,218 @@
// ObjectIndexDaemon.java
// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 11.03.2009 on http://yacy.net
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.kelondro.index;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import de.anomic.kelondro.index.Row.Entry;
import de.anomic.kelondro.index.Row.Queue;
import de.anomic.kelondro.order.CloneableIterator;
public class ObjectIndexDaemon implements ObjectIndex {
private Row.Entry poison;
private PutScheduler putScheduler;
private RowSet index;
private Queue queue;
private int queueCount;
public ObjectIndexDaemon(final Row rowdef, final int objectCount, final int queueCount) {
this.index = new RowSet(rowdef, objectCount);
assert rowdef.objectOrder != null;
this.poison = rowdef.newEntry();
this.queueCount = queueCount;
this.queue = index.rowdef.newQueue(queueCount);
this.putScheduler = new PutScheduler();
this.putScheduler.start();
}
public class PutScheduler extends Thread {
public PutScheduler() {}
public void run() {
Row.Entry next;
try {
while ((next = queue.take()) != poison) index.put(next);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
public void close() {
try {
this.queue.put(poison);
this.putScheduler.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
public void addUnique(Entry row) throws IOException {
index.addUnique(row);
}
public void addUniqueMultiple(List<Entry> rows) throws IOException {
index.addUniqueMultiple(rows);
}
public void clear() throws IOException {
try {
this.queue.put(poison);
this.putScheduler.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
this.index.clear();
this.queue = index.rowdef.newQueue(queueCount);
this.putScheduler = new PutScheduler();
this.putScheduler.start();
}
public void deleteOnExit() {
this.index.deleteOnExit();
}
public String filename() {
return this.index.filename();
}
public Entry get(byte[] key) throws IOException {
Entry entry = this.queue.get(key);
if (entry != null) return entry;
return this.index.get(key);
}
public boolean has(byte[] key) {
Entry entry = this.queue.get(key);
if (entry != null) return true;
return this.index.has(key);
}
public Entry put(Entry row) throws IOException {
Entry entry = get(row.getPrimaryKeyBytes());
try {
this.queue.put(row);
} catch (InterruptedException e) {
this.index.put(row);
}
return entry;
}
public void putMultiple(List<Entry> rows) throws IOException {
for (Entry entry: rows) try {
this.queue.put(entry);
} catch (InterruptedException e) {
this.index.put(entry);
}
}
public Entry remove(byte[] key) throws IOException {
Entry entry = this.queue.delete(key);
if (entry == null) return this.index.remove(key);
this.index.remove(key);
return entry;
}
public synchronized ArrayList<RowCollection> removeDoubles() throws IOException {
try {
this.queue.put(poison);
this.putScheduler.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
ArrayList<RowCollection> d = index.removeDoubles();
this.queue = index.rowdef.newQueue(queueCount);
this.putScheduler = new PutScheduler();
this.putScheduler.start();
return d;
}
public synchronized Entry removeOne() throws IOException {
try {
this.queue.put(poison);
this.putScheduler.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
Entry d = index.removeOne();
this.queue = index.rowdef.newQueue(queueCount);
this.putScheduler = new PutScheduler();
this.putScheduler.start();
return d;
}
public Row row() {
return this.index.row();
}
public synchronized CloneableIterator<byte[]> keys(boolean up, byte[] firstKey) throws IOException {
try {
this.queue.put(poison);
this.putScheduler.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
CloneableIterator<byte[]> keys = index.keys(up, firstKey);
this.queue = index.rowdef.newQueue(queueCount);
this.putScheduler = new PutScheduler();
this.putScheduler.start();
return keys;
}
public synchronized CloneableIterator<Entry> rows(boolean up, byte[] firstKey) throws IOException {
try {
this.queue.put(poison);
this.putScheduler.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
CloneableIterator<Entry> rows = index.rows(up, firstKey);
this.queue = index.rowdef.newQueue(queueCount);
this.putScheduler = new PutScheduler();
this.putScheduler.start();
return rows;
}
public CloneableIterator<Entry> rows() throws IOException {
return rows(true, null);
}
public synchronized int size() {
try {
this.queue.put(poison);
this.putScheduler.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
int size = index.size();
this.queue = index.rowdef.newQueue(queueCount);
this.putScheduler = new PutScheduler();
this.putScheduler.start();
return size;
}
}

4
source/de/anomic/kelondro/index/Row.java
Unescape View File

@ -636,6 +636,10 @@ public final class Row {
}
}
public Queue newQueue(int maxsize) {
return new Queue(maxsize);
}
public final class Queue {
private final ArrayBlockingQueue<Entry> queue;

3
source/de/anomic/kelondro/text/MetadataRepository.java
Unescape View File

@ -32,7 +32,6 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@ -134,7 +133,7 @@ public final class MetadataRepository implements Iterable<byte[]> {
return; // this did not need to be stored, but is updated
}
urlIndexFile.put(entry.toRowEntry(), new Date() /*entry.loaddate()*/);
urlIndexFile.put(entry.toRowEntry());
statsDump = null;
}

Write Preview
Loading…
Cancel
Save