boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- shifted some computation out of synchronization to allow more concurrency

Browse Source 
- removed synchronization where not necessary

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6814 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent f204076d25
commit dde394a977
4 changed files with 206 additions and 185 deletions
Show Stats Download Patch File Download Diff File

10
source/de/anomic/search/MetadataRepository.java
Unescape View File

@ -109,13 +109,13 @@ public final class MetadataRepository implements Iterable<byte[]> {
}
}
public synchronized int writeCacheSize() {
public int writeCacheSize() {
if (urlIndexFile instanceof SplitTable) return ((SplitTable) urlIndexFile).writeBufferSize();
if (urlIndexFile instanceof Cache) return ((Cache) urlIndexFile).writeBufferSize();
return 0;
}
public synchronized URIMetadataRow load(final byte[] urlHash, final WordReferenceVars searchedWord, final long ranking) {
public URIMetadataRow load(final byte[] urlHash, final WordReferenceVars searchedWord, final long ranking) {
// generates an plasmaLURLEntry using the url hash
// if the url cannot be found, this returns null
if (urlHash == null) return null;
@ -129,9 +129,10 @@ public final class MetadataRepository implements Iterable<byte[]> {
}
}
public synchronized void store(final URIMetadataRow entry) throws IOException {
public void store(final URIMetadataRow entry) throws IOException {
// Check if there is a more recent Entry already in the DB
URIMetadataRow oldEntry;
synchronized (this) {
try {
Row.Entry oe = (urlIndexFile == null) ? null : urlIndexFile.get(entry.hash());
oldEntry = (oe == null) ? null : new URIMetadataRow(oe, null, 0);
@ -153,10 +154,11 @@ public final class MetadataRepository implements Iterable<byte[]> {
} catch (RowSpaceExceededException e) {
throw new IOException("RowSpaceExceededException in " + this.urlIndexFile.filename() + ": " + e.getMessage());
}
}
statsDump = null;
}
public synchronized boolean remove(final byte[] urlHashBytes) {
public boolean remove(final byte[] urlHashBytes) {
if (urlHashBytes == null) return false;
try {
final Row.Entry r = urlIndexFile.remove(urlHashBytes);

49
source/net/yacy/kelondro/blob/Heap.java
Unescape View File

@ -31,6 +31,7 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.io.AbstractWriter;
@ -42,7 +43,7 @@ import net.yacy.kelondro.order.NaturalOrder;
public final class Heap extends HeapModifier implements BLOB {
private HashMap<String, byte[]> buffer; // a write buffer to limit IO to the file; attention: Maps cannot use byte[] as key
private TreeMap<byte[], byte[]> buffer; // a write buffer to limit IO to the file
private int buffersize; // bytes that are buffered in buffer
private final int buffermax; // maximum size of the buffer
@ -83,7 +84,7 @@ public final class Heap extends HeapModifier implements BLOB {
int buffermax) throws IOException {
super(heapFile, keylength, ordering);
this.buffermax = buffermax;
this.buffer = new HashMap<String, byte[]>();
this.buffer = new TreeMap<byte[], byte[]>(ordering);
this.buffersize = 0;
/*
// DEBUG
@ -119,13 +120,15 @@ public final class Heap extends HeapModifier implements BLOB {
* @return true if the key exists, false otherwise
*/
@Override
public synchronized boolean has(byte[] key) {
public boolean has(byte[] key) {
assert index != null;
key = normalizeKey(key);
synchronized (this) {
// check the buffer
if (this.buffer.containsKey(new String(key))) return true;
if (this.buffer.containsKey(key)) return true;
return super.has(key);
}
}
/**
* add a BLOB to the heap: this adds the blob always to the end of the file
@ -138,7 +141,6 @@ public final class Heap extends HeapModifier implements BLOB {
assert blob.length > 0;
if ((blob == null) || (blob.length == 0)) return;
final int pos = (int) file.length();
key = normalizeKey(key);
index.put(key, pos);
file.seek(pos);
file.writeInt(this.keylength + blob.length);
@ -154,7 +156,7 @@ public final class Heap extends HeapModifier implements BLOB {
*/
private void flushBuffer() throws IOException, RowSpaceExceededException {
// check size of buffer
Iterator<Map.Entry<String, byte[]>> i = this.buffer.entrySet().iterator();
Iterator<Map.Entry<byte[], byte[]>> i = this.buffer.entrySet().iterator();
int l = 0;
while (i.hasNext()) l += i.next().getValue().length;
assert l == this.buffersize;
@ -162,11 +164,11 @@ public final class Heap extends HeapModifier implements BLOB {
// simulate write: this whole code block is only here to test the assert at the end of the block; remove after testing
i = this.buffer.entrySet().iterator();
int posBuffer = 0;
Map.Entry<String, byte[]> entry;
Map.Entry<byte[], byte[]> entry;
byte[] key, blob;
while (i.hasNext()) {
entry = i.next();
key = normalizeKey(entry.getKey().getBytes());
key = normalizeKey(entry.getKey());
blob = entry.getValue();
posBuffer += 4 + this.keylength + blob.length;
}
@ -181,7 +183,7 @@ public final class Heap extends HeapModifier implements BLOB {
byte[] b;
while (i.hasNext()) {
entry = i.next();
key = normalizeKey(entry.getKey().getBytes());
key = normalizeKey(entry.getKey());
blob = entry.getValue();
index.put(key, posFile);
b = AbstractWriter.int2array(this.keylength + blob.length);
@ -211,15 +213,17 @@ public final class Heap extends HeapModifier implements BLOB {
* @throws IOException
*/
@Override
public synchronized byte[] get(byte[] key) throws IOException {
public byte[] get(byte[] key) throws IOException {
key = normalizeKey(key);
synchronized (this) {
// check the buffer
byte[] blob = this.buffer.get(new String(key));
byte[] blob = this.buffer.get(key);
if (blob != null) return blob;
return super.get(key);
}
}
/**
* retrieve the size of the BLOB
@ -228,14 +232,17 @@ public final class Heap extends HeapModifier implements BLOB {
* @throws IOException
*/
@Override
public synchronized long length(byte[] key) throws IOException {
public long length(byte[] key) throws IOException {
key = normalizeKey(key);
synchronized (this) {
// check the buffer
byte[] blob = this.buffer.get(new String(key));
byte[] blob = this.buffer.get(key);
if (blob != null) return blob.length;
return super.length(key);
}
}
/**
* clears the content of the database
@ -289,12 +296,13 @@ public final class Heap extends HeapModifier implements BLOB {
* @throws RowSpaceExceededException
*/
@Override
public synchronized void put(byte[] key, final byte[] b) throws IOException, RowSpaceExceededException {
public void put(byte[] key, final byte[] b) throws IOException, RowSpaceExceededException {
key = normalizeKey(key);
// we do not write records of length 0 into the BLOB
if (b.length == 0) return;
synchronized (this) {
// first remove the old entry (removes from buffer and file)
// TODO: this can be enhanced!
this.remove(key);
@ -310,20 +318,19 @@ public final class Heap extends HeapModifier implements BLOB {
if (b.length > buffermax) {
this.add(key, b);
} else {
this.buffer.put(new String(key), b);
this.buffer.put(key, b);
this.buffersize += b.length;
}
return;
}
// add entry to buffer
this.buffer.put(new String(key), b);
this.buffer.put(key, b);
this.buffersize += b.length;
}
}
private boolean putToGap(byte[] key, final byte[] b) throws IOException, RowSpaceExceededException {
key = normalizeKey(key);
// we do not write records of length 0 into the BLOB
if (b.length == 0) return true;
@ -415,11 +422,12 @@ public final class Heap extends HeapModifier implements BLOB {
* @throws IOException
*/
@Override
public synchronized void remove(byte[] key) throws IOException {
public void remove(byte[] key) throws IOException {
key = normalizeKey(key);
synchronized (this) {
// check the buffer
byte[] blob = this.buffer.remove(new String(key));
byte[] blob = this.buffer.remove(key);
if (blob != null) {
this.buffersize -= blob.length;
return;
@ -427,6 +435,7 @@ public final class Heap extends HeapModifier implements BLOB {
super.remove(key);
}
}
/**
* iterator over all keys

8
source/net/yacy/kelondro/blob/HeapModifier.java
Unescape View File

@ -94,9 +94,10 @@ public class HeapModifier extends HeapReader implements BLOB {
* @param key the primary key
* @throws IOException
*/
public synchronized void remove(byte[] key) throws IOException {
public void remove(byte[] key) throws IOException {
key = normalizeKey(key);
synchronized (this) {
// check if the index contains the key
final long seek = index.get(key);
if (seek < 0) return;
@ -129,6 +130,7 @@ public class HeapModifier extends HeapReader implements BLOB {
tryMergeNextGaps(seek, size);
tryMergePreviousGap(seek);
}
}
private void tryMergePreviousGap(final long thisSeek) throws IOException {
// this is called after a record has been removed. That may cause that a new
@ -233,10 +235,11 @@ public class HeapModifier extends HeapReader implements BLOB {
throw new UnsupportedOperationException("put is not supported in BLOBHeapModifier");
}
public synchronized int replace(byte[] key, final Rewriter rewriter) throws IOException {
public int replace(byte[] key, final Rewriter rewriter) throws IOException {
key = normalizeKey(key);
assert key.length == this.keylength;
synchronized (this) {
// check if the index contains the key
final long pos = index.get(key);
if (pos < 0) return 0;
@ -294,5 +297,6 @@ public class HeapModifier extends HeapReader implements BLOB {
this.free.put(pos + 4 + blob.length + key.length, newfreereclen);
return reduction;
}
}
}

12
source/net/yacy/kelondro/blob/HeapReader.java
Unescape View File

@ -306,13 +306,15 @@ public class HeapReader {
* @param key
* @return true if the key exists, false otherwise
*/
public synchronized boolean has(byte[] key) {
public boolean has(byte[] key) {
assert index != null;
key = normalizeKey(key);
synchronized (this) {
// check if the file index contains the key
return index.get(key) >= 0;
}
}
public ByteOrder ordering() {
return this.ordering;
@ -372,9 +374,10 @@ public class HeapReader {
* @return
* @throws IOException
*/
public synchronized byte[] get(byte[] key) throws IOException {
public byte[] get(byte[] key) throws IOException {
key = normalizeKey(key);
synchronized (this) {
// check if the index contains the key
final long pos = index.get(key);
if (pos < 0) return null;
@ -404,6 +407,7 @@ public class HeapReader {
return blob;
}
}
protected boolean checkKey(byte[] key, final long pos) throws IOException {
key = normalizeKey(key);
@ -422,9 +426,10 @@ public class HeapReader {
* @return the size of the BLOB or -1 if the BLOB does not exist
* @throws IOException
*/
public synchronized long length(byte[] key) throws IOException {
public long length(byte[] key) throws IOException {
key = normalizeKey(key);
synchronized (this) {
// check if the index contains the key
final long pos = index.get(key);
if (pos < 0) return -1;
@ -433,6 +438,7 @@ public class HeapReader {
file.seek(pos);
return file.readInt() - index.row().primaryKeyLength;
}
}
/**
* close the BLOB table

Write Preview
Loading…
Cancel
Save