boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fix for "negative seek offset" error during extension of heap files.

Browse Source 
This would have always happend when a heap file exceeds 2GB.
should fix https://github.com/yacy/yacy_search_server/issues/372
pull/607/head
Michael Peter Christen 1 year ago
parent 9c8fb97985
commit 4a54b24703
1 changed files with 47 additions and 48 deletions
Show Stats Download Patch File Download Diff File

95
source/net/yacy/kelondro/blob/Heap.java
Unescape View File

@ -107,7 +107,7 @@ public final class Heap extends HeapModifier implements BLOB {
System.out.println("*** DEBUG - counted " + c + " BLOBs"); System.out.println("*** DEBUG - counted " + c + " BLOBs");
*/ */
} }
/** /**
* the number of BLOBs in the heap * the number of BLOBs in the heap
* @return the number of BLOBs in the heap * @return the number of BLOBs in the heap
@ -117,7 +117,6 @@ public final class Heap extends HeapModifier implements BLOB {
return super.size() + ((this.buffer == null) ? 0 : this.buffer.size()); return super.size() + ((this.buffer == null) ? 0 : this.buffer.size());
} }
/** /**
* test if a key is in the heap file. This does not need any IO, because it uses only the ram index * test if a key is in the heap file. This does not need any IO, because it uses only the ram index
* @param key * @param key
@ -136,7 +135,7 @@ public final class Heap extends HeapModifier implements BLOB {
return super.containsKey(key); return super.containsKey(key);
} }
} }
/** /**
* add a BLOB to the heap: this adds the blob always to the end of the file * add a BLOB to the heap: this adds the blob always to the end of the file
* @param key * @param key
@ -147,7 +146,7 @@ public final class Heap extends HeapModifier implements BLOB {
private void add(byte[] key, final byte[] blob) throws IOException { private void add(byte[] key, final byte[] blob) throws IOException {
assert blob.length > 0; assert blob.length > 0;
if ((blob == null) || (blob.length == 0)) return; if ((blob == null) || (blob.length == 0)) return;
final int pos = (int) this.file.length(); final long pos = this.file.length();
try { try {
this.index.put(key, pos); this.index.put(key, pos);
this.file.seek(pos); this.file.seek(pos);
@ -158,7 +157,7 @@ public final class Heap extends HeapModifier implements BLOB {
throw new IOException(e.getMessage()); // should never occur; throw new IOException(e.getMessage()); // should never occur;
} }
} }
/** /**
* flush the buffer completely * flush the buffer completely
* this is like adding all elements of the buffer, but it needs only one IO access * this is like adding all elements of the buffer, but it needs only one IO access
@ -167,13 +166,13 @@ public final class Heap extends HeapModifier implements BLOB {
*/ */
public void flushBuffer() throws IOException { public void flushBuffer() throws IOException {
if (this.buffer == null) return; if (this.buffer == null) return;
// check size of buffer // check size of buffer
Iterator<Map.Entry<byte[], byte[]>> i = this.buffer.entrySet().iterator(); Iterator<Map.Entry<byte[], byte[]>> i = this.buffer.entrySet().iterator();
int l = 0; int l = 0;
while (i.hasNext()) l += i.next().getValue().length; while (i.hasNext()) l += i.next().getValue().length;
assert l == this.buffersize; assert l == this.buffersize;
int posBuffer = 0; int posBuffer = 0;
Map.Entry<byte[], byte[]> entry; Map.Entry<byte[], byte[]> entry;
byte[] key, blob; byte[] key, blob;
@ -188,11 +187,11 @@ public final class Heap extends HeapModifier implements BLOB {
} }
assert l + (4 + this.keylength) * this.buffer.size() == posBuffer : "l = " + l + ", this.keylength = " + this.keylength + ", this.buffer.size() = " + this.buffer.size() + ", posBuffer = " + posBuffer; assert l + (4 + this.keylength) * this.buffer.size() == posBuffer : "l = " + l + ", this.keylength = " + this.keylength + ", this.buffer.size() = " + this.buffer.size() + ", posBuffer = " + posBuffer;
*/ */
synchronized (this) { synchronized (this) {
super.deleteFingerprint(); super.deleteFingerprint();
} }
// append all contents of the buffer into one byte[] // append all contents of the buffer into one byte[]
i = this.buffer.entrySet().iterator(); i = this.buffer.entrySet().iterator();
final long pos = this.file.length(); final long pos = this.file.length();
@ -231,7 +230,7 @@ public final class Heap extends HeapModifier implements BLOB {
this.buffer.putAll(nextBuffer); this.buffer.putAll(nextBuffer);
this.buffersize = 0; this.buffersize = 0;
} }
/** /**
* read a blob from the heap * read a blob from the heap
* @param key * @param key
@ -241,14 +240,14 @@ public final class Heap extends HeapModifier implements BLOB {
@Override @Override
public byte[] get(byte[] key) throws IOException, SpaceExceededException { public byte[] get(byte[] key) throws IOException, SpaceExceededException {
key = normalizeKey(key); key = normalizeKey(key);
synchronized (this) { synchronized (this) {
// check the buffer // check the buffer
if (this.buffer != null) { if (this.buffer != null) {
byte[] blob = this.buffer.get(key); byte[] blob = this.buffer.get(key);
if (blob != null) return blob; if (blob != null) return blob;
} }
return super.get(key); return super.get(key);
} }
} }
@ -269,11 +268,11 @@ public final class Heap extends HeapModifier implements BLOB {
byte[] blob = this.buffer.get(key); byte[] blob = this.buffer.get(key);
if (blob != null) return blob.length; if (blob != null) return blob.length;
} }
return super.length(key); return super.length(key);
} }
} }
/** /**
* clears the content of the database * clears the content of the database
* @throws IOException * @throws IOException
@ -283,7 +282,7 @@ public final class Heap extends HeapModifier implements BLOB {
ConcurrentLog.info("Heap", "clearing heap " + this.name()); ConcurrentLog.info("Heap", "clearing heap " + this.name());
assert this.buffer != null; assert this.buffer != null;
if (this.buffer == null) this.buffer = new TreeMap<byte[], byte[]>(this.ordering); if (this.buffer == null) this.buffer = new TreeMap<byte[], byte[]>(this.ordering);
this.buffer.clear(); this.buffer.clear();
this.buffersize = 0; this.buffersize = 0;
super.clear(); super.clear();
} }
@ -294,23 +293,23 @@ public final class Heap extends HeapModifier implements BLOB {
@Override @Override
public synchronized void close(final boolean writeIDX) { public synchronized void close(final boolean writeIDX) {
ConcurrentLog.info("Heap", "closing heap " + this.name()); ConcurrentLog.info("Heap", "closing heap " + this.name());
if (this.file != null && this.buffer != null) { if (this.file != null && this.buffer != null) {
try { try {
flushBuffer(); flushBuffer();
} catch (final IOException e) { } catch (final IOException e) {
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
} }
} }
this.buffer = null; this.buffer = null;
super.close(writeIDX); super.close(writeIDX);
assert this.file == null; assert this.file == null;
} }
@Override @Override
public synchronized void close() { public synchronized void close() {
this.close(true); this.close(true);
} }
public int getBuffermax() { public int getBuffermax() {
return this.buffermax; return this.buffermax;
} }
@ -326,22 +325,22 @@ public final class Heap extends HeapModifier implements BLOB {
@Override @Override
public void insert(byte[] key, final byte[] b) throws IOException { public void insert(byte[] key, final byte[] b) throws IOException {
key = normalizeKey(key); key = normalizeKey(key);
// we do not write records of length 0 into the BLOB // we do not write records of length 0 into the BLOB
if (b.length == 0) return; if (b.length == 0) return;
synchronized (this) { synchronized (this) {
// first remove the old entry (removes from buffer and file) // first remove the old entry (removes from buffer and file)
// TODO: this can be enhanced! // TODO: this can be enhanced!
this.delete(key); this.delete(key);
// then look if we can use a free entry // then look if we can use a free entry
try { try {
if (putToGap(key, b)) return; if (putToGap(key, b)) return;
} catch (final SpaceExceededException e) {} // too less space can be ignored, we have a second try } catch (final SpaceExceededException e) {} // too less space can be ignored, we have a second try
assert this.buffer != null; assert this.buffer != null;
// if there is not enough space in the buffer, flush all // if there is not enough space in the buffer, flush all
if (this.buffersize + b.length > this.buffermax || MemoryControl.shortStatus()) { if (this.buffersize + b.length > this.buffermax || MemoryControl.shortStatus()) {
// this is too big. Flush everything // this is too big. Flush everything
@ -357,7 +356,7 @@ public final class Heap extends HeapModifier implements BLOB {
} }
return; return;
} }
// add entry to buffer // add entry to buffer
if (this.buffer != null) { if (this.buffer != null) {
this.buffer.put(key, b); this.buffer.put(key, b);
@ -365,14 +364,14 @@ public final class Heap extends HeapModifier implements BLOB {
} }
} }
} }
private boolean putToGap(byte[] key, final byte[] b) throws IOException, SpaceExceededException { private boolean putToGap(byte[] key, final byte[] b) throws IOException, SpaceExceededException {
// we do not write records of length 0 into the BLOB // we do not write records of length 0 into the BLOB
if (b.length == 0) return true; if (b.length == 0) return true;
// then look if we can use a free entry // then look if we can use a free entry
if (this.free == null || this.free.isEmpty()) return false; if (this.free == null || this.free.isEmpty()) return false;
// find the largest entry // find the largest entry
long lseek = -1; long lseek = -1;
int lsize = 0; int lsize = 0;
@ -385,10 +384,10 @@ public final class Heap extends HeapModifier implements BLOB {
if (entry.getValue().intValue() == reclen) { if (entry.getValue().intValue() == reclen) {
// we found an entry that has exactly the size that we need! // we found an entry that has exactly the size that we need!
// we use that entry and stop looking for a larger entry // we use that entry and stop looking for a larger entry
// add the entry to the index // add the entry to the index
this.index.put(key, entry.getKey()); this.index.put(key, entry.getKey());
// write to file // write to file
this.file.seek(entry.getKey().longValue()); this.file.seek(entry.getKey().longValue());
final int reclenf = this.file.readInt(); final int reclenf = this.file.readInt();
@ -401,9 +400,9 @@ public final class Heap extends HeapModifier implements BLOB {
// remove the entry from the free list // remove the entry from the free list
i.remove(); i.remove();
//System.out.println("*** DEBUG BLOB: replaced-fit record at " + entry.seek + ", reclen=" + reclen + ", key=" + UTF8.String(key)); //System.out.println("*** DEBUG BLOB: replaced-fit record at " + entry.seek + ", reclen=" + reclen + ", key=" + UTF8.String(key));
// finished! // finished!
return true; return true;
} }
@ -416,14 +415,14 @@ public final class Heap extends HeapModifier implements BLOB {
if (acount > 100 || bcount > 10) break; // in case that we have really a lot break here if (acount > 100 || bcount > 10) break; // in case that we have really a lot break here
} }
} }
// check if the found entry is large enough // check if the found entry is large enough
if (lsize > reclen + 4) { if (lsize > reclen + 4) {
// split the free entry into two new entries // split the free entry into two new entries
// if would be sufficient if lsize = reclen + 4, but this would mean to create // if would be sufficient if lsize = reclen + 4, but this would mean to create
// an empty entry with zero next bytes for BLOB and key, which is not very good for the // an empty entry with zero next bytes for BLOB and key, which is not very good for the
// data structure in the file // data structure in the file
// write the new entry // write the new entry
this.file.seek(lseek); this.file.seek(lseek);
this.file.writeInt(reclen); this.file.writeInt(reclen);
@ -432,23 +431,23 @@ public final class Heap extends HeapModifier implements BLOB {
for (int j = 0; j < this.keylength - key.length; j++) this.file.write(HeapWriter.ZERO); for (int j = 0; j < this.keylength - key.length; j++) this.file.write(HeapWriter.ZERO);
} }
this.file.write(b); this.file.write(b);
// add the index to the new entry // add the index to the new entry
this.index.put(key, lseek); this.index.put(key, lseek);
// define the new empty entry // define the new empty entry
final int newfreereclen = lsize - reclen - 4; final int newfreereclen = lsize - reclen - 4;
assert newfreereclen > 0; assert newfreereclen > 0;
this.file.writeInt(newfreereclen); this.file.writeInt(newfreereclen);
// remove the old free entry // remove the old free entry
this.free.remove(lseek); this.free.remove(lseek);
// add a new free entry // add a new free entry
this.free.put(lseek + 4 + reclen, newfreereclen); this.free.put(lseek + 4 + reclen, newfreereclen);
//System.out.println("*** DEBUG BLOB: replaced-split record at " + lseek + ", reclen=" + reclen + ", new reclen=" + newfreereclen + ", key=" + UTF8.String(key)); //System.out.println("*** DEBUG BLOB: replaced-split record at " + lseek + ", reclen=" + reclen + ", new reclen=" + newfreereclen + ", key=" + UTF8.String(key));
// finished! // finished!
return true; return true;
} }
@ -464,10 +463,10 @@ public final class Heap extends HeapModifier implements BLOB {
@Override @Override
public void delete(byte[] key) throws IOException { public void delete(byte[] key) throws IOException {
key = normalizeKey(key); key = normalizeKey(key);
synchronized (this) { synchronized (this) {
super.deleteFingerprint(); super.deleteFingerprint();
// check the buffer // check the buffer
assert this.buffer != null; assert this.buffer != null;
if (this.buffer != null) { if (this.buffer != null) {
@ -477,11 +476,11 @@ public final class Heap extends HeapModifier implements BLOB {
return; return;
} }
} }
super.delete(key); super.delete(key);
} }
} }
/** /**
* iterator over all keys * iterator over all keys
* @param up * @param up
@ -545,7 +544,7 @@ public final class Heap extends HeapModifier implements BLOB {
m.put(a, b); m.put(a, b);
return m; return m;
} }
public static void maptest() { public static void maptest() {
final File f = new File("/Users/admin/blobtest.heap"); final File f = new File("/Users/admin/blobtest.heap");
try { try {
@ -565,7 +564,7 @@ public final class Heap extends HeapModifier implements BLOB {
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
} }
} }
public static void main(final String[] args) { public static void main(final String[] args) {
//heaptest(); //heaptest();
maptest(); maptest();

Write Preview
Loading…
Cancel
Save