boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

bad initialization

Browse Source
pull/1/head
Michael Christen 13 years ago
parent 14e45e90fd
commit 078fcde0dd
1 changed files with 88 additions and 81 deletions
Show Stats Download Patch File Download Diff File

169
source/net/yacy/kelondro/blob/HeapReader.java
Unescape View File

@ -80,21 +80,21 @@ public class HeapReader {
this.file = new CachedFileWriter(this.heapFile);
// read or initialize the index
fingerprintFileIdx = null;
fingerprintFileGap = null;
this.fingerprintFileIdx = null;
this.fingerprintFileGap = null;
if (initIndexReadDump()) {
// verify that everything worked just fine
// pick some elements of the index
Iterator<byte[]> i = this.index.keys(true, null);
int c = 3;
byte[] b, b1 = new byte[index.row().primaryKeyLength];
byte[] b, b1 = new byte[this.index.row().primaryKeyLength];
long pos;
boolean ok = true;
while (i.hasNext() && c-- > 0) {
b = i.next();
pos = this.index.get(b);
file.seek(pos + 4);
file.readFully(b1, 0, b1.length);
this.file.seek(pos + 4);
this.file.readFully(b1, 0, b1.length);
if (!this.ordering.equal(b, b1)) {
ok = false;
break;
@ -122,7 +122,7 @@ public class HeapReader {
}
public long mem() {
return index.mem(); // don't add the memory for free here since then the asserts for memory management don't work
return this.index.mem(); // don't add the memory for free here since then the asserts for memory management don't work
}
public void trim() {
@ -181,7 +181,7 @@ public class HeapReader {
// check saturation
int[] saturation = this.index.saturation();
Log.logInfo("HeapReader", "saturation of " + this.fingerprintFileIdx.getName() + ": keylength = " + saturation[0] + ", vallength = " + saturation[1] + ", possible saving: " + ((this.keylength - saturation[0] + 8 - saturation[1]) * index.size() / 1024 / 1024) + " MB");
Log.logInfo("HeapReader", "saturation of " + this.fingerprintFileIdx.getName() + ": keylength = " + saturation[0] + ", vallength = " + saturation[1] + ", possible saving: " + ((this.keylength - saturation[0] + 8 - saturation[1]) * this.index.size() / 1024 / 1024) + " MB");
// read the gap file:
try {
@ -239,31 +239,32 @@ public class HeapReader {
private void initIndexReadFromHeap() throws IOException {
// this initializes the this.index object by reading positions from the heap file
Log.logInfo("HeapReader", "generating index for " + heapFile.toString() + ", " + (file.length() / 1024 / 1024) + " MB. Please wait.");
Log.logInfo("HeapReader", "generating index for " + this.heapFile.toString() + ", " + (this.file.length() / 1024 / 1024) + " MB. Please wait.");
this.free = new Gap();
HandleMap.initDataConsumer indexready = HandleMap.asynchronusInitializer(this.name() + ".initializer", keylength, this.ordering, 8, Math.max(10, (int) (Runtime.getRuntime().freeMemory() / (10 * 1024 * 1024))));
byte[] key = new byte[keylength];
HandleMap.initDataConsumer indexready = HandleMap.asynchronusInitializer(this.name() + ".initializer", this.keylength, this.ordering, 8, Math.max(10, (int) (Runtime.getRuntime().freeMemory() / (10 * 1024 * 1024))));
byte[] key = new byte[this.keylength];
int reclen;
long seek = 0;
if (this.file.length() > 0) {
loop: while (true) { // don't test available() here because this does not work for files > 2GB
try {
// go to seek position
file.seek(seek);
this.file.seek(seek);
// read length of the following record without the length of the record size bytes
reclen = file.readInt();
reclen = this.file.readInt();
//assert reclen > 0 : " reclen == 0 at seek pos " + seek;
if (reclen == 0) {
// very bad file inconsistency
Log.logSevere("HeapReader", "reclen == 0 at seek pos " + seek + " in file " + heapFile);
Log.logSevere("HeapReader", "reclen == 0 at seek pos " + seek + " in file " + this.heapFile);
this.file.setLength(seek); // delete everything else at the remaining of the file :-(
break loop;
}
// read key
file.readFully(key, 0, key.length);
this.file.readFully(key, 0, key.length);
} catch (final IOException e) {
// EOF reached
@ -273,23 +274,24 @@ public class HeapReader {
// check if this record is empty
if (key == null || key[0] == 0) {
// it is an empty record, store to free list
if (reclen > 0) free.put(seek, reclen);
if (reclen > 0) this.free.put(seek, reclen);
} else {
if (this.ordering.wellformed(key)) {
indexready.consume(key, seek);
key = new byte[keylength];
key = new byte[this.keylength];
} else {
// free the lost space
free.put(seek, reclen);
file.seek(seek + 4);
this.free.put(seek, reclen);
this.file.seek(seek + 4);
Arrays.fill(key, (byte) 0);
file.write(key); // mark the place as empty record
Log.logWarning("HeapReader", "BLOB " + heapFile.getName() + ": skiped not wellformed key " + UTF8.String(key) + " at seek pos " + seek);
this.file.write(key); // mark the place as empty record
Log.logWarning("HeapReader", "BLOB " + this.heapFile.getName() + ": skiped not wellformed key " + UTF8.String(key) + " at seek pos " + seek);
}
}
// new seek position
seek += 4L + reclen;
}
}
indexready.finish();
// finish the index generation
@ -300,13 +302,13 @@ public class HeapReader {
} catch (ExecutionException e) {
Log.logException(e);
}
Log.logInfo("HeapReader", "finished index generation for " + heapFile.toString() + ", " + index.size() + " entries, " + free.size() + " gaps.");
Log.logInfo("HeapReader", "finished index generation for " + this.heapFile.toString() + ", " + this.index.size() + " entries, " + this.free.size() + " gaps.");
}
private void mergeFreeEntries() throws IOException {
// try to merge free entries
if (free.size() > 1) {
if (this.free.size() > 1) {
int merged = 0;
Map.Entry<Long, Integer> lastFree, nextFree;
final Iterator<Map.Entry<Long, Integer>> i = this.free.entrySet().iterator();
@ -328,7 +330,7 @@ public class HeapReader {
lastFree = nextFree;
}
}
Log.logInfo("HeapReader", "BLOB " + heapFile.toString() + ": merged " + merged + " free records");
Log.logInfo("HeapReader", "BLOB " + this.heapFile.toString() + ": merged " + merged + " free records");
if (merged > 0) deleteFingerprint();
}
}
@ -347,14 +349,14 @@ public class HeapReader {
*/
public int size() {
if (this.index == null) return 0;
synchronized (index) {
synchronized (this.index) {
return (this.index == null) ? 0 : this.index.size();
}
}
public boolean isEmpty() {
if (this.index == null) return true;
synchronized (index) {
synchronized (this.index) {
return this.index.isEmpty();
}
}
@ -365,12 +367,12 @@ public class HeapReader {
* @return true if the key exists, false otherwise
*/
public boolean containsKey(byte[] key) {
assert index != null;
assert this.index != null;
key = normalizeKey(key);
synchronized (this.index) {
// check if the file index contains the key
return index.get(key) >= 0;
return this.index.get(key) >= 0;
}
}
@ -387,7 +389,7 @@ public class HeapReader {
*/
protected synchronized byte[] firstKey() throws IOException {
synchronized (this.index) {
return index.smallestKey();
return this.index.smallestKey();
}
}
@ -400,7 +402,7 @@ public class HeapReader {
*/
protected byte[] first() throws IOException, RowSpaceExceededException {
synchronized (this.index) {
byte[] key = index.smallestKey();
byte[] key = this.index.smallestKey();
if (key == null) return null;
return get(key);
}
@ -415,7 +417,7 @@ public class HeapReader {
*/
protected byte[] lastKey() throws IOException {
synchronized (this.index) {
return index.largestKey();
return this.index.largestKey();
}
}
@ -428,7 +430,7 @@ public class HeapReader {
*/
protected byte[] last() throws IOException, RowSpaceExceededException {
synchronized (this.index) {
byte[] key = index.largestKey();
byte[] key = this.index.largestKey();
if (key == null) return null;
return get(key);
}
@ -446,20 +448,20 @@ public class HeapReader {
synchronized (this.index) {
// check if the index contains the key
final long pos = index.get(key);
final long pos = this.index.get(key);
if (pos < 0) return null;
// access the file and read the container
file.seek(pos);
final int len = file.readInt() - index.row().primaryKeyLength;
this.file.seek(pos);
final int len = this.file.readInt() - this.index.row().primaryKeyLength;
if (len < 0) {
// database file may be corrupted and should be deleted :-((
Log.logSevere("HeapReader", "file " + file.file() + " corrupted at " + pos + ": negative len. len = " + len + ", pk.len = " + index.row().primaryKeyLength);
Log.logSevere("HeapReader", "file " + this.file.file() + " corrupted at " + pos + ": negative len. len = " + len + ", pk.len = " + this.index.row().primaryKeyLength);
// to get lazy over that problem (who wants to tell the user to stop operation and delete the file???) we work on like the entry does not exist
index.remove(key);
this.index.remove(key);
return null;
}
long memr = len + index.row().primaryKeyLength + 64;
long memr = len + this.index.row().primaryKeyLength + 64;
if (MemoryControl.available() < memr) {
if (!MemoryControl.request(memr, true)) throw new RowSpaceExceededException(memr, "HeapReader.get()/check"); // not enough memory available for this blob
}
@ -467,17 +469,17 @@ public class HeapReader {
// read the key
byte[] keyf;
try {
keyf = new byte[index.row().primaryKeyLength];
keyf = new byte[this.index.row().primaryKeyLength];
} catch (OutOfMemoryError e) {
throw new RowSpaceExceededException(index.row().primaryKeyLength, "HeapReader.get()/keyf");
throw new RowSpaceExceededException(this.index.row().primaryKeyLength, "HeapReader.get()/keyf");
}
file.readFully(keyf, 0, keyf.length);
this.file.readFully(keyf, 0, keyf.length);
if (!this.ordering.equal(key, keyf)) {
// verification of the indexed access failed. we must re-read the index
Log.logSevere("HeapReader", "indexed verification access failed for " + heapFile.toString());
Log.logSevere("HeapReader", "indexed verification access failed for " + this.heapFile.toString());
// this is a severe operation, it should never happen.
// remove entry from index because keeping that element in the index would not make sense
index.remove(key);
this.index.remove(key);
// nothing to return
return null;
// but if the process ends in this state, it would completely fail
@ -492,7 +494,7 @@ public class HeapReader {
} catch (OutOfMemoryError e) {
throw new RowSpaceExceededException(len, "HeapReader.get()/blob");
}
file.readFully(blob, 0, blob.length);
this.file.readFully(blob, 0, blob.length);
return blob;
}
@ -512,12 +514,12 @@ public class HeapReader {
protected boolean checkKey(byte[] key, final long pos) throws IOException {
key = normalizeKey(key);
file.seek(pos);
file.readInt(); // skip the size value
this.file.seek(pos);
this.file.readInt(); // skip the size value
// read the key
final byte[] keyf = new byte[index.row().primaryKeyLength];
file.readFully(keyf, 0, keyf.length);
final byte[] keyf = new byte[this.index.row().primaryKeyLength];
this.file.readFully(keyf, 0, keyf.length);
return this.ordering.equal(key, keyf);
}
@ -532,12 +534,12 @@ public class HeapReader {
synchronized (this.index) {
// check if the index contains the key
final long pos = index.get(key);
final long pos = this.index.get(key);
if (pos < 0) return -1;
// access the file and read the size of the container
file.seek(pos);
return file.readInt() - index.row().primaryKeyLength;
this.file.seek(pos);
return this.file.readInt() - this.index.row().primaryKeyLength;
}
}
@ -546,20 +548,20 @@ public class HeapReader {
*/
public void close(boolean writeIDX) {
synchronized (this.index) {
if (file != null)
if (this.file != null)
try {
file.close();
this.file.close();
} catch (IOException e) {
Log.logException(e);
}
file = null;
if (writeIDX && index != null && free != null && (index.size() > 3 || free.size() > 3)) {
this.file = null;
if (writeIDX && this.index != null && this.free != null && (this.index.size() > 3 || this.free.size() > 3)) {
// now we can create a dump of the index and the gap information
// to speed up the next start
try {
String fingerprint = fingerprintFileHash(this.heapFile);
if (fingerprint == null) {
Log.logSevere("HeapReader", "cannot write a dump for " + heapFile.getName()+ ": fingerprint is null");
Log.logSevere("HeapReader", "cannot write a dump for " + this.heapFile.getName()+ ": fingerprint is null");
} else {
File newFingerprintFileGap = HeapWriter.fingerprintGapFile(this.heapFile, fingerprint);
if (this.fingerprintFileGap != null &&
@ -568,12 +570,12 @@ public class HeapReader {
Log.logInfo("HeapReader", "using existing gap dump instead of writing a new one: " + this.fingerprintFileGap.getName());
} else {
long start = System.currentTimeMillis();
free.dump(newFingerprintFileGap);
Log.logInfo("HeapReader", "wrote a dump for the " + this.free.size() + " gap entries of " + heapFile.getName()+ " in " + (System.currentTimeMillis() - start) + " milliseconds.");
this.free.dump(newFingerprintFileGap);
Log.logInfo("HeapReader", "wrote a dump for the " + this.free.size() + " gap entries of " + this.heapFile.getName()+ " in " + (System.currentTimeMillis() - start) + " milliseconds.");
}
}
free.clear();
free = null;
this.free.clear();
this.free = null;
if (fingerprint != null) {
File newFingerprintFileIdx = HeapWriter.fingerprintIndexFile(this.heapFile, fingerprint);
if (this.fingerprintFileIdx != null &&
@ -582,20 +584,20 @@ public class HeapReader {
Log.logInfo("HeapReader", "using existing idx dump instead of writing a new one: " + this.fingerprintFileIdx.getName());
} else {
long start = System.currentTimeMillis();
index.dump(newFingerprintFileIdx);
Log.logInfo("HeapReader", "wrote a dump for the " + this.index.size() + " index entries of " + heapFile.getName()+ " in " + (System.currentTimeMillis() - start) + " milliseconds.");
this.index.dump(newFingerprintFileIdx);
Log.logInfo("HeapReader", "wrote a dump for the " + this.index.size() + " index entries of " + this.heapFile.getName()+ " in " + (System.currentTimeMillis() - start) + " milliseconds.");
}
}
index.close();
index = null;
this.index.close();
this.index = null;
} catch (IOException e) {
Log.logException(e);
}
}
if (free != null) free.clear();
free = null;
if (index != null) index.close();
index = null;
if (this.free != null) this.free.clear();
this.free = null;
if (this.index != null) this.index.close();
this.index = null;
}
}
@ -672,18 +674,20 @@ public class HeapReader {
this.blobFile = blobFile;
}
@Override
public CloneableIterator<Entry<byte[], byte[]>> clone(Object modifier) {
// if the entries iterator is cloned, close the file!
if (is != null) try { is.close(); } catch (final IOException e) {}
is = null;
if (this.is != null) try { this.is.close(); } catch (final IOException e) {}
this.is = null;
try {
return new entries(blobFile, keylen);
return new entries(this.blobFile, this.keylen);
} catch (IOException e) {
Log.logException(e);
return null;
}
}
@Override
public Map.Entry<byte[], byte[]> next0() {
try {
byte b;
@ -692,14 +696,14 @@ public class HeapReader {
byte[] key;
final int keylen1 = this.keylen - 1;
while (true) {
len = is.readInt();
len = this.is.readInt();
if (len == 0) continue; // rare, but possible: zero length record (takes 4 bytes)
b = is.readByte(); // read a single by te to check for empty record
b = this.is.readByte(); // read a single by te to check for empty record
if (b == 0) {
// this is empty
// read some more bytes to consume the empty record
if (len > 1) {
if (len - 1 != is.skipBytes(len - 1)) { // all that is remaining
if (len - 1 != this.is.skipBytes(len - 1)) { // all that is remaining
Log.logWarning("HeapReader", "problem skiping " + + len + " bytes in " + this.blobFile.getName());
return null;
}
@ -709,13 +713,13 @@ public class HeapReader {
// we are now ahead of remaining this.keylen - 1 bytes of the key
key = new byte[this.keylen];
key[0] = b; // the first entry that we know already
if (is.read(key, 1, keylen1) < keylen1) return null; // read remaining key bytes
if (this.is.read(key, 1, keylen1) < keylen1) return null; // read remaining key bytes
// so far we have read this.keylen - 1 + 1 = this.keylen bytes.
// there must be a remaining number of len - this.keylen bytes left for the BLOB
if (len < this.keylen) return null; // a strange case that can only happen in case of corrupted data
try {
payload = new byte[len - this.keylen]; // the remaining record entries
if (is.read(payload) < payload.length) return null;
if (this.is.read(payload) < payload.length) return null;
return new entry(key, payload);
} catch (OutOfMemoryError e) {
// the allocation of memory for the payload may fail
@ -731,8 +735,8 @@ public class HeapReader {
}
public void close() {
if (is != null) try { is.close(); } catch (final IOException e) {}
is = null;
if (this.is != null) try { this.is.close(); } catch (final IOException e) {}
this.is = null;
}
@Override
@ -750,17 +754,20 @@ public class HeapReader {
this.b = b;
}
@Override
public byte[] getKey() {
return s;
return this.s;
}
@Override
public byte[] getValue() {
return b;
return this.b;
}
@Override
public byte[] setValue(byte[] value) {
byte[] b1 = b;
b = value;
byte[] b1 = this.b;
this.b = value;
return b1;
}
}

Write Preview
Loading…
Cancel
Save