boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

bad initialization

Browse Source
pull/1/head
Michael Christen 13 years ago
parent 14e45e90fd
commit 078fcde0dd
1 changed files with 88 additions and 81 deletions
Show Stats Download Patch File Download Diff File

169
source/net/yacy/kelondro/blob/HeapReader.java
Unescape View File

@ -80,21 +80,21 @@ public class HeapReader {
this.file = new CachedFileWriter(this.heapFile); this.file = new CachedFileWriter(this.heapFile);
// read or initialize the index // read or initialize the index
fingerprintFileIdx = null; this.fingerprintFileIdx = null;
fingerprintFileGap = null; this.fingerprintFileGap = null;
if (initIndexReadDump()) { if (initIndexReadDump()) {
// verify that everything worked just fine // verify that everything worked just fine
// pick some elements of the index // pick some elements of the index
Iterator<byte[]> i = this.index.keys(true, null); Iterator<byte[]> i = this.index.keys(true, null);
int c = 3; int c = 3;
byte[] b, b1 = new byte[index.row().primaryKeyLength]; byte[] b, b1 = new byte[this.index.row().primaryKeyLength];
long pos; long pos;
boolean ok = true; boolean ok = true;
while (i.hasNext() && c-- > 0) { while (i.hasNext() && c-- > 0) {
b = i.next(); b = i.next();
pos = this.index.get(b); pos = this.index.get(b);
file.seek(pos + 4); this.file.seek(pos + 4);
file.readFully(b1, 0, b1.length); this.file.readFully(b1, 0, b1.length);
if (!this.ordering.equal(b, b1)) { if (!this.ordering.equal(b, b1)) {
ok = false; ok = false;
break; break;
@ -122,7 +122,7 @@ public class HeapReader {
} }
public long mem() { public long mem() {
return index.mem(); // don't add the memory for free here since then the asserts for memory management don't work return this.index.mem(); // don't add the memory for free here since then the asserts for memory management don't work
} }
public void trim() { public void trim() {
@ -181,7 +181,7 @@ public class HeapReader {
// check saturation // check saturation
int[] saturation = this.index.saturation(); int[] saturation = this.index.saturation();
Log.logInfo("HeapReader", "saturation of " + this.fingerprintFileIdx.getName() + ": keylength = " + saturation[0] + ", vallength = " + saturation[1] + ", possible saving: " + ((this.keylength - saturation[0] + 8 - saturation[1]) * index.size() / 1024 / 1024) + " MB"); Log.logInfo("HeapReader", "saturation of " + this.fingerprintFileIdx.getName() + ": keylength = " + saturation[0] + ", vallength = " + saturation[1] + ", possible saving: " + ((this.keylength - saturation[0] + 8 - saturation[1]) * this.index.size() / 1024 / 1024) + " MB");
// read the gap file: // read the gap file:
try { try {
@ -239,31 +239,32 @@ public class HeapReader {
private void initIndexReadFromHeap() throws IOException { private void initIndexReadFromHeap() throws IOException {
// this initializes the this.index object by reading positions from the heap file // this initializes the this.index object by reading positions from the heap file
Log.logInfo("HeapReader", "generating index for " + heapFile.toString() + ", " + (file.length() / 1024 / 1024) + " MB. Please wait."); Log.logInfo("HeapReader", "generating index for " + this.heapFile.toString() + ", " + (this.file.length() / 1024 / 1024) + " MB. Please wait.");
this.free = new Gap(); this.free = new Gap();
HandleMap.initDataConsumer indexready = HandleMap.asynchronusInitializer(this.name() + ".initializer", keylength, this.ordering, 8, Math.max(10, (int) (Runtime.getRuntime().freeMemory() / (10 * 1024 * 1024)))); HandleMap.initDataConsumer indexready = HandleMap.asynchronusInitializer(this.name() + ".initializer", this.keylength, this.ordering, 8, Math.max(10, (int) (Runtime.getRuntime().freeMemory() / (10 * 1024 * 1024))));
byte[] key = new byte[keylength]; byte[] key = new byte[this.keylength];
int reclen; int reclen;
long seek = 0; long seek = 0;
if (this.file.length() > 0) {
loop: while (true) { // don't test available() here because this does not work for files > 2GB loop: while (true) { // don't test available() here because this does not work for files > 2GB
try { try {
// go to seek position // go to seek position
file.seek(seek); this.file.seek(seek);
// read length of the following record without the length of the record size bytes // read length of the following record without the length of the record size bytes
reclen = file.readInt(); reclen = this.file.readInt();
//assert reclen > 0 : " reclen == 0 at seek pos " + seek; //assert reclen > 0 : " reclen == 0 at seek pos " + seek;
if (reclen == 0) { if (reclen == 0) {
// very bad file inconsistency // very bad file inconsistency
Log.logSevere("HeapReader", "reclen == 0 at seek pos " + seek + " in file " + heapFile); Log.logSevere("HeapReader", "reclen == 0 at seek pos " + seek + " in file " + this.heapFile);
this.file.setLength(seek); // delete everything else at the remaining of the file :-( this.file.setLength(seek); // delete everything else at the remaining of the file :-(
break loop; break loop;
} }
// read key // read key
file.readFully(key, 0, key.length); this.file.readFully(key, 0, key.length);
} catch (final IOException e) { } catch (final IOException e) {
// EOF reached // EOF reached
@ -273,23 +274,24 @@ public class HeapReader {
// check if this record is empty // check if this record is empty
if (key == null || key[0] == 0) { if (key == null || key[0] == 0) {
// it is an empty record, store to free list // it is an empty record, store to free list
if (reclen > 0) free.put(seek, reclen); if (reclen > 0) this.free.put(seek, reclen);
} else { } else {
if (this.ordering.wellformed(key)) { if (this.ordering.wellformed(key)) {
indexready.consume(key, seek); indexready.consume(key, seek);
key = new byte[keylength]; key = new byte[this.keylength];
} else { } else {
// free the lost space // free the lost space
free.put(seek, reclen); this.free.put(seek, reclen);
file.seek(seek + 4); this.file.seek(seek + 4);
Arrays.fill(key, (byte) 0); Arrays.fill(key, (byte) 0);
file.write(key); // mark the place as empty record this.file.write(key); // mark the place as empty record
Log.logWarning("HeapReader", "BLOB " + heapFile.getName() + ": skiped not wellformed key " + UTF8.String(key) + " at seek pos " + seek); Log.logWarning("HeapReader", "BLOB " + this.heapFile.getName() + ": skiped not wellformed key " + UTF8.String(key) + " at seek pos " + seek);
} }
} }
// new seek position // new seek position
seek += 4L + reclen; seek += 4L + reclen;
} }
}
indexready.finish(); indexready.finish();
// finish the index generation // finish the index generation
@ -300,13 +302,13 @@ public class HeapReader {
} catch (ExecutionException e) { } catch (ExecutionException e) {
Log.logException(e); Log.logException(e);
} }
Log.logInfo("HeapReader", "finished index generation for " + heapFile.toString() + ", " + index.size() + " entries, " + free.size() + " gaps."); Log.logInfo("HeapReader", "finished index generation for " + this.heapFile.toString() + ", " + this.index.size() + " entries, " + this.free.size() + " gaps.");
} }
private void mergeFreeEntries() throws IOException { private void mergeFreeEntries() throws IOException {
// try to merge free entries // try to merge free entries
if (free.size() > 1) { if (this.free.size() > 1) {
int merged = 0; int merged = 0;
Map.Entry<Long, Integer> lastFree, nextFree; Map.Entry<Long, Integer> lastFree, nextFree;
final Iterator<Map.Entry<Long, Integer>> i = this.free.entrySet().iterator(); final Iterator<Map.Entry<Long, Integer>> i = this.free.entrySet().iterator();
@ -328,7 +330,7 @@ public class HeapReader {
lastFree = nextFree; lastFree = nextFree;
} }
} }
Log.logInfo("HeapReader", "BLOB " + heapFile.toString() + ": merged " + merged + " free records"); Log.logInfo("HeapReader", "BLOB " + this.heapFile.toString() + ": merged " + merged + " free records");
if (merged > 0) deleteFingerprint(); if (merged > 0) deleteFingerprint();
} }
} }
@ -347,14 +349,14 @@ public class HeapReader {
*/ */
public int size() { public int size() {
if (this.index == null) return 0; if (this.index == null) return 0;
synchronized (index) { synchronized (this.index) {
return (this.index == null) ? 0 : this.index.size(); return (this.index == null) ? 0 : this.index.size();
} }
} }
public boolean isEmpty() { public boolean isEmpty() {
if (this.index == null) return true; if (this.index == null) return true;
synchronized (index) { synchronized (this.index) {
return this.index.isEmpty(); return this.index.isEmpty();
} }
} }
@ -365,12 +367,12 @@ public class HeapReader {
* @return true if the key exists, false otherwise * @return true if the key exists, false otherwise
*/ */
public boolean containsKey(byte[] key) { public boolean containsKey(byte[] key) {
assert index != null; assert this.index != null;
key = normalizeKey(key); key = normalizeKey(key);
synchronized (this.index) { synchronized (this.index) {
// check if the file index contains the key // check if the file index contains the key
return index.get(key) >= 0; return this.index.get(key) >= 0;
} }
} }
@ -387,7 +389,7 @@ public class HeapReader {
*/ */
protected synchronized byte[] firstKey() throws IOException { protected synchronized byte[] firstKey() throws IOException {
synchronized (this.index) { synchronized (this.index) {
return index.smallestKey(); return this.index.smallestKey();
} }
} }
@ -400,7 +402,7 @@ public class HeapReader {
*/ */
protected byte[] first() throws IOException, RowSpaceExceededException { protected byte[] first() throws IOException, RowSpaceExceededException {
synchronized (this.index) { synchronized (this.index) {
byte[] key = index.smallestKey(); byte[] key = this.index.smallestKey();
if (key == null) return null; if (key == null) return null;
return get(key); return get(key);
} }
@ -415,7 +417,7 @@ public class HeapReader {
*/ */
protected byte[] lastKey() throws IOException { protected byte[] lastKey() throws IOException {
synchronized (this.index) { synchronized (this.index) {
return index.largestKey(); return this.index.largestKey();
} }
} }
@ -428,7 +430,7 @@ public class HeapReader {
*/ */
protected byte[] last() throws IOException, RowSpaceExceededException { protected byte[] last() throws IOException, RowSpaceExceededException {
synchronized (this.index) { synchronized (this.index) {
byte[] key = index.largestKey(); byte[] key = this.index.largestKey();
if (key == null) return null; if (key == null) return null;
return get(key); return get(key);
} }
@ -446,20 +448,20 @@ public class HeapReader {
synchronized (this.index) { synchronized (this.index) {
// check if the index contains the key // check if the index contains the key
final long pos = index.get(key); final long pos = this.index.get(key);
if (pos < 0) return null; if (pos < 0) return null;
// access the file and read the container // access the file and read the container
file.seek(pos); this.file.seek(pos);
final int len = file.readInt() - index.row().primaryKeyLength; final int len = this.file.readInt() - this.index.row().primaryKeyLength;
if (len < 0) { if (len < 0) {
// database file may be corrupted and should be deleted :-(( // database file may be corrupted and should be deleted :-((
Log.logSevere("HeapReader", "file " + file.file() + " corrupted at " + pos + ": negative len. len = " + len + ", pk.len = " + index.row().primaryKeyLength); Log.logSevere("HeapReader", "file " + this.file.file() + " corrupted at " + pos + ": negative len. len = " + len + ", pk.len = " + this.index.row().primaryKeyLength);
// to get lazy over that problem (who wants to tell the user to stop operation and delete the file???) we work on like the entry does not exist // to get lazy over that problem (who wants to tell the user to stop operation and delete the file???) we work on like the entry does not exist
index.remove(key); this.index.remove(key);
return null; return null;
} }
long memr = len + index.row().primaryKeyLength + 64; long memr = len + this.index.row().primaryKeyLength + 64;
if (MemoryControl.available() < memr) { if (MemoryControl.available() < memr) {
if (!MemoryControl.request(memr, true)) throw new RowSpaceExceededException(memr, "HeapReader.get()/check"); // not enough memory available for this blob if (!MemoryControl.request(memr, true)) throw new RowSpaceExceededException(memr, "HeapReader.get()/check"); // not enough memory available for this blob
} }
@ -467,17 +469,17 @@ public class HeapReader {
// read the key // read the key
byte[] keyf; byte[] keyf;
try { try {
keyf = new byte[index.row().primaryKeyLength]; keyf = new byte[this.index.row().primaryKeyLength];
} catch (OutOfMemoryError e) { } catch (OutOfMemoryError e) {
throw new RowSpaceExceededException(index.row().primaryKeyLength, "HeapReader.get()/keyf"); throw new RowSpaceExceededException(this.index.row().primaryKeyLength, "HeapReader.get()/keyf");
} }
file.readFully(keyf, 0, keyf.length); this.file.readFully(keyf, 0, keyf.length);
if (!this.ordering.equal(key, keyf)) { if (!this.ordering.equal(key, keyf)) {
// verification of the indexed access failed. we must re-read the index // verification of the indexed access failed. we must re-read the index
Log.logSevere("HeapReader", "indexed verification access failed for " + heapFile.toString()); Log.logSevere("HeapReader", "indexed verification access failed for " + this.heapFile.toString());
// this is a severe operation, it should never happen. // this is a severe operation, it should never happen.
// remove entry from index because keeping that element in the index would not make sense // remove entry from index because keeping that element in the index would not make sense
index.remove(key); this.index.remove(key);
// nothing to return // nothing to return
return null; return null;
// but if the process ends in this state, it would completely fail // but if the process ends in this state, it would completely fail
@ -492,7 +494,7 @@ public class HeapReader {
} catch (OutOfMemoryError e) { } catch (OutOfMemoryError e) {
throw new RowSpaceExceededException(len, "HeapReader.get()/blob"); throw new RowSpaceExceededException(len, "HeapReader.get()/blob");
} }
file.readFully(blob, 0, blob.length); this.file.readFully(blob, 0, blob.length);
return blob; return blob;
} }
@ -512,12 +514,12 @@ public class HeapReader {
protected boolean checkKey(byte[] key, final long pos) throws IOException { protected boolean checkKey(byte[] key, final long pos) throws IOException {
key = normalizeKey(key); key = normalizeKey(key);
file.seek(pos); this.file.seek(pos);
file.readInt(); // skip the size value this.file.readInt(); // skip the size value
// read the key // read the key
final byte[] keyf = new byte[index.row().primaryKeyLength]; final byte[] keyf = new byte[this.index.row().primaryKeyLength];
file.readFully(keyf, 0, keyf.length); this.file.readFully(keyf, 0, keyf.length);
return this.ordering.equal(key, keyf); return this.ordering.equal(key, keyf);
} }
@ -532,12 +534,12 @@ public class HeapReader {
synchronized (this.index) { synchronized (this.index) {
// check if the index contains the key // check if the index contains the key
final long pos = index.get(key); final long pos = this.index.get(key);
if (pos < 0) return -1; if (pos < 0) return -1;
// access the file and read the size of the container // access the file and read the size of the container
file.seek(pos); this.file.seek(pos);
return file.readInt() - index.row().primaryKeyLength; return this.file.readInt() - this.index.row().primaryKeyLength;
} }
} }
@ -546,20 +548,20 @@ public class HeapReader {
*/ */
public void close(boolean writeIDX) { public void close(boolean writeIDX) {
synchronized (this.index) { synchronized (this.index) {
if (file != null) if (this.file != null)
try { try {
file.close(); this.file.close();
} catch (IOException e) { } catch (IOException e) {
Log.logException(e); Log.logException(e);
} }
file = null; this.file = null;
if (writeIDX && index != null && free != null && (index.size() > 3 || free.size() > 3)) { if (writeIDX && this.index != null && this.free != null && (this.index.size() > 3 || this.free.size() > 3)) {
// now we can create a dump of the index and the gap information // now we can create a dump of the index and the gap information
// to speed up the next start // to speed up the next start
try { try {
String fingerprint = fingerprintFileHash(this.heapFile); String fingerprint = fingerprintFileHash(this.heapFile);
if (fingerprint == null) { if (fingerprint == null) {
Log.logSevere("HeapReader", "cannot write a dump for " + heapFile.getName()+ ": fingerprint is null"); Log.logSevere("HeapReader", "cannot write a dump for " + this.heapFile.getName()+ ": fingerprint is null");
} else { } else {
File newFingerprintFileGap = HeapWriter.fingerprintGapFile(this.heapFile, fingerprint); File newFingerprintFileGap = HeapWriter.fingerprintGapFile(this.heapFile, fingerprint);
if (this.fingerprintFileGap != null && if (this.fingerprintFileGap != null &&
@ -568,12 +570,12 @@ public class HeapReader {
Log.logInfo("HeapReader", "using existing gap dump instead of writing a new one: " + this.fingerprintFileGap.getName()); Log.logInfo("HeapReader", "using existing gap dump instead of writing a new one: " + this.fingerprintFileGap.getName());
} else { } else {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
free.dump(newFingerprintFileGap); this.free.dump(newFingerprintFileGap);
Log.logInfo("HeapReader", "wrote a dump for the " + this.free.size() + " gap entries of " + heapFile.getName()+ " in " + (System.currentTimeMillis() - start) + " milliseconds."); Log.logInfo("HeapReader", "wrote a dump for the " + this.free.size() + " gap entries of " + this.heapFile.getName()+ " in " + (System.currentTimeMillis() - start) + " milliseconds.");
} }
} }
free.clear(); this.free.clear();
free = null; this.free = null;
if (fingerprint != null) { if (fingerprint != null) {
File newFingerprintFileIdx = HeapWriter.fingerprintIndexFile(this.heapFile, fingerprint); File newFingerprintFileIdx = HeapWriter.fingerprintIndexFile(this.heapFile, fingerprint);
if (this.fingerprintFileIdx != null && if (this.fingerprintFileIdx != null &&
@ -582,20 +584,20 @@ public class HeapReader {
Log.logInfo("HeapReader", "using existing idx dump instead of writing a new one: " + this.fingerprintFileIdx.getName()); Log.logInfo("HeapReader", "using existing idx dump instead of writing a new one: " + this.fingerprintFileIdx.getName());
} else { } else {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
index.dump(newFingerprintFileIdx); this.index.dump(newFingerprintFileIdx);
Log.logInfo("HeapReader", "wrote a dump for the " + this.index.size() + " index entries of " + heapFile.getName()+ " in " + (System.currentTimeMillis() - start) + " milliseconds."); Log.logInfo("HeapReader", "wrote a dump for the " + this.index.size() + " index entries of " + this.heapFile.getName()+ " in " + (System.currentTimeMillis() - start) + " milliseconds.");
} }
} }
index.close(); this.index.close();
index = null; this.index = null;
} catch (IOException e) { } catch (IOException e) {
Log.logException(e); Log.logException(e);
} }
} }
if (free != null) free.clear(); if (this.free != null) this.free.clear();
free = null; this.free = null;
if (index != null) index.close(); if (this.index != null) this.index.close();
index = null; this.index = null;
} }
} }
@ -672,18 +674,20 @@ public class HeapReader {
this.blobFile = blobFile; this.blobFile = blobFile;
} }
@Override
public CloneableIterator<Entry<byte[], byte[]>> clone(Object modifier) { public CloneableIterator<Entry<byte[], byte[]>> clone(Object modifier) {
// if the entries iterator is cloned, close the file! // if the entries iterator is cloned, close the file!
if (is != null) try { is.close(); } catch (final IOException e) {} if (this.is != null) try { this.is.close(); } catch (final IOException e) {}
is = null; this.is = null;
try { try {
return new entries(blobFile, keylen); return new entries(this.blobFile, this.keylen);
} catch (IOException e) { } catch (IOException e) {
Log.logException(e); Log.logException(e);
return null; return null;
} }
} }
@Override
public Map.Entry<byte[], byte[]> next0() { public Map.Entry<byte[], byte[]> next0() {
try { try {
byte b; byte b;
@ -692,14 +696,14 @@ public class HeapReader {
byte[] key; byte[] key;
final int keylen1 = this.keylen - 1; final int keylen1 = this.keylen - 1;
while (true) { while (true) {
len = is.readInt(); len = this.is.readInt();
if (len == 0) continue; // rare, but possible: zero length record (takes 4 bytes) if (len == 0) continue; // rare, but possible: zero length record (takes 4 bytes)
b = is.readByte(); // read a single by te to check for empty record b = this.is.readByte(); // read a single by te to check for empty record
if (b == 0) { if (b == 0) {
// this is empty // this is empty
// read some more bytes to consume the empty record // read some more bytes to consume the empty record
if (len > 1) { if (len > 1) {
if (len - 1 != is.skipBytes(len - 1)) { // all that is remaining if (len - 1 != this.is.skipBytes(len - 1)) { // all that is remaining
Log.logWarning("HeapReader", "problem skiping " + + len + " bytes in " + this.blobFile.getName()); Log.logWarning("HeapReader", "problem skiping " + + len + " bytes in " + this.blobFile.getName());
return null; return null;
} }
@ -709,13 +713,13 @@ public class HeapReader {
// we are now ahead of remaining this.keylen - 1 bytes of the key // we are now ahead of remaining this.keylen - 1 bytes of the key
key = new byte[this.keylen]; key = new byte[this.keylen];
key[0] = b; // the first entry that we know already key[0] = b; // the first entry that we know already
if (is.read(key, 1, keylen1) < keylen1) return null; // read remaining key bytes if (this.is.read(key, 1, keylen1) < keylen1) return null; // read remaining key bytes
// so far we have read this.keylen - 1 + 1 = this.keylen bytes. // so far we have read this.keylen - 1 + 1 = this.keylen bytes.
// there must be a remaining number of len - this.keylen bytes left for the BLOB // there must be a remaining number of len - this.keylen bytes left for the BLOB
if (len < this.keylen) return null; // a strange case that can only happen in case of corrupted data if (len < this.keylen) return null; // a strange case that can only happen in case of corrupted data
try { try {
payload = new byte[len - this.keylen]; // the remaining record entries payload = new byte[len - this.keylen]; // the remaining record entries
if (is.read(payload) < payload.length) return null; if (this.is.read(payload) < payload.length) return null;
return new entry(key, payload); return new entry(key, payload);
} catch (OutOfMemoryError e) { } catch (OutOfMemoryError e) {
// the allocation of memory for the payload may fail // the allocation of memory for the payload may fail
@ -731,8 +735,8 @@ public class HeapReader {
} }
public void close() { public void close() {
if (is != null) try { is.close(); } catch (final IOException e) {} if (this.is != null) try { this.is.close(); } catch (final IOException e) {}
is = null; this.is = null;
} }
@Override @Override
@ -750,17 +754,20 @@ public class HeapReader {
this.b = b; this.b = b;
} }
@Override
public byte[] getKey() { public byte[] getKey() {
return s; return this.s;
} }
@Override
public byte[] getValue() { public byte[] getValue() {
return b; return this.b;
} }
@Override
public byte[] setValue(byte[] value) { public byte[] setValue(byte[] value) {
byte[] b1 = b; byte[] b1 = this.b;
b = value; this.b = value;
return b1; return b1;
} }
} }

Write Preview
Loading…
Cancel
Save