diff --git a/source/net/yacy/kelondro/blob/HeapReader.java b/source/net/yacy/kelondro/blob/HeapReader.java index fd454e305..4e25d7058 100644 --- a/source/net/yacy/kelondro/blob/HeapReader.java +++ b/source/net/yacy/kelondro/blob/HeapReader.java @@ -29,7 +29,6 @@ import java.io.DataInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; -import java.util.Arrays; import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; @@ -555,11 +554,12 @@ public class HeapReader { private Map.Entry next0() { try { byte b; + int len; byte[] payload; byte[] key; final int keylen1 = this.keylen - 1; while (true) { - int len = is.readInt(); + len = is.readInt(); if (len == 0) continue; // rare, but possible: zero length record (takes 4 bytes) b = is.readByte(); // read a single by te to check for empty record if (b == 0) { @@ -575,6 +575,10 @@ public class HeapReader { // so far we have read this.keylen - 1 + 1 = this.keylen bytes. // there must be a remaining number of len - this.keylen bytes left for the BLOB if (len < this.keylen) return null; // a strange case that can only happen in case of corrupted data + if (is.available() < (len - this.keylen)) { // this really indicates corrupted data + Log.logWarning("HeapReader", "corrupted data by entry of " + len + " bytes"); + return null; + } payload = new byte[len - this.keylen]; // the remaining record entries if (is.read(payload) < payload.length) return null; return new entry(key, payload); diff --git a/source/net/yacy/kelondro/blob/HeapWriter.java b/source/net/yacy/kelondro/blob/HeapWriter.java index abf64ce83..67a551a88 100644 --- a/source/net/yacy/kelondro/blob/HeapWriter.java +++ b/source/net/yacy/kelondro/blob/HeapWriter.java @@ -100,18 +100,17 @@ public final class HeapWriter { assert blob.length > 0; key = HeapReader.normalizeKey(key, this.keylength); assert index.row().primaryKeyLength == this.keylength : index.row().primaryKeyLength + "!=" + key.length; + assert key.length == this.keylength : "key.length == " + key.length + ", this.keylength = " + this.keylength; // after normalizing they should be equal in length assert index.get(key) < 0 : "index.get(key) = " + index.get(key) + ", index.size() = " + index.size() + ", file.length() = " + this.heapFileTMP.length() + ", key = " + new String(key); // must not occur before if ((blob == null) || (blob.length == 0)) return; index.putUnique(key, this.seek); int chunkl = this.keylength + blob.length; os.writeInt(chunkl); os.write(key); - if (this.keylength > key.length) { - for (int i = 0; i < this.keylength - key.length; i++) os.write(ZERO); - } os.write(blob); //assert (this.doublecheck.add(new String(key))) : "doublecheck failed for " + new String(key); this.seek += chunkl + 4; + os.flush(); // necessary? may cause bad IO performance :-( } protected static File fingerprintIndexFile(File f, String fingerprint) {