added sixcoolers patch and more checks/removed unnecessary code

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6636 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/net/yacy/kelondro/blob/HeapReader.java

@@ -29,7 +29,6 @@ import java.io.DataInputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -555,11 +554,12 @@ public class HeapReader {
         private Map.Entry<byte[], byte[]> next0() {
             try {
                 byte b;
+                int len;
                 byte[] payload;
                 byte[] key;
                 final int keylen1 = this.keylen - 1;
                 while (true) {
-                    int len = is.readInt();
+                    len = is.readInt();
                     if (len == 0) continue; // rare, but possible: zero length record (takes 4 bytes)
                     b = is.readByte();      // read a single by te to check for empty record
                     if (b == 0) {
@@ -575,6 +575,10 @@ public class HeapReader {
                     // so far we have read this.keylen - 1 + 1 = this.keylen bytes.
                     // there must be a remaining number of len - this.keylen bytes left for the BLOB
                     if (len < this.keylen) return null;    // a strange case that can only happen in case of corrupted data
+                    if (is.available() < (len - this.keylen)) { // this really indicates corrupted data
+                        Log.logWarning("HeapReader", "corrupted data by entry of " + len + " bytes");
+                        return null;
+                    }
                     payload = new byte[len - this.keylen]; // the remaining record entries
                     if (is.read(payload) < payload.length) return null;
                     return new entry(key, payload);

source/net/yacy/kelondro/blob/HeapWriter.java

@@ -100,18 +100,17 @@ public final class HeapWriter {
         assert blob.length > 0;
         key = HeapReader.normalizeKey(key, this.keylength);
         assert index.row().primaryKeyLength == this.keylength : index.row().primaryKeyLength + "!=" + key.length;
+        assert key.length == this.keylength : "key.length == " + key.length + ", this.keylength = " + this.keylength; // after normalizing they should be equal in length
         assert index.get(key) < 0 : "index.get(key) = " + index.get(key) + ", index.size() = " + index.size() + ", file.length() = " + this.heapFileTMP.length() +  ", key = " + new String(key); // must not occur before
         if ((blob == null) || (blob.length == 0)) return;
         index.putUnique(key, this.seek);
         int chunkl = this.keylength + blob.length;
         os.writeInt(chunkl);
         os.write(key);
-        if (this.keylength > key.length) {
-            for (int i = 0; i < this.keylength - key.length; i++) os.write(ZERO);
-        }
         os.write(blob);
         //assert (this.doublecheck.add(new String(key))) : "doublecheck failed for " + new String(key);
         this.seek += chunkl + 4;
+        os.flush(); // necessary? may cause bad IO performance :-(
     }
     
     protected static File fingerprintIndexFile(File f, String fingerprint) {