patch in HTCache and CitationIndex loading in case that a file is

broken: do not crash; instead ignore the file and delete it.

source/net/yacy/crawler/data/Cache.java

@@ -89,7 +89,7 @@ public final class Cache {
         }
         // open the cache file
         try {
-            fileDBunbuffered = new ArrayStack(new File(cachePath, FILE_DB_NAME), prefix, Base64Order.enhancedCoder, 12, 1024 * 1024 * 2, false);
+            fileDBunbuffered = new ArrayStack(new File(cachePath, FILE_DB_NAME), prefix, Base64Order.enhancedCoder, 12, 1024 * 1024 * 2, false, true);
             fileDBunbuffered.setMaxSize(maxCacheSize);
             fileDB = new Compressor(fileDBunbuffered, 6 * 1024 * 1024);
         } catch (final IOException e) {

source/net/yacy/kelondro/blob/ArrayStack.java

@@ -111,7 +111,8 @@ public class ArrayStack implements BLOB {
             final ByteOrder ordering,
             final int keylength,
             final int buffersize,
-            final boolean trimall) throws IOException {
+            final boolean trimall,
+            final boolean deleteonfail) throws IOException {
         this.keylength = keylength;
         this.prefix = prefix;
         this.ordering = ordering;
@@ -200,13 +201,22 @@ public class ArrayStack implements BLOB {
                    d = my_SHORT_MILSEC_FORMATTER.parse(file.substring(this.prefix.length() + 1, this.prefix.length() + 18));
                    f = new File(heapLocation, file);
                    time = d.getTime();
-                   if (time == maxtime && !trimall) {
-                       oneBlob = new Heap(f, keylength, ordering, buffersize);
-                   } else {
-                       oneBlob = new HeapModifier(f, keylength, ordering);
-                       oneBlob.trim(); // no writings here, can be used with minimum memory
+                   try {
+                       if (time == maxtime && !trimall) {
+                           oneBlob = new Heap(f, keylength, ordering, buffersize);
+                       } else {
+                           oneBlob = new HeapModifier(f, keylength, ordering);
+                           oneBlob.trim(); // no writings here, can be used with minimum memory
+                       }
+                       sortedItems.put(Long.valueOf(time), new blobItem(d, f, oneBlob));
+                   } catch (IOException e) {
+                       if (deleteonfail) {
+                           Log.logWarning("ArrayStack", "cannot read file " + f.getName() + ", deleting it (smart fail; alternative would be: crash; required user action would be same as deletion)");
+                           f.delete();
+                       } else {
+                           throw new IOException(e.getMessage(), e);
+                       }
                    }
-                   sortedItems.put(Long.valueOf(time), new blobItem(d, f, oneBlob));
                } catch (final ParseException e) {continue;}
             }
         }
@@ -1155,7 +1165,7 @@ public class ArrayStack implements BLOB {
         final File f = new File("/Users/admin/blobarraytest");
         try {
             //f.delete();
-            final ArrayStack heap = new ArrayStack(f, "test", NaturalOrder.naturalOrder, 12, 512 * 1024, false);
+            final ArrayStack heap = new ArrayStack(f, "test", NaturalOrder.naturalOrder, 12, 512 * 1024, false, true);
             heap.insert("aaaaaaaaaaaa".getBytes(), "eins zwei drei".getBytes());
             heap.insert("aaaaaaaaaaab".getBytes(), "vier fuenf sechs".getBytes());
             heap.insert("aaaaaaaaaaac".getBytes(), "sieben acht neun".getBytes());

source/net/yacy/kelondro/io/CachedFileWriter.java

@@ -87,7 +87,7 @@ public final class CachedFileWriter extends AbstractWriter implements Writer {
         long available = this.RAFile.length() - seek;
         if (available == -seek) return; // we don't know how this happens but we just silently ignore it by now TODO:fixme
         //System.out.println("*** available = " + available);
-        if (available < len) throw new IOException("EOF, available = " + available + ", requested = " + len + ", this.RAFile.length() = " + this.RAFile.length() + ", seek = " + seek);
+        if (available < len) throw new IOException("EOF in " + this.file.getName() + ", available = " + available + ", requested = " + len + ", this.RAFile.length() = " + this.RAFile.length() + ", seek = " + seek);
         if (this.cachestart + this.cachelen == seek && this.cache.length - this.cachelen >= len) {
             this.RAFile.readFully(this.cache, this.cachelen, len);
             //System.out.println("*** DEBUG FileRA " + this.file.getName() + ": append fill " + len + " bytes");

source/net/yacy/kelondro/rwi/ReferenceContainerArray.java

@@ -71,6 +71,7 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
             termOrder,
             termSize,
             0,
+            true,
             true);
     }