hardening of index dump write process

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5769 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/kelondro/blob/HeapWriter.java

@@ -136,20 +136,18 @@ public final class HeapWriter  {
      * close the BLOB table
      * @throws  
      */
-    public synchronized void close(boolean writeIDX) {
+    public synchronized void close(boolean writeIDX) throws IOException {
         // close the file
-        try {
         os.flush();
         os.close();
-        } catch (final IOException e) {
-            e.printStackTrace();
-        }
         os = null;
         
         // rename the file into final name
-        this.heapFileTMP.renameTo(this.heapFileREADY);
-        assert this.heapFileREADY.exists() : this.heapFileREADY.toString();
-        assert !this.heapFileTMP.exists() : this.heapFileTMP.toString();
+        if (this.heapFileREADY.exists()) FileUtils.deletedelete(this.heapFileREADY);
+        boolean renameok = this.heapFileTMP.renameTo(this.heapFileREADY);
+        if (!renameok) throw new IOException("cannot rename " + this.heapFileTMP + " to " + this.heapFileREADY);
+        if (!this.heapFileREADY.exists()) throw new IOException("renaming of " + this.heapFileREADY.toString() + " failed: files still exists");
+        if (this.heapFileTMP.exists()) throw new IOException("renaming to " + this.heapFileTMP.toString() + " failed: file does not exist");
         
         // generate index and gap files
         if (writeIDX && index.size() > 3) {

source/de/anomic/kelondro/text/IODispatcher.java

@@ -84,11 +84,7 @@ public class IODispatcher extends Thread {
     
     public synchronized void dump(ReferenceContainerCache cache, File file, ReferenceContainerArray array) {
         if (dumpQueue == null || !this.isAlive()) {
-            try {
             cache.dump(file, true);
-            } catch (IOException e) {
-                e.printStackTrace();
-            }
         } else {
             DumpJob job = new DumpJob(cache, file, array);
             try {
@@ -96,11 +92,7 @@ public class IODispatcher extends Thread {
                 controlQueue.put(vita);
             } catch (InterruptedException e) {
                 e.printStackTrace();
-                try {
                 cache.dump(file, true);
-                } catch (IOException ee) {
-                    e.printStackTrace();
-                }
             }
         }
     }
@@ -242,11 +234,18 @@ public class IODispatcher extends Thread {
         File tmpFile = new File(newFile.getParentFile(), newFile.getName() + ".tmp");
         HeapWriter writer = new HeapWriter(tmpFile, newFile, array.keylength(), array.ordering());
         merge(i1, i2, array.ordering(), writer);
+        try {
             writer.close(true);
             // we don't need the old files any more
             FileUtils.deletedelete(f1);
             FileUtils.deletedelete(f2);
             return newFile;
+        } catch (IOException e) {
+            FileUtils.deletedelete(tmpFile);
+            FileUtils.deletedelete(newFile);
+            e.printStackTrace();
+            return null;
+        }
     }
     
     private static void merge(CloneableIterator<ReferenceContainer> i1, CloneableIterator<ReferenceContainer> i2, ByteOrder ordering, HeapWriter writer) throws IOException {

source/de/anomic/kelondro/text/IndexBuffer.java

@@ -318,13 +318,7 @@ public final class IndexBuffer extends AbstractIndex implements Index, IndexRead
     }
 
     public synchronized void close() {
-        // dump cache
-        try {
-            //heap.dumpold(this.oldDumpFile);
         heap.dump(this.dumpFile, true);
-        } catch (final IOException e){
-            log.logSevere("unable to dump cache: " + e.getMessage(), e);
-        }
         heap = null;
         hashScore.clear();
         hashDate.clear();

source/de/anomic/kelondro/text/IndexCell.java

@@ -244,12 +244,7 @@ public final class IndexCell extends AbstractBufferedIndex implements BufferedIn
      * and is composed of the current date and the cell salt
      */
     public synchronized void close() {
-        // dump the ram
-        try {
         this.ram.dump(this.array.newContainerBLOBFile(), true);
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
         // close all
         this.ram.close();
         this.array.close();

source/de/anomic/kelondro/text/ReferenceContainerCache.java

@@ -110,12 +110,18 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde
         Log.logInfo("indexContainerRAMHeap", "finished rwi blob restore: " + cache.size() + " words, " + urlCount + " word/URL relations in " + (System.currentTimeMillis() - start) + " milliseconds");
     }
     
-    public void dump(final File heapFile, boolean writeIDX) throws IOException {
+    public void dump(final File heapFile, boolean writeIDX) {
         assert this.cache != null;
         Log.logInfo("indexContainerRAMHeap", "creating rwi heap dump '" + heapFile.getName() + "', " + cache.size() + " rwi's");
         if (heapFile.exists()) FileUtils.deletedelete(heapFile);
         File tmpFile = new File(heapFile.getParentFile(), heapFile.getName() + ".tmp");
-        HeapWriter dump = new HeapWriter(tmpFile, heapFile, payloadrow.primaryKeyLength, Base64Order.enhancedCoder);
+        HeapWriter dump;
+        try {
+            dump = new HeapWriter(tmpFile, heapFile, payloadrow.primaryKeyLength, Base64Order.enhancedCoder);
+        } catch (IOException e1) {
+            e1.printStackTrace();
+            return;
+        }
         final long startTime = System.currentTimeMillis();
         long wordcount = 0, urlcount = 0;
         String wordHash = null, lwh;
@@ -135,15 +141,25 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde
                 // put entries on heap
                 if (container != null && wordHash.length() == payloadrow.primaryKeyLength) {
                     //System.out.println("Dump: " + wordHash);
+                    try {
                         dump.add(wordHash.getBytes(), container.exportCollection());
+                    } catch (IOException e) {
+                        e.printStackTrace();
+                    }
                     urlcount += container.size();
                 }
                 wordcount++;
             }
         }
+        try {
             dump.close(writeIDX);
+            Log.logInfo("indexContainerRAMHeap", "finished rwi heap dump: " + wordcount + " words, " + urlcount + " word/URL relations in " + (System.currentTimeMillis() - startTime) + " milliseconds");
+        } catch (IOException e) {
+            e.printStackTrace();
+            Log.logInfo("indexContainerRAMHeap", "failed rwi heap dump: " + e.getMessage());
+        } finally {
             dump = null;
-        Log.logInfo("indexContainerRAMHeap", "finished alternative rwi heap dump: " + wordcount + " words, " + urlcount + " word/URL relations in " + (System.currentTimeMillis() - startTime) + " milliseconds");
+        }
     }
     
     public int size() {

source/de/anomic/plasma/plasmaWebStructure.java

@@ -106,11 +106,11 @@ public class plasmaWebStructure {
             nexturlhash = it.next().hash();
             if (nexturlhash != null) {
                 if (nexturlhash.substring(6).equals(lhp)) {
-                    // this is a inbound link
+                    // this is a local link
                     cpl.append(nexturlhash.substring(0, 6)); // store only local part
                     LCount++;
                 } else {
-                    // this is a outbound link
+                    // this is a global link
                     cpg.append(nexturlhash); // store complete hash
                     GCount++;
                 }