moved merger object from Segment to IndexCell to enable a correct shutdown sequence. This solves a bug where yacy cannot be shut down during an index merge that appears during the shutdown phase.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7924 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/search/Segment.java

@@ -55,7 +55,6 @@ import net.yacy.kelondro.index.RowSpaceExceededException;
 import net.yacy.kelondro.logging.Log;
 import net.yacy.kelondro.order.Base64Order;
 import net.yacy.kelondro.order.ByteOrder;
-import net.yacy.kelondro.rwi.IODispatcher;
 import net.yacy.kelondro.rwi.IndexCell;
 import net.yacy.kelondro.rwi.ReferenceContainer;
 import net.yacy.kelondro.rwi.ReferenceFactory;
@@ -83,7 +82,6 @@ public class Segment {
     //private   final IndexCell<NavigationReference> authorNavIndex;
     protected final MetadataRepository             urlMetadata;
     private   final File                           segmentPath;
-    private   final IODispatcher                   merger;
 
     public Segment(
             final Log log,
@@ -101,9 +99,6 @@ public class Segment {
         this.log = log;
         this.segmentPath = segmentPath;
 
-        this.merger = new IODispatcher(1, 1, writeBufferSize);
-        this.merger.start();
-
         this.termIndex = new IndexCell<WordReference>(
                 segmentPath,
                 "text.index",
@@ -113,7 +108,6 @@ public class Segment {
                 entityCacheMaxSize,
                 targetFileSize,
                 maxFileSize,
-                this.merger,
                 writeBufferSize);
         /*
         this.authorNavIndex = new IndexCell<NavigationReference>(
@@ -258,7 +252,6 @@ public class Segment {
     }
 
     public void close() {
-        if (this.merger != null) this.merger.terminate();
         this.termIndex.close();
         this.urlMetadata.close();
     }

source/net/yacy/kelondro/rwi/IndexCell.java

@@ -85,16 +85,16 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
             final int maxRamEntries,
             final long targetFileSize,
             final long maxFileSize,
-            final IODispatcher merger,
             final int writeBufferSize
             ) throws IOException {
         super(factory);
 
-        this.array = new ReferenceContainerArray<ReferenceType>(cellPath, prefix, factory, termOrder, termSize, merger);
+        this.merger = new IODispatcher(1, 1, writeBufferSize);
+        this.array = new ReferenceContainerArray<ReferenceType>(cellPath, prefix, factory, termOrder, termSize);
         this.ram = new ReferenceContainerCache<ReferenceType>(factory, termOrder, termSize);
         this.countCache = new ComparableARC<byte[], Integer>(1000, termOrder);
         this.maxRamEntries = maxRamEntries;
-        this.merger = merger;
+        this.merger.start();
         this.lastCleanup = System.currentTimeMillis();
         this.lastDump = System.currentTimeMillis();
         this.targetFileSize = targetFileSize;
@@ -118,7 +118,6 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
             }
         }
 
-
         private void cleanCache() {
 
             // dump the cache if necessary
@@ -161,7 +160,7 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
                 synchronized (IndexCell.this.array) {
                     if (IndexCell.this.array.entries() > 50 || (IndexCell.this.lastCleanup + cleanupCycle < System.currentTimeMillis())) try {
                         IndexCell.this.lastCleanup = System.currentTimeMillis(); // set time to prevent that this is called to soon again
-                        IndexCell.this.array.shrink(IndexCell.this.targetFileSize, IndexCell.this.maxFileSize);
+                        IndexCell.this.shrink(IndexCell.this.targetFileSize, IndexCell.this.maxFileSize);
                         IndexCell.this.lastCleanup = System.currentTimeMillis(); // set again to mark end of procedure
                     } catch (final Exception e) {
                         // catch all exceptions
@@ -173,6 +172,33 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
 
     }
 
+    public boolean shrink(final long targetFileSize, final long maxFileSize) {
+        if (this.array.entries() < 2) return false;
+        boolean donesomething = false;
+
+        // first try to merge small files that match
+        while (this.merger.queueLength() < 3 || this.array.entries() >= 50) {
+            donesomething = this.array.shrinkBestSmallFiles(this.merger, targetFileSize);
+        }
+
+        // then try to merge simply any small file
+        while (this.merger.queueLength() < 2) {
+            donesomething = this.array.shrinkAnySmallFiles(this.merger, targetFileSize);
+        }
+
+        // if there is no small file, then merge matching files up to limit
+        while (this.merger.queueLength() < 1) {
+            donesomething = this.array.shrinkUpToMaxSizeFiles(this.merger, maxFileSize);
+        }
+
+        // rewrite old files (hack from sixcooler, see http://forum.yacy-websuche.de/viewtopic.php?p=15004#p15004)
+        while (this.merger.queueLength() < 1) {
+            donesomething = this.array.shrinkOldFiles(this.merger, targetFileSize);
+        }
+
+        return donesomething;
+    }
+
     /*
      * methods to implement Index
      */
@@ -508,6 +534,7 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
         // close all
         this.cleanupShallRun = false;
         if (this.cleanupThread != null) try { this.cleanupThread.join(); } catch (final InterruptedException e) {}
+        this.merger.terminate();
         this.ram.close();
         this.array.close();
     }

source/net/yacy/kelondro/rwi/ReferenceContainerArray.java

@@ -45,7 +45,6 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
 
     protected final ReferenceFactory<ReferenceType> factory;
     protected final ArrayStack array;
-    private   final IODispatcher merger;
 
     /**
      * open a index container array based on BLOB dumps. The content of the BLOBs will not be read
@@ -62,8 +61,7 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
     		final String prefix,
     		final ReferenceFactory<ReferenceType> factory,
     		final ByteOrder termOrder,
-    		final int termSize,
+    		final int termSize) throws IOException {
-    		final IODispatcher merger) throws IOException {
         this.factory = factory;
         this.array = new ArrayStack(
             heapLocation,
@@ -72,8 +70,6 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
             termSize,
             0,
             true);
-        assert merger != null;
-        this.merger = merger;
     }
 
     public void close() {
@@ -379,47 +375,36 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
         return this.array.entries();
     }
 
-    public boolean shrink(final long targetFileSize, final long maxFileSize) {
+    public boolean shrinkBestSmallFiles(final IODispatcher merger, final long targetFileSize) {
-        if (this.array.entries() < 2) return false;
+        final File[] ff = this.array.unmountBestMatch(2.0f, targetFileSize);
-        boolean donesomething = false;
+        if (ff == null) return false;
-
+        Log.logInfo("RICELL-shrink1", "unmountBestMatch(2.0, " + targetFileSize + ")");
-        // first try to merge small files that match
+        merger.merge(ff[0], ff[1], this.factory, this.array, newContainerBLOBFile());
-        while (this.merger.queueLength() < 3 || this.array.entries() >= 50) {
+        return true;
-            final File[] ff = this.array.unmountBestMatch(2.0f, targetFileSize);
+    }
-            if (ff == null) break;
-            Log.logInfo("RICELL-shrink1", "unmountBestMatch(2.0, " + targetFileSize + ")");
-            this.merger.merge(ff[0], ff[1], this.factory, this.array, newContainerBLOBFile());
-            donesomething = true;
-        }
-
-        // then try to merge simply any small file
-        while (this.merger.queueLength() < 2) {
-            final File[] ff = this.array.unmountSmallest(targetFileSize);
-            if (ff == null) break;
-            Log.logInfo("RICELL-shrink2", "unmountSmallest(" + targetFileSize + ")");
-            this.merger.merge(ff[0], ff[1], this.factory, this.array, newContainerBLOBFile());
-            donesomething = true;
-        }
 
-        // if there is no small file, then merge matching files up to limit
+    public boolean shrinkAnySmallFiles(final IODispatcher merger, final long targetFileSize) {
-        while (this.merger.queueLength() < 1) {
+        final File[] ff = this.array.unmountSmallest(targetFileSize);
-            final File[] ff = this.array.unmountBestMatch(2.0f, maxFileSize);
+        if (ff == null) return false;
-            if (ff == null) break;
+        Log.logInfo("RICELL-shrink2", "unmountSmallest(" + targetFileSize + ")");
-            Log.logInfo("RICELL-shrink3", "unmountBestMatch(2.0, " + maxFileSize + ")");
+        merger.merge(ff[0], ff[1], this.factory, this.array, newContainerBLOBFile());
-            this.merger.merge(ff[0], ff[1], this.factory, this.array, newContainerBLOBFile());
+        return true;
-            donesomething = true;
+    }
-        }
 
-        // rewrite old files (hack from sixcooler, see http://forum.yacy-websuche.de/viewtopic.php?p=15004#p15004)
+    public boolean shrinkUpToMaxSizeFiles(final IODispatcher merger, final long maxFileSize) {
-        while (this.merger.queueLength() < 1) {
+        final File[] ff = this.array.unmountBestMatch(2.0f, maxFileSize);
-            final File ff = this.array.unmountOldest();
+        if (ff == null) return false;
-            if (ff == null) break;
+        Log.logInfo("RICELL-shrink3", "unmountBestMatch(2.0, " + maxFileSize + ")");
-            Log.logInfo("RICELL-shrink4/rewrite", "unmountOldest()");
+        merger.merge(ff[0], ff[1], this.factory, this.array, newContainerBLOBFile());
-            this.merger.merge(ff, null, this.factory, this.array, newContainerBLOBFile());
+        return true;
-            donesomething = true;
+    }
-        }
 
-        return donesomething;
+    public boolean shrinkOldFiles(final IODispatcher merger, final long targetFileSize) {
+        final File ff = this.array.unmountOldest();
+        if (ff == null) return false;
+        Log.logInfo("RICELL-shrink4/rewrite", "unmountOldest()");
+        merger.merge(ff, null, this.factory, this.array, newContainerBLOBFile());
+        return true;
     }
 
     public static <ReferenceType extends Reference> HandleMap referenceHashes(