From e1a3d609aab6e327f9aa43308e74da7fe0d70db7 Mon Sep 17 00:00:00 2001 From: orbiter Date: Sun, 4 Sep 2011 23:27:12 +0000 Subject: [PATCH] moved merger object from Segment to IndexCell to enable a correct shutdown sequence. This solves a bug where yacy cannot be shut down during an index merge that appears during the shutdown phase. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7924 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/de/anomic/search/Segment.java | 7 -- source/net/yacy/kelondro/rwi/IndexCell.java | 37 ++++++++-- .../kelondro/rwi/ReferenceContainerArray.java | 71 ++++++++----------- 3 files changed, 60 insertions(+), 55 deletions(-) diff --git a/source/de/anomic/search/Segment.java b/source/de/anomic/search/Segment.java index 7ac19d40f..863264750 100644 --- a/source/de/anomic/search/Segment.java +++ b/source/de/anomic/search/Segment.java @@ -55,7 +55,6 @@ import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.ByteOrder; -import net.yacy.kelondro.rwi.IODispatcher; import net.yacy.kelondro.rwi.IndexCell; import net.yacy.kelondro.rwi.ReferenceContainer; import net.yacy.kelondro.rwi.ReferenceFactory; @@ -83,7 +82,6 @@ public class Segment { //private final IndexCell authorNavIndex; protected final MetadataRepository urlMetadata; private final File segmentPath; - private final IODispatcher merger; public Segment( final Log log, @@ -101,9 +99,6 @@ public class Segment { this.log = log; this.segmentPath = segmentPath; - this.merger = new IODispatcher(1, 1, writeBufferSize); - this.merger.start(); - this.termIndex = new IndexCell( segmentPath, "text.index", @@ -113,7 +108,6 @@ public class Segment { entityCacheMaxSize, targetFileSize, maxFileSize, - this.merger, writeBufferSize); /* this.authorNavIndex = new IndexCell( @@ -258,7 +252,6 @@ public class Segment { } public void close() { - if (this.merger != null) this.merger.terminate(); this.termIndex.close(); this.urlMetadata.close(); } diff --git a/source/net/yacy/kelondro/rwi/IndexCell.java b/source/net/yacy/kelondro/rwi/IndexCell.java index 0f80d1b32..1a7a4a9d0 100644 --- a/source/net/yacy/kelondro/rwi/IndexCell.java +++ b/source/net/yacy/kelondro/rwi/IndexCell.java @@ -85,16 +85,16 @@ public final class IndexCell extends AbstractBu final int maxRamEntries, final long targetFileSize, final long maxFileSize, - final IODispatcher merger, final int writeBufferSize ) throws IOException { super(factory); - this.array = new ReferenceContainerArray(cellPath, prefix, factory, termOrder, termSize, merger); + this.merger = new IODispatcher(1, 1, writeBufferSize); + this.array = new ReferenceContainerArray(cellPath, prefix, factory, termOrder, termSize); this.ram = new ReferenceContainerCache(factory, termOrder, termSize); this.countCache = new ComparableARC(1000, termOrder); this.maxRamEntries = maxRamEntries; - this.merger = merger; + this.merger.start(); this.lastCleanup = System.currentTimeMillis(); this.lastDump = System.currentTimeMillis(); this.targetFileSize = targetFileSize; @@ -118,7 +118,6 @@ public final class IndexCell extends AbstractBu } } - private void cleanCache() { // dump the cache if necessary @@ -161,7 +160,7 @@ public final class IndexCell extends AbstractBu synchronized (IndexCell.this.array) { if (IndexCell.this.array.entries() > 50 || (IndexCell.this.lastCleanup + cleanupCycle < System.currentTimeMillis())) try { IndexCell.this.lastCleanup = System.currentTimeMillis(); // set time to prevent that this is called to soon again - IndexCell.this.array.shrink(IndexCell.this.targetFileSize, IndexCell.this.maxFileSize); + IndexCell.this.shrink(IndexCell.this.targetFileSize, IndexCell.this.maxFileSize); IndexCell.this.lastCleanup = System.currentTimeMillis(); // set again to mark end of procedure } catch (final Exception e) { // catch all exceptions @@ -173,6 +172,33 @@ public final class IndexCell extends AbstractBu } + public boolean shrink(final long targetFileSize, final long maxFileSize) { + if (this.array.entries() < 2) return false; + boolean donesomething = false; + + // first try to merge small files that match + while (this.merger.queueLength() < 3 || this.array.entries() >= 50) { + donesomething = this.array.shrinkBestSmallFiles(this.merger, targetFileSize); + } + + // then try to merge simply any small file + while (this.merger.queueLength() < 2) { + donesomething = this.array.shrinkAnySmallFiles(this.merger, targetFileSize); + } + + // if there is no small file, then merge matching files up to limit + while (this.merger.queueLength() < 1) { + donesomething = this.array.shrinkUpToMaxSizeFiles(this.merger, maxFileSize); + } + + // rewrite old files (hack from sixcooler, see http://forum.yacy-websuche.de/viewtopic.php?p=15004#p15004) + while (this.merger.queueLength() < 1) { + donesomething = this.array.shrinkOldFiles(this.merger, targetFileSize); + } + + return donesomething; + } + /* * methods to implement Index */ @@ -508,6 +534,7 @@ public final class IndexCell extends AbstractBu // close all this.cleanupShallRun = false; if (this.cleanupThread != null) try { this.cleanupThread.join(); } catch (final InterruptedException e) {} + this.merger.terminate(); this.ram.close(); this.array.close(); } diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java index 0bf8834e1..079aeac19 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java @@ -45,7 +45,6 @@ public final class ReferenceContainerArray { protected final ReferenceFactory factory; protected final ArrayStack array; - private final IODispatcher merger; /** * open a index container array based on BLOB dumps. The content of the BLOBs will not be read @@ -62,8 +61,7 @@ public final class ReferenceContainerArray { final String prefix, final ReferenceFactory factory, final ByteOrder termOrder, - final int termSize, - final IODispatcher merger) throws IOException { + final int termSize) throws IOException { this.factory = factory; this.array = new ArrayStack( heapLocation, @@ -72,8 +70,6 @@ public final class ReferenceContainerArray { termSize, 0, true); - assert merger != null; - this.merger = merger; } public void close() { @@ -379,47 +375,36 @@ public final class ReferenceContainerArray { return this.array.entries(); } - public boolean shrink(final long targetFileSize, final long maxFileSize) { - if (this.array.entries() < 2) return false; - boolean donesomething = false; - - // first try to merge small files that match - while (this.merger.queueLength() < 3 || this.array.entries() >= 50) { - final File[] ff = this.array.unmountBestMatch(2.0f, targetFileSize); - if (ff == null) break; - Log.logInfo("RICELL-shrink1", "unmountBestMatch(2.0, " + targetFileSize + ")"); - this.merger.merge(ff[0], ff[1], this.factory, this.array, newContainerBLOBFile()); - donesomething = true; - } - - // then try to merge simply any small file - while (this.merger.queueLength() < 2) { - final File[] ff = this.array.unmountSmallest(targetFileSize); - if (ff == null) break; - Log.logInfo("RICELL-shrink2", "unmountSmallest(" + targetFileSize + ")"); - this.merger.merge(ff[0], ff[1], this.factory, this.array, newContainerBLOBFile()); - donesomething = true; - } + public boolean shrinkBestSmallFiles(final IODispatcher merger, final long targetFileSize) { + final File[] ff = this.array.unmountBestMatch(2.0f, targetFileSize); + if (ff == null) return false; + Log.logInfo("RICELL-shrink1", "unmountBestMatch(2.0, " + targetFileSize + ")"); + merger.merge(ff[0], ff[1], this.factory, this.array, newContainerBLOBFile()); + return true; + } - // if there is no small file, then merge matching files up to limit - while (this.merger.queueLength() < 1) { - final File[] ff = this.array.unmountBestMatch(2.0f, maxFileSize); - if (ff == null) break; - Log.logInfo("RICELL-shrink3", "unmountBestMatch(2.0, " + maxFileSize + ")"); - this.merger.merge(ff[0], ff[1], this.factory, this.array, newContainerBLOBFile()); - donesomething = true; - } + public boolean shrinkAnySmallFiles(final IODispatcher merger, final long targetFileSize) { + final File[] ff = this.array.unmountSmallest(targetFileSize); + if (ff == null) return false; + Log.logInfo("RICELL-shrink2", "unmountSmallest(" + targetFileSize + ")"); + merger.merge(ff[0], ff[1], this.factory, this.array, newContainerBLOBFile()); + return true; + } - // rewrite old files (hack from sixcooler, see http://forum.yacy-websuche.de/viewtopic.php?p=15004#p15004) - while (this.merger.queueLength() < 1) { - final File ff = this.array.unmountOldest(); - if (ff == null) break; - Log.logInfo("RICELL-shrink4/rewrite", "unmountOldest()"); - this.merger.merge(ff, null, this.factory, this.array, newContainerBLOBFile()); - donesomething = true; - } + public boolean shrinkUpToMaxSizeFiles(final IODispatcher merger, final long maxFileSize) { + final File[] ff = this.array.unmountBestMatch(2.0f, maxFileSize); + if (ff == null) return false; + Log.logInfo("RICELL-shrink3", "unmountBestMatch(2.0, " + maxFileSize + ")"); + merger.merge(ff[0], ff[1], this.factory, this.array, newContainerBLOBFile()); + return true; + } - return donesomething; + public boolean shrinkOldFiles(final IODispatcher merger, final long targetFileSize) { + final File ff = this.array.unmountOldest(); + if (ff == null) return false; + Log.logInfo("RICELL-shrink4/rewrite", "unmountOldest()"); + merger.merge(ff, null, this.factory, this.array, newContainerBLOBFile()); + return true; } public static HandleMap referenceHashes(