boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

moved merger object from Segment to IndexCell to enable a correct shutdown sequence. This solves a bug where yacy cannot be shut down during an index merge that appears during the shutdown phase.

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7924 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent 656286347e
commit e1a3d609aa
3 changed files with 60 additions and 55 deletions
Show Stats Download Patch File Download Diff File

7
source/de/anomic/search/Segment.java
Unescape View File

@ -55,7 +55,6 @@ import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.order.ByteOrder;
import net.yacy.kelondro.rwi.IODispatcher;
import net.yacy.kelondro.rwi.IndexCell;
import net.yacy.kelondro.rwi.ReferenceContainer;
import net.yacy.kelondro.rwi.ReferenceFactory;
@ -83,7 +82,6 @@ public class Segment {
//private final IndexCell<NavigationReference> authorNavIndex;
protected final MetadataRepository urlMetadata;
private final File segmentPath;
private final IODispatcher merger;
public Segment(
final Log log,
@ -101,9 +99,6 @@ public class Segment {
this.log = log;
this.segmentPath = segmentPath;
this.merger = new IODispatcher(1, 1, writeBufferSize);
this.merger.start();
this.termIndex = new IndexCell<WordReference>(
segmentPath,
"text.index",
@ -113,7 +108,6 @@ public class Segment {
entityCacheMaxSize,
targetFileSize,
maxFileSize,
this.merger,
writeBufferSize);
/*
this.authorNavIndex = new IndexCell<NavigationReference>(
@ -258,7 +252,6 @@ public class Segment {
}
public void close() {
if (this.merger != null) this.merger.terminate();
this.termIndex.close();
this.urlMetadata.close();
}

37
source/net/yacy/kelondro/rwi/IndexCell.java
Unescape View File

@ -85,16 +85,16 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
final int maxRamEntries,
final long targetFileSize,
final long maxFileSize,
final IODispatcher merger,
final int writeBufferSize
) throws IOException {
super(factory);
this.array = new ReferenceContainerArray<ReferenceType>(cellPath, prefix, factory, termOrder, termSize, merger);
this.merger = new IODispatcher(1, 1, writeBufferSize);
this.array = new ReferenceContainerArray<ReferenceType>(cellPath, prefix, factory, termOrder, termSize);
this.ram = new ReferenceContainerCache<ReferenceType>(factory, termOrder, termSize);
this.countCache = new ComparableARC<byte[], Integer>(1000, termOrder);
this.maxRamEntries = maxRamEntries;
this.merger = merger;
this.merger.start();
this.lastCleanup = System.currentTimeMillis();
this.lastDump = System.currentTimeMillis();
this.targetFileSize = targetFileSize;
@ -118,7 +118,6 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
}
}
private void cleanCache() {
// dump the cache if necessary
@ -161,7 +160,7 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
synchronized (IndexCell.this.array) {
if (IndexCell.this.array.entries() > 50 || (IndexCell.this.lastCleanup + cleanupCycle < System.currentTimeMillis())) try {
IndexCell.this.lastCleanup = System.currentTimeMillis(); // set time to prevent that this is called to soon again
IndexCell.this.array.shrink(IndexCell.this.targetFileSize, IndexCell.this.maxFileSize);
IndexCell.this.shrink(IndexCell.this.targetFileSize, IndexCell.this.maxFileSize);
IndexCell.this.lastCleanup = System.currentTimeMillis(); // set again to mark end of procedure
} catch (final Exception e) {
// catch all exceptions
@ -173,6 +172,33 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
}
public boolean shrink(final long targetFileSize, final long maxFileSize) {
if (this.array.entries() < 2) return false;
boolean donesomething = false;
// first try to merge small files that match
while (this.merger.queueLength() < 3 || this.array.entries() >= 50) {
donesomething = this.array.shrinkBestSmallFiles(this.merger, targetFileSize);
}
// then try to merge simply any small file
while (this.merger.queueLength() < 2) {
donesomething = this.array.shrinkAnySmallFiles(this.merger, targetFileSize);
}
// if there is no small file, then merge matching files up to limit
while (this.merger.queueLength() < 1) {
donesomething = this.array.shrinkUpToMaxSizeFiles(this.merger, maxFileSize);
}
// rewrite old files (hack from sixcooler, see http://forum.yacy-websuche.de/viewtopic.php?p=15004#p15004)
while (this.merger.queueLength() < 1) {
donesomething = this.array.shrinkOldFiles(this.merger, targetFileSize);
}
return donesomething;
}
/*
* methods to implement Index
*/
@ -508,6 +534,7 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
// close all
this.cleanupShallRun = false;
if (this.cleanupThread != null) try { this.cleanupThread.join(); } catch (final InterruptedException e) {}
this.merger.terminate();
this.ram.close();
this.array.close();
}

71
source/net/yacy/kelondro/rwi/ReferenceContainerArray.java
Unescape View File

@ -45,7 +45,6 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
protected final ReferenceFactory<ReferenceType> factory;
protected final ArrayStack array;
private final IODispatcher merger;
/**
* open a index container array based on BLOB dumps. The content of the BLOBs will not be read
@ -62,8 +61,7 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
final String prefix,
final ReferenceFactory<ReferenceType> factory,
final ByteOrder termOrder,
final int termSize,
final IODispatcher merger) throws IOException {
final int termSize) throws IOException {
this.factory = factory;
this.array = new ArrayStack(
heapLocation,
@ -72,8 +70,6 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
termSize,
0,
true);
assert merger != null;
this.merger = merger;
}
public void close() {
@ -379,47 +375,36 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
return this.array.entries();
}
public boolean shrink(final long targetFileSize, final long maxFileSize) {
if (this.array.entries() < 2) return false;
boolean donesomething = false;
// first try to merge small files that match
while (this.merger.queueLength() < 3 || this.array.entries() >= 50) {
final File[] ff = this.array.unmountBestMatch(2.0f, targetFileSize);
if (ff == null) break;
Log.logInfo("RICELL-shrink1", "unmountBestMatch(2.0, " + targetFileSize + ")");
this.merger.merge(ff[0], ff[1], this.factory, this.array, newContainerBLOBFile());
donesomething = true;
}
// then try to merge simply any small file
while (this.merger.queueLength() < 2) {
final File[] ff = this.array.unmountSmallest(targetFileSize);
if (ff == null) break;
Log.logInfo("RICELL-shrink2", "unmountSmallest(" + targetFileSize + ")");
this.merger.merge(ff[0], ff[1], this.factory, this.array, newContainerBLOBFile());
donesomething = true;
}
public boolean shrinkBestSmallFiles(final IODispatcher merger, final long targetFileSize) {
final File[] ff = this.array.unmountBestMatch(2.0f, targetFileSize);
if (ff == null) return false;
Log.logInfo("RICELL-shrink1", "unmountBestMatch(2.0, " + targetFileSize + ")");
merger.merge(ff[0], ff[1], this.factory, this.array, newContainerBLOBFile());
return true;
}
// if there is no small file, then merge matching files up to limit
while (this.merger.queueLength() < 1) {
final File[] ff = this.array.unmountBestMatch(2.0f, maxFileSize);
if (ff == null) break;
Log.logInfo("RICELL-shrink3", "unmountBestMatch(2.0, " + maxFileSize + ")");
this.merger.merge(ff[0], ff[1], this.factory, this.array, newContainerBLOBFile());
donesomething = true;
}
public boolean shrinkAnySmallFiles(final IODispatcher merger, final long targetFileSize) {
final File[] ff = this.array.unmountSmallest(targetFileSize);
if (ff == null) return false;
Log.logInfo("RICELL-shrink2", "unmountSmallest(" + targetFileSize + ")");
merger.merge(ff[0], ff[1], this.factory, this.array, newContainerBLOBFile());
return true;
}
// rewrite old files (hack from sixcooler, see http://forum.yacy-websuche.de/viewtopic.php?p=15004#p15004)
while (this.merger.queueLength() < 1) {
final File ff = this.array.unmountOldest();
if (ff == null) break;
Log.logInfo("RICELL-shrink4/rewrite", "unmountOldest()");
this.merger.merge(ff, null, this.factory, this.array, newContainerBLOBFile());
donesomething = true;
}
public boolean shrinkUpToMaxSizeFiles(final IODispatcher merger, final long maxFileSize) {
final File[] ff = this.array.unmountBestMatch(2.0f, maxFileSize);
if (ff == null) return false;
Log.logInfo("RICELL-shrink3", "unmountBestMatch(2.0, " + maxFileSize + ")");
merger.merge(ff[0], ff[1], this.factory, this.array, newContainerBLOBFile());
return true;
}
return donesomething;
public boolean shrinkOldFiles(final IODispatcher merger, final long targetFileSize) {
final File ff = this.array.unmountOldest();
if (ff == null) return false;
Log.logInfo("RICELL-shrink4/rewrite", "unmountOldest()");
merger.merge(ff, null, this.factory, this.array, newContainerBLOBFile());
return true;
}
public static <ReferenceType extends Reference> HandleMap referenceHashes(

Write Preview
Loading…
Cancel
Save