reduced memory footprint during indexing/crawling

pull/436/head
admin 3 years ago
parent fbf8ddd32d
commit 9b7668fa58

@ -788,7 +788,7 @@ javastart_priority=10
# wordCacheMaxLow/High is the number of word indexes that shall be held in the # wordCacheMaxLow/High is the number of word indexes that shall be held in the
# ram cache during indexing. If you want to increase indexing speed, increase this # ram cache during indexing. If you want to increase indexing speed, increase this
# value i.e. up to one million, but increase also the memory limit to a minimum of 2GB # value i.e. up to one million, but increase also the memory limit to a minimum of 2GB
wordCacheMaxCount = 50000 wordCacheMaxCount = 20000
# Specifies if yacy can be used as transparent http proxy. # Specifies if yacy can be used as transparent http proxy.
# #

@ -56,7 +56,7 @@
</div> </div>
<div class="form-group"> <div class="form-group">
<label for="diskFreeHardlimit">Absolute minimum</label> <label for="diskFreeHardlimit">Absolute minimum</label>
<input name="diskFreeHardlimit" id="diskFreeHardlimit" type="text" size="4" value="#[diskFreeHardlimit]#" title="Amount of space (in Mebibytes) that should at least be kept free as hard limit"/> <abbr title="Mebibyte">MiB</abbr>. Disable <abbr title="Distributed Hash Table">DHT</abbr>-in when free space is below. <input name="diskFreeHardlimit" id="diskFreeHardlimit" type="text" size="4" value="#[diskFreeHardlimit]#" title="Amount of space (in Megabytes) that should at least be kept free as hard limit"/> <abbr title="Mebibyte">MiB</abbr>. Disable <abbr title="Distributed Hash Table">DHT</abbr>-in when free space is below.
</div> </div>
<div class="form-group"> <div class="form-group">
<label for="diskFreeAutoregulate">Autoregulate</label> <label for="diskFreeAutoregulate">Autoregulate</label>

@ -65,7 +65,7 @@ import net.yacy.search.Switchboard;
public final class IndexCell<ReferenceType extends Reference> extends AbstractBufferedIndex<ReferenceType> implements BufferedIndex<ReferenceType>, Iterable<ReferenceContainer<ReferenceType>> { public final class IndexCell<ReferenceType extends Reference> extends AbstractBufferedIndex<ReferenceType> implements BufferedIndex<ReferenceType>, Iterable<ReferenceContainer<ReferenceType>> {
private static final long cleanupCycle = 60000; private static final long cleanupCycle = 60000;
private static final long dumpCycle = 600000; private static final long dumpCycle = 300000;
// class variables // class variables
private final ReferenceContainerArray<ReferenceType> array; private final ReferenceContainerArray<ReferenceType> array;
@ -115,7 +115,7 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
public FlushThread(String name) { public FlushThread(String name) {
super("IndexCell.FlushThread(" + name + ")"); super("IndexCell.FlushThread(" + name + ")");
} }
@Override @Override
public void run() { public void run() {
while (IndexCell.this.flushShallRun) { while (IndexCell.this.flushShallRun) {
@ -133,7 +133,7 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
// dump the cache if necessary // dump the cache if necessary
final long t = System.currentTimeMillis(); final long t = System.currentTimeMillis();
if ((IndexCell.this.ram.size() >= IndexCell.this.maxRamEntries || if ((IndexCell.this.ram.size() >= IndexCell.this.maxRamEntries ||
(IndexCell.this.ram.size() > 3000 && !MemoryControl.request(80L * 1024L * 1024L, false)) || (IndexCell.this.ram.size() > 2000 && !MemoryControl.request(120L * 1024L * 1024L, false)) ||
(!IndexCell.this.ram.isEmpty() && IndexCell.this.lastDump + dumpCycle < t))) { (!IndexCell.this.ram.isEmpty() && IndexCell.this.lastDump + dumpCycle < t))) {
synchronized (IndexCell.this.merger) { synchronized (IndexCell.this.merger) {
if (IndexCell.this.ram.size() >= IndexCell.this.maxRamEntries || if (IndexCell.this.ram.size() >= IndexCell.this.maxRamEntries ||

Loading…
Cancel
Save