reduced memory footprint during indexing/crawling

pull/436/head
admin 3 years ago
parent fbf8ddd32d
commit 9b7668fa58

@ -788,7 +788,7 @@ javastart_priority=10
# wordCacheMaxLow/High is the number of word indexes that shall be held in the
# ram cache during indexing. If you want to increase indexing speed, increase this
# value i.e. up to one million, but increase also the memory limit to a minimum of 2GB
wordCacheMaxCount = 50000
wordCacheMaxCount = 20000
# Specifies if yacy can be used as transparent http proxy.
#

@ -56,7 +56,7 @@
</div>
<div class="form-group">
<label for="diskFreeHardlimit">Absolute minimum</label>
<input name="diskFreeHardlimit" id="diskFreeHardlimit" type="text" size="4" value="#[diskFreeHardlimit]#" title="Amount of space (in Mebibytes) that should at least be kept free as hard limit"/> <abbr title="Mebibyte">MiB</abbr>. Disable <abbr title="Distributed Hash Table">DHT</abbr>-in when free space is below.
<input name="diskFreeHardlimit" id="diskFreeHardlimit" type="text" size="4" value="#[diskFreeHardlimit]#" title="Amount of space (in Megabytes) that should at least be kept free as hard limit"/> <abbr title="Mebibyte">MiB</abbr>. Disable <abbr title="Distributed Hash Table">DHT</abbr>-in when free space is below.
</div>
<div class="form-group">
<label for="diskFreeAutoregulate">Autoregulate</label>

@ -65,7 +65,7 @@ import net.yacy.search.Switchboard;
public final class IndexCell<ReferenceType extends Reference> extends AbstractBufferedIndex<ReferenceType> implements BufferedIndex<ReferenceType>, Iterable<ReferenceContainer<ReferenceType>> {
private static final long cleanupCycle = 60000;
private static final long dumpCycle = 600000;
private static final long dumpCycle = 300000;
// class variables
private final ReferenceContainerArray<ReferenceType> array;
@ -115,7 +115,7 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
public FlushThread(String name) {
super("IndexCell.FlushThread(" + name + ")");
}
@Override
public void run() {
while (IndexCell.this.flushShallRun) {
@ -133,7 +133,7 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
// dump the cache if necessary
final long t = System.currentTimeMillis();
if ((IndexCell.this.ram.size() >= IndexCell.this.maxRamEntries ||
(IndexCell.this.ram.size() > 3000 && !MemoryControl.request(80L * 1024L * 1024L, false)) ||
(IndexCell.this.ram.size() > 2000 && !MemoryControl.request(120L * 1024L * 1024L, false)) ||
(!IndexCell.this.ram.isEmpty() && IndexCell.this.lastDump + dumpCycle < t))) {
synchronized (IndexCell.this.merger) {
if (IndexCell.this.ram.size() >= IndexCell.this.maxRamEntries ||

Loading…
Cancel
Save