diff --git a/htroot/ContentAnalysis_p.java b/htroot/ContentAnalysis_p.java index 7d5649553..2ba573ab0 100644 --- a/htroot/ContentAnalysis_p.java +++ b/htroot/ContentAnalysis_p.java @@ -34,7 +34,7 @@ public class ContentAnalysis_p { // clean up all search events SearchEventCache.cleanupEvents(true); - sb.index.fulltext().clearCache(); // every time the ranking is changed we need to remove old orderings + sb.index.clearCache(); // every time the ranking is changed we need to remove old orderings if (post != null && post.containsKey("EnterDoublecheck")) { Ranking.setMinTokenLen(post.getInt("minTokenLen", 3)); diff --git a/htroot/RankingSolr_p.java b/htroot/RankingSolr_p.java index f47c0784e..2184efe5d 100644 --- a/htroot/RankingSolr_p.java +++ b/htroot/RankingSolr_p.java @@ -38,7 +38,7 @@ public class RankingSolr_p { // clean up all search events SearchEventCache.cleanupEvents(true); - sb.index.fulltext().clearCache(); // every time the ranking is changed we need to remove old orderings + sb.index.clearCache(); // every time the ranking is changed we need to remove old orderings if (post != null && post.containsKey("EnterBoosts")) { StringBuilder boostString = new StringBuilder(); // SwitchboardConstants.SEARCH_RANKING_SOLR_BOOST; diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index 7ba0cc135..cfcd28a8d 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -351,7 +351,7 @@ public class yacysearch { // check available memory and clean up if necessary if ( !MemoryControl.request(8000000L, false) ) { - indexSegment.fulltext().clearCache(); + indexSegment.clearCache(); SearchEventCache.cleanupEvents(false); } diff --git a/source/net/yacy/kelondro/rwi/IndexCell.java b/source/net/yacy/kelondro/rwi/IndexCell.java index ed9554cb6..ca61b9c91 100644 --- a/source/net/yacy/kelondro/rwi/IndexCell.java +++ b/source/net/yacy/kelondro/rwi/IndexCell.java @@ -75,8 +75,8 @@ public final class IndexCell extends AbstractBu private final long targetFileSize, maxFileSize; private final int writeBufferSize; private final Map removeDelayedURLs; // mapping from word hashes to a list of url hashes - private boolean cleanupShallRun; - private final Thread cleanupThread; + private boolean flushShallRun; + private final Thread flushThread; public IndexCell( final File cellPath, @@ -103,17 +103,17 @@ public final class IndexCell extends AbstractBu this.maxFileSize = maxFileSize; this.writeBufferSize = writeBufferSize; this.removeDelayedURLs = new TreeMap(URIMetadataRow.rowdef.objectOrder); - this.cleanupShallRun = true; - this.cleanupThread = new CleanupThread(); - this.cleanupThread.start(); + this.flushShallRun = true; + this.flushThread = new FlushThread(); + this.flushThread.start(); } - private class CleanupThread extends Thread { + private class FlushThread extends Thread { @Override public void run() { - while (IndexCell.this.cleanupShallRun) { + while (IndexCell.this.flushShallRun) { try { - cleanCache(); + flushBuffer(); } catch (final Throwable e) { Log.logException(e); } @@ -121,7 +121,7 @@ public final class IndexCell extends AbstractBu } } - private void cleanCache() { + private void flushBuffer() { // dump the cache if necessary final long t = System.currentTimeMillis(); @@ -542,6 +542,10 @@ public final class IndexCell extends AbstractBu Switchboard.getSwitchboard().peers != null && Switchboard.getSwitchboard().peers.mySeed() != null) Switchboard.getSwitchboard().peers.mySeed().resetCounters(); } + + public synchronized void clearCache() { + this.countCache.clear(); + } /** * when a cell is closed, the current RAM is dumped to a file which will be opened as @@ -554,8 +558,8 @@ public final class IndexCell extends AbstractBu try {removeDelayed();} catch (final IOException e) {} if (!this.ram.isEmpty()) this.ram.dump(this.array.newContainerBLOBFile(), (int) Math.min(MemoryControl.available() / 3, this.writeBufferSize), true); // close all - this.cleanupShallRun = false; - if (this.cleanupThread != null) try { this.cleanupThread.join(); } catch (final InterruptedException e) {} + this.flushShallRun = false; + if (this.flushThread != null) try { this.flushThread.join(); } catch (final InterruptedException e) {} this.merger.terminate(); this.ram.close(); this.array.close(); diff --git a/source/net/yacy/search/ResourceObserver.java b/source/net/yacy/search/ResourceObserver.java index 24727c5a5..806c09622 100644 --- a/source/net/yacy/search/ResourceObserver.java +++ b/source/net/yacy/search/ResourceObserver.java @@ -89,7 +89,7 @@ public class ResourceObserver { } if (this.normalizedMemoryFree.compareTo(Space.HIGH) < 0 ) { // clear some caches - @all: are there more of these, we could clear here? - this.sb.index.fulltext().clearCache(); + this.sb.index.clearCache(); SearchEventCache.cleanupEvents(true); this.sb.trail.clear(); Switchboard.urlBlacklist.clearblacklistCache(); diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index fe4835b7f..df543f3a3 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -82,6 +82,7 @@ import java.util.zip.GZIPOutputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; +import org.apache.pdfbox.pdmodel.font.PDFont; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrInputDocument; @@ -2042,20 +2043,24 @@ public final class Switchboard extends serverSwitch { public boolean cleanupJob() { - if (MemoryControl.shortStatus()) { + try { + // flush caches in used libraries + PDFont.clearResources(); // eats up megabytes, see http://markmail.org/thread/quk5odee4hbsauhu + + // clear caches WordCache.clear(); Domains.clear(); - Digest.cleanup(); - } - - try { + + // clean up image stack + ResultImages.clearQueues(); + // flush the document compressor cache Cache.commit(); Digest.cleanup(); // don't let caches become permanent memory leaks // clear caches if necessary - if ( !MemoryControl.request(8000000L, false) ) { - this.index.fulltext().clearCache(); + if ( !MemoryControl.request(128000000L, false) ) { + this.index.clearCache(); SearchEventCache.cleanupEvents(false); this.trail.clear(); } @@ -2152,9 +2157,6 @@ public final class Switchboard extends serverSwitch { } } - // clean up image stack - ResultImages.clearQueues(); - // clean up profiles checkInterruption(); //cleanProfiles(); diff --git a/source/net/yacy/search/index/Segment.java b/source/net/yacy/search/index/Segment.java index c951e973e..b2e1bc7a2 100644 --- a/source/net/yacy/search/index/Segment.java +++ b/source/net/yacy/search/index/Segment.java @@ -296,6 +296,12 @@ public class Segment { Log.logException(e); } } + + public void clearCache() { + if (this.urlCitationIndex != null) this.urlCitationIndex.clearCache(); + if (this.termIndex != null) this.termIndex.clearCache(); + this.fulltext.clearCache(); + } public File getLocation() { return this.segmentPath;