fix for possible memory leaks

htroot/ContentAnalysis_p.java

@@ -34,7 +34,7 @@ public class ContentAnalysis_p {
 
         // clean up all search events
         SearchEventCache.cleanupEvents(true);
-        sb.index.fulltext().clearCache(); // every time the ranking is changed we need to remove old orderings
+        sb.index.clearCache(); // every time the ranking is changed we need to remove old orderings
 
         if (post != null && post.containsKey("EnterDoublecheck")) {
             Ranking.setMinTokenLen(post.getInt("minTokenLen", 3));

htroot/RankingSolr_p.java

@@ -38,7 +38,7 @@ public class RankingSolr_p {
 
         // clean up all search events
         SearchEventCache.cleanupEvents(true);
-        sb.index.fulltext().clearCache(); // every time the ranking is changed we need to remove old orderings
+        sb.index.clearCache(); // every time the ranking is changed we need to remove old orderings
         
         if (post != null && post.containsKey("EnterBoosts")) {
             StringBuilder boostString = new StringBuilder(); // SwitchboardConstants.SEARCH_RANKING_SOLR_BOOST;

htroot/yacysearch.java

@@ -351,7 +351,7 @@ public class yacysearch {
 
             // check available memory and clean up if necessary
             if ( !MemoryControl.request(8000000L, false) ) {
-                indexSegment.fulltext().clearCache();
+                indexSegment.clearCache();
                 SearchEventCache.cleanupEvents(false);
             }
 

source/net/yacy/kelondro/rwi/IndexCell.java

@@ -75,8 +75,8 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
     private final long                                   targetFileSize, maxFileSize;
     private final int                                    writeBufferSize;
     private final Map<byte[], HandleSet>                 removeDelayedURLs; // mapping from word hashes to a list of url hashes
-    private       boolean                                cleanupShallRun;
-    private final Thread                                 cleanupThread;
+    private       boolean                                flushShallRun;
+    private final Thread                                 flushThread;
 
     public IndexCell(
             final File cellPath,
@@ -103,17 +103,17 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
         this.maxFileSize = maxFileSize;
         this.writeBufferSize = writeBufferSize;
         this.removeDelayedURLs = new TreeMap<byte[], HandleSet>(URIMetadataRow.rowdef.objectOrder);
-        this.cleanupShallRun = true;
-        this.cleanupThread = new CleanupThread();
-        this.cleanupThread.start();
+        this.flushShallRun = true;
+        this.flushThread = new FlushThread();
+        this.flushThread.start();
     }
 
-    private class CleanupThread extends Thread {
+    private class FlushThread extends Thread {
         @Override
         public void run() {
-            while (IndexCell.this.cleanupShallRun) {
+            while (IndexCell.this.flushShallRun) {
                 try {
-                    cleanCache();
+                    flushBuffer();
                 } catch (final Throwable e) {
                     Log.logException(e);
                 }
@@ -121,7 +121,7 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
             }
         }
 
-        private void cleanCache() {
+        private void flushBuffer() {
 
             // dump the cache if necessary
             final long t = System.currentTimeMillis();
@@ -542,6 +542,10 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
                 Switchboard.getSwitchboard().peers != null &&
                 Switchboard.getSwitchboard().peers.mySeed() != null) Switchboard.getSwitchboard().peers.mySeed().resetCounters();
     }
+    
+    public synchronized void clearCache() {
+        this.countCache.clear();
+    }
 
     /**
      * when a cell is closed, the current RAM is dumped to a file which will be opened as
@@ -554,8 +558,8 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
         try {removeDelayed();} catch (final IOException e) {}
         if (!this.ram.isEmpty()) this.ram.dump(this.array.newContainerBLOBFile(), (int) Math.min(MemoryControl.available() / 3, this.writeBufferSize), true);
         // close all
-        this.cleanupShallRun = false;
-        if (this.cleanupThread != null) try { this.cleanupThread.join(); } catch (final InterruptedException e) {}
+        this.flushShallRun = false;
+        if (this.flushThread != null) try { this.flushThread.join(); } catch (final InterruptedException e) {}
         this.merger.terminate();
         this.ram.close();
         this.array.close();

source/net/yacy/search/ResourceObserver.java

@@ -89,7 +89,7 @@ public class ResourceObserver {
     		}
     		if (this.normalizedMemoryFree.compareTo(Space.HIGH) < 0 ) {
     			// clear some caches - @all: are there more of these, we could clear here?
-    			this.sb.index.fulltext().clearCache();
+    			this.sb.index.clearCache();
                 SearchEventCache.cleanupEvents(true);
                 this.sb.trail.clear();
                 Switchboard.urlBlacklist.clearblacklistCache();

source/net/yacy/search/Switchboard.java

@@ -82,6 +82,7 @@ import java.util.zip.GZIPOutputStream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
+import org.apache.pdfbox.pdmodel.font.PDFont;
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrInputDocument;
 
@@ -2042,20 +2043,24 @@ public final class Switchboard extends serverSwitch {
 
     public boolean cleanupJob() {
         
-        if (MemoryControl.shortStatus()) {
+        try {
+            // flush caches in used libraries
+            PDFont.clearResources(); // eats up megabytes, see http://markmail.org/thread/quk5odee4hbsauhu
+            
+            // clear caches
             WordCache.clear();
             Domains.clear();
-            Digest.cleanup();
-        }
-        
-        try {
+            
+            // clean up image stack
+            ResultImages.clearQueues();
+            
         	// flush the document compressor cache
         	Cache.commit();
         	Digest.cleanup(); // don't let caches become permanent memory leaks
 
             // clear caches if necessary
-            if ( !MemoryControl.request(8000000L, false) ) {
-                this.index.fulltext().clearCache();
+            if ( !MemoryControl.request(128000000L, false) ) {
+                this.index.clearCache();
                 SearchEventCache.cleanupEvents(false);
                 this.trail.clear();
             }
@@ -2152,9 +2157,6 @@ public final class Switchboard extends serverSwitch {
                 }
             }
 
-            // clean up image stack
-            ResultImages.clearQueues();
-
             // clean up profiles
             checkInterruption();
             //cleanProfiles();

source/net/yacy/search/index/Segment.java

@@ -296,6 +296,12 @@ public class Segment {
             Log.logException(e);
         }
     }
+    
+    public void clearCache() {
+        if (this.urlCitationIndex != null) this.urlCitationIndex.clearCache();
+        if (this.termIndex != null) this.termIndex.clearCache();
+        this.fulltext.clearCache();
+    }
 
     public File getLocation() {
         return this.segmentPath;