found and fixed a huge memory leak in solr caching (inside Solr). The

not-flushed Solr cache is now handled in this way:
- it is smaller by default
- an Solr-internal process is started to flush the cache periodically
(this does NOT clean the cache, just removes old objects)
- a Solr-external process (the standard YaCy cleanup-process) now has
direct access to the solr internal cache and flushes them completely.
The time frame for such a flush is defined by the cleanup-process
frequency, by default 10 minutes.

defaults/solr/solrconfig.xml

@@ -461,19 +461,21 @@
          and old cache.
          -->
         <filterCache class="solr.FastLRUCache"
-        size="512"
-        initialSize="512"
-        autowarmCount="0"/>
+        size="64"
+        initialSize="64"
+        autowarmCount="4"
+        cleanupThread="true"/>
         
         <!-- Query Result Cache
          
          Caches results of searches - ordered lists of document ids
          (DocList) based on a query, a sort, and the range of documents requested.
          -->
-        <queryResultCache class="solr.LRUCache"
-        size="512"
-        initialSize="512"
-        autowarmCount="0"/>
+        <queryResultCache class="solr.FastLRUCache"
+        size="64"
+        initialSize="64"
+        autowarmCount="4"
+        cleanupThread="true"/>
         
         <!-- Document Cache
          
@@ -481,10 +483,11 @@
          document).  Since Lucene internal document ids are transient,
          this cache will not be autowarmed.
          -->
-        <documentCache class="solr.LRUCache"
-        size="512"
-        initialSize="512"
-        autowarmCount="0"/>
+        <documentCache class="solr.FastLRUCache"
+        size="64"
+        initialSize="64"
+        autowarmCount="4"
+        cleanupThread="true"/>
         
         <!-- Field Value Cache
          
@@ -494,9 +497,10 @@
          -->
         <!--
          <fieldValueCache class="solr.FastLRUCache"
-         size="512"
+         size="64"
          autowarmCount="128"
-         showItems="32" />
+         showItems="32"
+         cleanupThread="true"/>
          -->
         
         <!-- Custom Cache
@@ -510,11 +514,12 @@
          -->
         <!--
          <cache name="myUserCache"
-         class="solr.LRUCache"
-         size="4096"
-         initialSize="1024"
-         autowarmCount="1024"
+         class="solr.FastLRUCache"
+         size="64"
+         initialSize="64"
+         autowarmCount="64"
          regenerator="com.mycompany.MyRegenerator"
+         cleanupThread="true"
          />
          -->
         

htroot/ContentAnalysis_p.java

@@ -34,7 +34,7 @@ public class ContentAnalysis_p {
 
         // clean up all search events
         SearchEventCache.cleanupEvents(true);
-        sb.index.clearCache(); // every time the ranking is changed we need to remove old orderings
+        sb.index.clearCaches(); // every time the ranking is changed we need to remove old orderings
 
         if (post != null && post.containsKey("EnterDoublecheck")) {
             Ranking.setMinTokenLen(post.getInt("minTokenLen", 3));

htroot/RankingSolr_p.java

@@ -38,7 +38,7 @@ public class RankingSolr_p {
 
         // clean up all search events
         SearchEventCache.cleanupEvents(true);
-        sb.index.clearCache(); // every time the ranking is changed we need to remove old orderings
+        sb.index.clearCaches(); // every time the ranking is changed we need to remove old orderings
 
         int profileNr = 0;
         if (post != null) profileNr = post.getInt("profileNr", profileNr);

htroot/yacysearch.java

@@ -360,7 +360,7 @@ public class yacysearch {
 
             // check available memory and clean up if necessary
             if ( !MemoryControl.request(8000000L, false) ) {
-                indexSegment.clearCache();
+                indexSegment.clearCaches();
                 SearchEventCache.cleanupEvents(false);
             }
 

source/net/yacy/cora/federate/solr/connector/CachedSolrConnector.java

@@ -61,7 +61,7 @@ public class CachedSolrConnector extends AbstractSolrConnector implements SolrCo
         this.missCache = new ConcurrentARC<String, Object>(missCacheMax, partitions);
     }
 
-    public void clearCache() {
+    public void clearCaches() {
         this.hitCache.clear();
         this.missCache.clear();
         this.documentCache.clear();
@@ -70,9 +70,9 @@ public class CachedSolrConnector extends AbstractSolrConnector implements SolrCo
 
     @Override
     public synchronized void close() {
+        this.clearCaches();
         if (this.solr != null) this.solr.close();
         this.solr = null;
-        this.clearCache();
     }
 
     /**
@@ -81,7 +81,7 @@ public class CachedSolrConnector extends AbstractSolrConnector implements SolrCo
      */
     @Override
     public void clear() throws IOException {
-        this.clearCache();
+        this.clearCaches();
         if (this.solr != null) this.solr.clear();
     }
 
@@ -119,7 +119,7 @@ public class CachedSolrConnector extends AbstractSolrConnector implements SolrCo
 
     @Override
     public void deleteByQuery(final String querystring) throws IOException {
-        this.clearCache();
+        this.clearCaches();
         this.solr.deleteByQuery(querystring);
     }
 
@@ -261,7 +261,7 @@ public class CachedSolrConnector extends AbstractSolrConnector implements SolrCo
     }
     
     private void addToCache(SolrDocumentList list, boolean doccache) {
-        if (MemoryControl.shortStatus()) clearCache();
+        if (MemoryControl.shortStatus()) clearCaches();
         for (final SolrDocument solrdoc: list) {
             addToCache(solrdoc, doccache);
         }

source/net/yacy/cora/federate/solr/connector/ConcurrentUpdateSolrConnector.java

@@ -118,6 +118,12 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector {
         ensureAliveUpdateHandler();
     }
 
+    @Override
+    public void clearCaches() {
+        this.connector.clearCaches();
+        this.idCache.clear();
+    }
+
     /**
      * used for debugging
      */

source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java

@@ -34,6 +34,7 @@ import net.yacy.search.schema.CollectionSchema;
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.search.Query;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.response.QueryResponse;
@@ -47,10 +48,14 @@ import org.apache.solr.core.SolrCore;
 import org.apache.solr.handler.component.SearchHandler;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.request.SolrQueryRequestBase;
+import org.apache.solr.request.UnInvertedField;
 import org.apache.solr.response.ResultContext;
 import org.apache.solr.response.SolrQueryResponse;
 import org.apache.solr.search.DocIterator;
 import org.apache.solr.search.DocList;
+import org.apache.solr.search.DocSet;
+import org.apache.solr.search.QueryResultKey;
+import org.apache.solr.search.SolrCache;
 import org.apache.solr.search.SolrIndexSearcher;
 import org.apache.solr.util.RefCounted;
 
@@ -88,6 +93,22 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
         super.init(this.instance.getServer(coreName));
     }
 
+    public void clearCaches() {
+        SolrConfig solrConfig = this.core.getSolrConfig();
+        @SuppressWarnings("unchecked")
+        SolrCache<String, UnInvertedField> fieldValueCache = solrConfig.fieldValueCacheConfig == null ? null : solrConfig.fieldValueCacheConfig.newInstance();
+        if (fieldValueCache != null) fieldValueCache.clear();
+        @SuppressWarnings("unchecked")
+        SolrCache<Query, DocSet> filterCache= solrConfig.filterCacheConfig == null ? null : solrConfig.filterCacheConfig.newInstance();
+        if (filterCache != null) filterCache.clear();
+        @SuppressWarnings("unchecked")
+        SolrCache<QueryResultKey, DocList> queryResultCache = solrConfig.queryResultCacheConfig == null ? null : solrConfig.queryResultCacheConfig.newInstance();
+        if (queryResultCache != null) queryResultCache.clear();
+        @SuppressWarnings("unchecked")
+        SolrCache<Integer, Document> documentCache = solrConfig.documentCacheConfig == null ? null : solrConfig.documentCacheConfig.newInstance();
+        if (documentCache != null) documentCache.clear();
+    }
+    
     public SolrInstance getInstance() {
         return this.instance;
     }

source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java

@@ -54,6 +54,12 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
         this.solr1 = solr1;
     }
 
+    @Override
+    public void clearCaches() {
+        if (this.solr0 != null) this.solr0.clearCaches();
+        if (this.solr1 != null) this.solr1.clearCaches();
+    }
+    
     public boolean isConnected0() {
         return this.solr0 != null;
     }

source/net/yacy/cora/federate/solr/connector/RemoteSolrConnector.java

@@ -71,6 +71,11 @@ public class RemoteSolrConnector extends SolrServerConnector implements SolrConn
         super.close();
     }
 
+    @Override
+    public void clearCaches() {
+        // we do not have a direct access to the caches here, thus we simply do nothing.
+    }
+
     @Override
     public QueryResponse getResponseByParams(ModifiableSolrParams params) throws IOException {
         // during the solr query we set the thread name to the query string to get more debugging info in thread dumps
@@ -134,4 +139,5 @@ public class RemoteSolrConnector extends SolrServerConnector implements SolrConn
         }
         System.exit(0);
     }
+
 }

source/net/yacy/cora/federate/solr/connector/SolrConnector.java

@@ -37,6 +37,11 @@ import org.apache.solr.common.params.ModifiableSolrParams;
 
 public interface SolrConnector extends Iterable<String> /* Iterable of document IDs */ {
    
+    /**
+     * clear all caches: inside solr and ouside solr within the implementations of this interface
+     */
+    public void clearCaches();
+    
     /**
      * get the size of the index
      * @return number of results if solr is queries with a catch-all pattern

source/net/yacy/cora/federate/solr/instance/InstanceMirror.java

@@ -24,7 +24,6 @@ import java.util.Collection;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
-import net.yacy.cora.federate.solr.connector.CachedSolrConnector;
 import net.yacy.cora.federate.solr.connector.ConcurrentUpdateSolrConnector;
 import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector;
 import net.yacy.cora.federate.solr.connector.MirrorSolrConnector;
@@ -161,9 +160,9 @@ public class InstanceMirror {
         return msc;
     }
     
-    public void clearCache() {
+    public void clearCaches() {
         for (SolrConnector csc: this.connectorCache.values()) {
-            if (csc instanceof CachedSolrConnector) ((CachedSolrConnector) csc).clearCache();
+            csc.clearCaches();
         }
         for (EmbeddedSolrConnector ssc: this.embeddedCache.values()) ssc.commit(true);
     }

source/net/yacy/search/ResourceObserver.java

@@ -129,7 +129,7 @@ public class ResourceObserver {
     	if(MemoryControl.properState()) return Space.HIGH;
     	
         // clear some caches - @all: are there more of these, we could clear here?
-		this.sb.index.clearCache();
+		this.sb.index.clearCaches();
         SearchEventCache.cleanupEvents(true);
         this.sb.trail.clear();
         Switchboard.urlBlacklist.clearblacklistCache();

source/net/yacy/search/Switchboard.java

@@ -2031,7 +2031,7 @@ public final class Switchboard extends serverSwitch {
 
             // clear caches if necessary
             if ( !MemoryControl.request(128000000L, false) ) {
-                this.index.clearCache();
+                this.index.clearCaches();
                 SearchEventCache.cleanupEvents(false);
                 this.trail.clear();
                 GuiHandler.clear();

source/net/yacy/search/index/Fulltext.java

@@ -225,10 +225,10 @@ public final class Fulltext {
         }
     }
 
-    public void clearCache() {
+    public void clearCaches() {
         if (this.urlIndexFile != null && this.urlIndexFile instanceof Cache) ((Cache) this.urlIndexFile).clearCache();
         if (this.statsDump != null) this.statsDump.clear();
-        this.solrInstances.clearCache();
+        this.solrInstances.clearCaches();
         this.statsDump = null;
     }
 
@@ -250,7 +250,7 @@ public final class Fulltext {
                 for (String name: instance.getCoreNames()) new EmbeddedSolrConnector(instance, name).clear();
             }
             this.commit(false);
-            this.solrInstances.clearCache();
+            this.solrInstances.clearCaches();
         }
     }
 
@@ -260,7 +260,7 @@ public final class Fulltext {
             if (instance != null) {
                 for (String name: instance.getCoreNames()) new RemoteSolrConnector(instance, name).clear();
             }
-            this.solrInstances.clearCache();
+            this.solrInstances.clearCaches();
         }
     }
 
@@ -400,7 +400,7 @@ public final class Fulltext {
             throw new IOException(e.getMessage(), e);
         }
         this.statsDump = null;
-        if (MemoryControl.shortStatus()) clearCache();
+        if (MemoryControl.shortStatus()) clearCaches();
     }
 
     public void putEdges(final Collection<SolrInputDocument> edges) throws IOException {
@@ -412,7 +412,7 @@ public final class Fulltext {
             throw new IOException(e.getMessage(), e);
         }
         this.statsDump = null;
-        if (MemoryControl.shortStatus()) clearCache();
+        if (MemoryControl.shortStatus()) clearCaches();
     }
 
     /**
@@ -432,7 +432,7 @@ public final class Fulltext {
             throw new IOException(e.getMessage(), e);
         }
         this.statsDump = null;
-        if (MemoryControl.shortStatus()) clearCache();
+        if (MemoryControl.shortStatus()) clearCaches();
     }
 
     /**

source/net/yacy/search/index/Segment.java

@@ -503,10 +503,10 @@ public class Segment {
         }
     }
     
-    public void clearCache() {
+    public void clearCaches() {
         if (this.urlCitationIndex != null) this.urlCitationIndex.clearCache();
         if (this.termIndex != null) this.termIndex.clearCache();
-        this.fulltext.clearCache();
+        this.fulltext.clearCaches();
     }
 
     public File getLocation() {