diff --git a/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java index fc73c4b96..7039f964e 100644 --- a/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java @@ -51,6 +51,7 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.core.SolrConfig; import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SolrInfoMBean; import org.apache.solr.handler.component.SearchHandler; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequestBase; @@ -123,7 +124,11 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo @SuppressWarnings("unchecked") SolrCache documentCache = solrConfig.documentCacheConfig == null ? null : solrConfig.documentCacheConfig.newInstance(); if (documentCache != null) documentCache.clear(); - this.core.getInfoRegistry().clear(); // don't know what this is for - but this is getting huge! + for (SolrInfoMBean ib: this.core.getInfoRegistry().values()) { + // clear 'lost' caches + if (ib instanceof SolrCache) ((SolrCache) ib).clear(); + } + // this.core.getInfoRegistry().clear(); } public SolrInstance getInstance() { @@ -205,24 +210,21 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo */ public SolrDocumentList SolrQueryResponse2SolrDocumentList(final SolrQueryRequest req, final SolrQueryResponse rsp) { SolrDocumentList sdl = new SolrDocumentList(); - @SuppressWarnings("rawtypes") - NamedList nl = rsp.getValues(); + NamedList nl = rsp.getValues(); ResultContext resultContext = (ResultContext) nl.get("response"); DocList response = resultContext == null ? new DocSlice(0, 0, new int[0], new float[0], 0, 0.0f) : resultContext.docs; - sdl.setNumFound(response == null ? 0 : response.matches()); sdl.setStart(response == null ? 0 : response.offset()); - if (response != null) { - final int responseCount = response.size(); - SolrIndexSearcher searcher = req.getSearcher(); - DocIterator iterator = response.iterator(); - for (int i = 0; i < responseCount; i++) { - try { + try { + SolrIndexSearcher searcher = req.getSearcher(); + final int responseCount = response.size(); + DocIterator iterator = response.iterator(); + for (int i = 0; i < responseCount; i++) { sdl.add(doc2SolrDoc(searcher.doc(iterator.nextDoc(), (Set) null))); - } catch (IOException e) { - ConcurrentLog.logException(e); } + } catch (IOException e) { + ConcurrentLog.logException(e); } } return sdl; @@ -332,8 +334,8 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo } private class DocListSearcher { - public SolrQueryRequest request; - public DocList response; + private SolrQueryRequest request; + private DocList response; public DocListSearcher(final String querystring, final int offset, final int count, final String ... fields) { // construct query @@ -347,10 +349,9 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo params.setIncludeScore(false); // query the server - this.request = request(params); + this.request = EmbeddedSolrConnector.this.request(params); SolrQueryResponse rsp = query(request); - @SuppressWarnings("rawtypes") - NamedList nl = rsp.getValues(); + NamedList nl = rsp.getValues(); ResultContext resultContext = (ResultContext) nl.get("response"); if (resultContext == null) log.warn("DocListSearcher: no response for query '" + querystring + "'"); this.response = resultContext == null ? new DocSlice(0, 0, new int[0], new float[0], 0, 0.0f) : resultContext.docs; @@ -360,6 +361,9 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo this.request = null; this.response = null; } + protected void finalize() throws Throwable { + try {close();} finally {super.finalize();} + } } @Override @@ -418,12 +422,13 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo public void run() { int o = offset, responseCount = 0; DocListSearcher docListSearcher = null; + SolrIndexSearcher searcher = null; while (System.currentTimeMillis() < endtime) { try { responseCount = 0; docListSearcher = new DocListSearcher(querystring, o, pagesize, CollectionSchema.id.getSolrFieldName()); responseCount = docListSearcher.response.size(); - SolrIndexSearcher searcher = docListSearcher.request.getSearcher(); + searcher = docListSearcher.request.getSearcher(); DocIterator iterator = docListSearcher.response.iterator(); for (int i = 0; i < responseCount; i++) { Document doc = searcher.doc(iterator.nextDoc(), SOLR_ID_FIELDS); @@ -433,6 +438,7 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo break; } catch (IOException e) { } finally { + if (searcher != null) try {searcher.close();} catch (IOException e) {} if (docListSearcher != null) docListSearcher.close(); } if (responseCount < pagesize) break; diff --git a/source/net/yacy/cora/federate/solr/instance/InstanceMirror.java b/source/net/yacy/cora/federate/solr/instance/InstanceMirror.java index 7831dba79..a18b51620 100644 --- a/source/net/yacy/cora/federate/solr/instance/InstanceMirror.java +++ b/source/net/yacy/cora/federate/solr/instance/InstanceMirror.java @@ -32,144 +32,140 @@ import net.yacy.cora.federate.solr.connector.SolrConnector; public class InstanceMirror { - private EmbeddedInstance solr0; - private ShardInstance solr1; - private SolrConnector defaultConnector; - private Map connectorCache; - private EmbeddedSolrConnector defaultEmbeddedConnector; - private Map embeddedCache; + private EmbeddedInstance embeddedSolrInstance; + private ShardInstance remoteSolrInstance; + private Map mirrorConnectorCache; + private Map embeddedConnectorCache; + private Map remoteConnectorCache; public InstanceMirror() { - this.solr0 = null; - this.solr1 = null; - this.defaultConnector = null; - this.connectorCache = new ConcurrentHashMap(); - this.defaultEmbeddedConnector = null; - this.embeddedCache = new ConcurrentHashMap(); + this.embeddedSolrInstance = null; + this.remoteSolrInstance = null; + this.mirrorConnectorCache = new ConcurrentHashMap(); + this.embeddedConnectorCache = new ConcurrentHashMap(); + this.remoteConnectorCache = new ConcurrentHashMap(); } - public boolean isConnected0() { - return this.solr0 != null; + public boolean isConnectedEmbedded() { + return this.embeddedSolrInstance != null; } - public void connect0(EmbeddedInstance c) { - for (SolrConnector connector: connectorCache.values()) connector.close(); - this.defaultConnector = null; - this.connectorCache.clear(); - this.defaultEmbeddedConnector = null; - this.embeddedCache.clear(); - this.solr0 = c; + public void connectEmbedded(EmbeddedInstance c) { + disconnectEmbedded(); + this.embeddedSolrInstance = c; } - public EmbeddedInstance getSolr0() { - return this.solr0; + public EmbeddedInstance getEmbedded() { + return this.embeddedSolrInstance; } - public void disconnect0() { - if (this.solr0 == null) return; - for (SolrConnector connector: connectorCache.values()) connector.close(); - this.defaultConnector = null; - this.connectorCache.clear(); - this.defaultEmbeddedConnector = null; - this.embeddedCache.clear(); - this.solr0.close(); - this.solr0 = null; + public void disconnectEmbedded() { + if (this.embeddedSolrInstance == null) return; + for (SolrConnector connector: this.mirrorConnectorCache.values()) connector.close(); + this.mirrorConnectorCache.clear(); + for (EmbeddedSolrConnector connector: this.embeddedConnectorCache.values()) connector.close(); + this.embeddedConnectorCache.clear(); + this.embeddedSolrInstance.close(); + this.embeddedSolrInstance = null; } - public boolean isConnected1() { - return this.solr1 != null; + public boolean isConnectedRemote() { + return this.remoteSolrInstance != null; } - public void connect1(ShardInstance c) { - for (SolrConnector connector: connectorCache.values()) connector.close(); - this.defaultConnector = null; - this.connectorCache.clear(); - this.defaultEmbeddedConnector = null; - this.embeddedCache.clear(); - this.solr1 = c; + public void connectRemote(ShardInstance c) { + disconnectRemote(); + this.remoteSolrInstance = c; } - public ShardInstance getSolr1() { - return this.solr1; + public ShardInstance getRemote() { + return this.remoteSolrInstance; } - public void disconnect1() { - if (this.solr1 == null) return; - for (SolrConnector connector: connectorCache.values()) connector.close(); - this.defaultConnector = null; - this.connectorCache.clear(); - this.defaultEmbeddedConnector = null; - this.embeddedCache.clear(); - this.solr1.close(); - this.solr1 = null; + public void disconnectRemote() { + if (this.remoteSolrInstance == null) return; + for (SolrConnector connector: this.mirrorConnectorCache.values()) connector.close(); + this.mirrorConnectorCache.clear(); + for (RemoteSolrConnector connector: this.remoteConnectorCache.values()) connector.close(); + this.remoteConnectorCache.clear(); + this.remoteSolrInstance.close(); + this.remoteSolrInstance = null; } public synchronized void close() { - this.disconnect0(); - this.disconnect1(); + this.disconnectEmbedded(); + this.disconnectRemote(); } public String getDefaultCoreName() { - if (this.solr0 != null) return this.solr0.getDefaultCoreName(); - if (this.solr1 != null) return this.solr1.getDefaultCoreName(); + if (this.embeddedSolrInstance != null) return this.embeddedSolrInstance.getDefaultCoreName(); + if (this.remoteSolrInstance != null) return this.remoteSolrInstance.getDefaultCoreName(); return null; } public Collection getCoreNames() { - if (this.solr0 != null) return this.solr0.getCoreNames(); - if (this.solr1 != null) return this.solr1.getCoreNames(); + if (this.embeddedSolrInstance != null) return this.embeddedSolrInstance.getCoreNames(); + if (this.remoteSolrInstance != null) return this.remoteSolrInstance.getCoreNames(); return null; } public EmbeddedSolrConnector getDefaultEmbeddedConnector() { - if (this.defaultEmbeddedConnector != null) return this.defaultEmbeddedConnector; - if (this.solr0 == null) return null; - this.defaultEmbeddedConnector = new EmbeddedSolrConnector(this.solr0); + if (this.embeddedSolrInstance == null) return null; String coreName = this.getDefaultCoreName(); if (coreName == null) return null; - this.embeddedCache.put(coreName, this.defaultEmbeddedConnector); - return this.defaultEmbeddedConnector; + EmbeddedSolrConnector esc = this.embeddedConnectorCache.get(coreName); + if (esc != null) return esc; + esc = new EmbeddedSolrConnector(this.embeddedSolrInstance); + this.embeddedConnectorCache.put(coreName, esc); + return esc; } public EmbeddedSolrConnector getEmbeddedConnector(String corename) { - EmbeddedSolrConnector ec = this.embeddedCache.get(corename); - if (ec != null) return ec; - ec = this.solr0 == null ? null : new EmbeddedSolrConnector(this.solr0, corename); - this.embeddedCache.put(corename, ec); - return ec; + if (this.embeddedSolrInstance == null) return null; + EmbeddedSolrConnector esc = this.embeddedConnectorCache.get(corename); + if (esc != null) return esc; + esc = new EmbeddedSolrConnector(this.embeddedSolrInstance, corename); + this.embeddedConnectorCache.put(corename, esc); + return esc; + } + + public RemoteSolrConnector getRemoteConnector(String corename) { + if (this.remoteSolrInstance == null) return null; + RemoteSolrConnector rsc = this.remoteConnectorCache.get(corename); + if (rsc != null) return rsc; + rsc = new RemoteSolrConnector(this.remoteSolrInstance, true, corename); + this.remoteConnectorCache.put(corename, rsc); + return rsc; } public SolrConnector getDefaultMirrorConnector() { - if (this.defaultConnector != null) return this.defaultConnector; - String defaultCoreName = this.getDefaultCoreName(); - if (defaultCoreName == null) return null; - EmbeddedSolrConnector esc = this.solr0 == null ? null : new EmbeddedSolrConnector(this.solr0, defaultCoreName); - RemoteSolrConnector rsc = this.solr1 == null ? null : new RemoteSolrConnector(this.solr1, true, defaultCoreName); - this.defaultConnector = new ConcurrentUpdateSolrConnector(new MirrorSolrConnector(esc, rsc), 100, 100000, Runtime.getRuntime().availableProcessors()); - this.connectorCache.put(defaultCoreName, this.defaultConnector); - return this.defaultConnector; - } - - public SolrConnector getMirrorConnector(String corename) { - SolrConnector msc = this.connectorCache.get(corename); + String coreName = this.getDefaultCoreName(); + if (coreName == null) return null; + return getGenericMirrorConnector(coreName); + } + + public SolrConnector getGenericMirrorConnector(String corename) { + SolrConnector msc = this.mirrorConnectorCache.get(corename); if (msc != null) return msc; - EmbeddedSolrConnector esc = this.solr0 == null ? null : new EmbeddedSolrConnector(this.solr0, corename); - RemoteSolrConnector rsc = this.solr1 == null ? null : new RemoteSolrConnector(this.solr1, true, corename); + EmbeddedSolrConnector esc = getEmbeddedConnector(corename); + RemoteSolrConnector rsc = getRemoteConnector(corename); msc = new ConcurrentUpdateSolrConnector(new MirrorSolrConnector(esc, rsc), 100, 100000, Runtime.getRuntime().availableProcessors()); - this.connectorCache.put(corename, msc); + this.mirrorConnectorCache.put(corename, msc); return msc; } public int bufferSize() { int b = 0; - for (SolrConnector csc: this.connectorCache.values()) b += csc.bufferSize(); - for (EmbeddedSolrConnector ssc: this.embeddedCache.values()) b += ssc.bufferSize(); + for (SolrConnector sc: this.mirrorConnectorCache.values()) b += sc.bufferSize(); + for (EmbeddedSolrConnector esc: this.embeddedConnectorCache.values()) b += esc.bufferSize(); + for (RemoteSolrConnector rsc: this.remoteConnectorCache.values()) b += rsc.bufferSize(); return b; } public void clearCaches() { - for (SolrConnector csc: this.connectorCache.values()) csc.clearCaches(); - for (EmbeddedSolrConnector ssc: this.embeddedCache.values()) ssc.commit(true); + for (SolrConnector csc: this.mirrorConnectorCache.values()) csc.clearCaches(); + for (EmbeddedSolrConnector esc: this.embeddedConnectorCache.values()) esc.clearCaches(); + for (RemoteSolrConnector rsc: this.remoteConnectorCache.values()) rsc.clearCaches(); } } diff --git a/source/net/yacy/search/index/Fulltext.java b/source/net/yacy/search/index/Fulltext.java index 758454451..e9e60b8d8 100644 --- a/source/net/yacy/search/index/Fulltext.java +++ b/source/net/yacy/search/index/Fulltext.java @@ -121,7 +121,7 @@ public final class Fulltext { } public boolean connectedLocalSolr() { - return this.solrInstances.isConnected0(); + return this.solrInstances.isConnectedEmbedded(); } public void connectLocalSolr() throws IOException { @@ -141,23 +141,23 @@ public final class Fulltext { int p = lvn.indexOf('_'); assert SOLR_PATH.endsWith(lvn.substring(p)) : "luceneVersion = " + lvn + ", solrPath = " + SOLR_PATH + ", p = " + p + ", check defaults/solr/solrconfig.xml"; ConcurrentLog.info("Fulltext", "connected solr in " + solrLocation.toString() + ", lucene version " + lvn + ", default core size: " + localCollectionConnector.getSize()); - this.solrInstances.connect0(localCollectionInstance); + this.solrInstances.connectEmbedded(localCollectionInstance); } public void disconnectLocalSolr() { - this.solrInstances.disconnect0(); + this.solrInstances.disconnectEmbedded(); } public boolean connectedRemoteSolr() { - return this.solrInstances.isConnected1(); + return this.solrInstances.isConnectedRemote(); } public void connectRemoteSolr(final ArrayList instances, final boolean writeEnabled) { - this.solrInstances.connect1(new ShardInstance(instances, ShardSelection.Method.MODULO_HOST_MD5, writeEnabled)); + this.solrInstances.connectRemote(new ShardInstance(instances, ShardSelection.Method.MODULO_HOST_MD5, writeEnabled)); } public void disconnectRemoteSolr() { - this.solrInstances.disconnect1(); + this.solrInstances.disconnectRemote(); } public EmbeddedSolrConnector getDefaultEmbeddedConnector() { @@ -169,9 +169,9 @@ public final class Fulltext { } public RemoteSolrConnector getDefaultRemoteSolrConnector() { - if (this.solrInstances.getSolr1() == null) return null; + if (this.solrInstances.getRemote() == null) return null; try { - return new RemoteSolrConnector(this.solrInstances.getSolr1(), true); + return new RemoteSolrConnector(this.solrInstances.getRemote(), true); } catch (final IOException e) { return null; } @@ -179,7 +179,7 @@ public final class Fulltext { public EmbeddedInstance getEmbeddedInstance() { synchronized (this.solrInstances) { - if (this.solrInstances.isConnected0()) return this.solrInstances.getSolr0(); + if (this.solrInstances.isConnectedEmbedded()) return this.solrInstances.getEmbedded(); return null; } } @@ -193,7 +193,7 @@ public final class Fulltext { public SolrConnector getWebgraphConnector() { if (!this.writeWebgraph) return null; synchronized (this.solrInstances) { - return this.solrInstances.getMirrorConnector(WebgraphSchema.CORE_NAME); + return this.solrInstances.getGenericMirrorConnector(WebgraphSchema.CORE_NAME); } } @@ -215,7 +215,7 @@ public final class Fulltext { public void clearLocalSolr() throws IOException { synchronized (this.solrInstances) { - EmbeddedInstance instance = this.solrInstances.getSolr0(); + EmbeddedInstance instance = this.solrInstances.getEmbedded(); if (instance != null) { for (String name: instance.getCoreNames()) new EmbeddedSolrConnector(instance, name).clear(); } @@ -226,7 +226,7 @@ public final class Fulltext { public void clearRemoteSolr() throws IOException { synchronized (this.solrInstances) { - ShardInstance instance = this.solrInstances.getSolr1(); + ShardInstance instance = this.solrInstances.getRemote(); if (instance != null) { for (String name: instance.getCoreNames()) new RemoteSolrConnector(instance, true, name).clear(); } @@ -552,7 +552,7 @@ public final class Fulltext { } public List dumpFiles() { - EmbeddedInstance esc = this.solrInstances.getSolr0(); + EmbeddedInstance esc = this.solrInstances.getEmbedded(); ArrayList zips = new ArrayList(); if (esc == null) { ConcurrentLog.warn("Fulltext", "HOT DUMP selected solr0 == NULL, no dump list!"); @@ -578,7 +578,7 @@ public final class Fulltext { * @return */ public File dumpSolr() { - EmbeddedInstance esc = this.solrInstances.getSolr0(); + EmbeddedInstance esc = this.solrInstances.getEmbedded(); File storagePath = esc.getContainerPath(); File zipOut = new File(this.archivePath, storagePath.getName() + "_" + GenericFormatter.SHORT_DAY_FORMATTER.format() + ".zip"); synchronized (this.solrInstances) { @@ -605,7 +605,7 @@ public final class Fulltext { * @param solrDumpZipFile */ public void restoreSolr(File solrDumpZipFile) { - EmbeddedInstance esc = this.solrInstances.getSolr0(); + EmbeddedInstance esc = this.solrInstances.getEmbedded(); File storagePath = esc.getContainerPath(); synchronized (this.solrInstances) { this.disconnectLocalSolr();