boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

enhanced strategy to clear solr caches

Browse Source 
- redesigned the instance mirror class (which was a mess)
- added final method to close a searcher (which otherwise keeps a cache)
- changed cache clear method which iterates over resources and calls
clear to all caches in the searcher resources
pull/1/head
Michael Peter Christen 11 years ago
parent 52599a11b3
commit 456e52e0d5
3 changed files with 122 additions and 120 deletions
Show Stats Download Patch File Download Diff File

42
source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java
Unescape View File

@ -51,6 +51,7 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoMBean;
import org.apache.solr.handler.component.SearchHandler;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryRequestBase;
@ -123,7 +124,11 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
@SuppressWarnings("unchecked")
SolrCache<Integer, Document> documentCache = solrConfig.documentCacheConfig == null ? null : solrConfig.documentCacheConfig.newInstance();
if (documentCache != null) documentCache.clear();
this.core.getInfoRegistry().clear(); // don't know what this is for - but this is getting huge!
for (SolrInfoMBean ib: this.core.getInfoRegistry().values()) {
// clear 'lost' caches
if (ib instanceof SolrCache) ((SolrCache<?,?>) ib).clear();
}
// this.core.getInfoRegistry().clear();
}
public SolrInstance getInstance() {
@ -205,24 +210,21 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
*/
public SolrDocumentList SolrQueryResponse2SolrDocumentList(final SolrQueryRequest req, final SolrQueryResponse rsp) {
SolrDocumentList sdl = new SolrDocumentList();
@SuppressWarnings("rawtypes")
NamedList nl = rsp.getValues();
NamedList<?> nl = rsp.getValues();
ResultContext resultContext = (ResultContext) nl.get("response");
DocList response = resultContext == null ? new DocSlice(0, 0, new int[0], new float[0], 0, 0.0f) : resultContext.docs;
sdl.setNumFound(response == null ? 0 : response.matches());
sdl.setStart(response == null ? 0 : response.offset());
if (response != null) {
final int responseCount = response.size();
SolrIndexSearcher searcher = req.getSearcher();
DocIterator iterator = response.iterator();
for (int i = 0; i < responseCount; i++) {
try {
try {
SolrIndexSearcher searcher = req.getSearcher();
final int responseCount = response.size();
DocIterator iterator = response.iterator();
for (int i = 0; i < responseCount; i++) {
sdl.add(doc2SolrDoc(searcher.doc(iterator.nextDoc(), (Set<String>) null)));
} catch (IOException e) {
ConcurrentLog.logException(e);
}
} catch (IOException e) {
ConcurrentLog.logException(e);
}
}
return sdl;
@ -332,8 +334,8 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
}
private class DocListSearcher {
public SolrQueryRequest request;
public DocList response;
private SolrQueryRequest request;
private DocList response;
public DocListSearcher(final String querystring, final int offset, final int count, final String ... fields) {
// construct query
@ -347,10 +349,9 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
params.setIncludeScore(false);
// query the server
this.request = request(params);
this.request = EmbeddedSolrConnector.this.request(params);
SolrQueryResponse rsp = query(request);
@SuppressWarnings("rawtypes")
NamedList nl = rsp.getValues();
NamedList<?> nl = rsp.getValues();
ResultContext resultContext = (ResultContext) nl.get("response");
if (resultContext == null) log.warn("DocListSearcher: no response for query '" + querystring + "'");
this.response = resultContext == null ? new DocSlice(0, 0, new int[0], new float[0], 0, 0.0f) : resultContext.docs;
@ -360,6 +361,9 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
this.request = null;
this.response = null;
}
protected void finalize() throws Throwable {
try {close();} finally {super.finalize();}
}
}
@Override
@ -418,12 +422,13 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
public void run() {
int o = offset, responseCount = 0;
DocListSearcher docListSearcher = null;
SolrIndexSearcher searcher = null;
while (System.currentTimeMillis() < endtime) {
try {
responseCount = 0;
docListSearcher = new DocListSearcher(querystring, o, pagesize, CollectionSchema.id.getSolrFieldName());
responseCount = docListSearcher.response.size();
SolrIndexSearcher searcher = docListSearcher.request.getSearcher();
searcher = docListSearcher.request.getSearcher();
DocIterator iterator = docListSearcher.response.iterator();
for (int i = 0; i < responseCount; i++) {
Document doc = searcher.doc(iterator.nextDoc(), SOLR_ID_FIELDS);
@ -433,6 +438,7 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
break;
} catch (IOException e) {
} finally {
if (searcher != null) try {searcher.close();} catch (IOException e) {}
if (docListSearcher != null) docListSearcher.close();
}
if (responseCount < pagesize) break;

170
source/net/yacy/cora/federate/solr/instance/InstanceMirror.java
Unescape View File

@ -32,144 +32,140 @@ import net.yacy.cora.federate.solr.connector.SolrConnector;
public class InstanceMirror {
private EmbeddedInstance solr0;
private ShardInstance solr1;
private SolrConnector defaultConnector;
private Map<String, SolrConnector> connectorCache;
private EmbeddedSolrConnector defaultEmbeddedConnector;
private Map<String, EmbeddedSolrConnector> embeddedCache;
private EmbeddedInstance embeddedSolrInstance;
private ShardInstance remoteSolrInstance;
private Map<String, SolrConnector> mirrorConnectorCache;
private Map<String, EmbeddedSolrConnector> embeddedConnectorCache;
private Map<String, RemoteSolrConnector> remoteConnectorCache;
public InstanceMirror() {
this.solr0 = null;
this.solr1 = null;
this.defaultConnector = null;
this.connectorCache = new ConcurrentHashMap<String, SolrConnector>();
this.defaultEmbeddedConnector = null;
this.embeddedCache = new ConcurrentHashMap<String, EmbeddedSolrConnector>();
this.embeddedSolrInstance = null;
this.remoteSolrInstance = null;
this.mirrorConnectorCache = new ConcurrentHashMap<String, SolrConnector>();
this.embeddedConnectorCache = new ConcurrentHashMap<String, EmbeddedSolrConnector>();
this.remoteConnectorCache = new ConcurrentHashMap<String, RemoteSolrConnector>();
}
public boolean isConnected0() {
return this.solr0 != null;
public boolean isConnectedEmbedded() {
return this.embeddedSolrInstance != null;
}
public void connect0(EmbeddedInstance c) {
for (SolrConnector connector: connectorCache.values()) connector.close();
this.defaultConnector = null;
this.connectorCache.clear();
this.defaultEmbeddedConnector = null;
this.embeddedCache.clear();
this.solr0 = c;
public void connectEmbedded(EmbeddedInstance c) {
disconnectEmbedded();
this.embeddedSolrInstance = c;
}
public EmbeddedInstance getSolr0() {
return this.solr0;
public EmbeddedInstance getEmbedded() {
return this.embeddedSolrInstance;
}
public void disconnect0() {
if (this.solr0 == null) return;
for (SolrConnector connector: connectorCache.values()) connector.close();
this.defaultConnector = null;
this.connectorCache.clear();
this.defaultEmbeddedConnector = null;
this.embeddedCache.clear();
this.solr0.close();
this.solr0 = null;
public void disconnectEmbedded() {
if (this.embeddedSolrInstance == null) return;
for (SolrConnector connector: this.mirrorConnectorCache.values()) connector.close();
this.mirrorConnectorCache.clear();
for (EmbeddedSolrConnector connector: this.embeddedConnectorCache.values()) connector.close();
this.embeddedConnectorCache.clear();
this.embeddedSolrInstance.close();
this.embeddedSolrInstance = null;
}
public boolean isConnected1() {
return this.solr1 != null;
public boolean isConnectedRemote() {
return this.remoteSolrInstance != null;
}
public void connect1(ShardInstance c) {
for (SolrConnector connector: connectorCache.values()) connector.close();
this.defaultConnector = null;
this.connectorCache.clear();
this.defaultEmbeddedConnector = null;
this.embeddedCache.clear();
this.solr1 = c;
public void connectRemote(ShardInstance c) {
disconnectRemote();
this.remoteSolrInstance = c;
}
public ShardInstance getSolr1() {
return this.solr1;
public ShardInstance getRemote() {
return this.remoteSolrInstance;
}
public void disconnect1() {
if (this.solr1 == null) return;
for (SolrConnector connector: connectorCache.values()) connector.close();
this.defaultConnector = null;
this.connectorCache.clear();
this.defaultEmbeddedConnector = null;
this.embeddedCache.clear();
this.solr1.close();
this.solr1 = null;
public void disconnectRemote() {
if (this.remoteSolrInstance == null) return;
for (SolrConnector connector: this.mirrorConnectorCache.values()) connector.close();
this.mirrorConnectorCache.clear();
for (RemoteSolrConnector connector: this.remoteConnectorCache.values()) connector.close();
this.remoteConnectorCache.clear();
this.remoteSolrInstance.close();
this.remoteSolrInstance = null;
}
public synchronized void close() {
this.disconnect0();
this.disconnect1();
this.disconnectEmbedded();
this.disconnectRemote();
}
public String getDefaultCoreName() {
if (this.solr0 != null) return this.solr0.getDefaultCoreName();
if (this.solr1 != null) return this.solr1.getDefaultCoreName();
if (this.embeddedSolrInstance != null) return this.embeddedSolrInstance.getDefaultCoreName();
if (this.remoteSolrInstance != null) return this.remoteSolrInstance.getDefaultCoreName();
return null;
}
public Collection<String> getCoreNames() {
if (this.solr0 != null) return this.solr0.getCoreNames();
if (this.solr1 != null) return this.solr1.getCoreNames();
if (this.embeddedSolrInstance != null) return this.embeddedSolrInstance.getCoreNames();
if (this.remoteSolrInstance != null) return this.remoteSolrInstance.getCoreNames();
return null;
}
public EmbeddedSolrConnector getDefaultEmbeddedConnector() {
if (this.defaultEmbeddedConnector != null) return this.defaultEmbeddedConnector;
if (this.solr0 == null) return null;
this.defaultEmbeddedConnector = new EmbeddedSolrConnector(this.solr0);
if (this.embeddedSolrInstance == null) return null;
String coreName = this.getDefaultCoreName();
if (coreName == null) return null;
this.embeddedCache.put(coreName, this.defaultEmbeddedConnector);
return this.defaultEmbeddedConnector;
EmbeddedSolrConnector esc = this.embeddedConnectorCache.get(coreName);
if (esc != null) return esc;
esc = new EmbeddedSolrConnector(this.embeddedSolrInstance);
this.embeddedConnectorCache.put(coreName, esc);
return esc;
}
public EmbeddedSolrConnector getEmbeddedConnector(String corename) {
EmbeddedSolrConnector ec = this.embeddedCache.get(corename);
if (ec != null) return ec;
ec = this.solr0 == null ? null : new EmbeddedSolrConnector(this.solr0, corename);
this.embeddedCache.put(corename, ec);
return ec;
if (this.embeddedSolrInstance == null) return null;
EmbeddedSolrConnector esc = this.embeddedConnectorCache.get(corename);
if (esc != null) return esc;
esc = new EmbeddedSolrConnector(this.embeddedSolrInstance, corename);
this.embeddedConnectorCache.put(corename, esc);
return esc;
}
public RemoteSolrConnector getRemoteConnector(String corename) {
if (this.remoteSolrInstance == null) return null;
RemoteSolrConnector rsc = this.remoteConnectorCache.get(corename);
if (rsc != null) return rsc;
rsc = new RemoteSolrConnector(this.remoteSolrInstance, true, corename);
this.remoteConnectorCache.put(corename, rsc);
return rsc;
}
public SolrConnector getDefaultMirrorConnector() {
if (this.defaultConnector != null) return this.defaultConnector;
String defaultCoreName = this.getDefaultCoreName();
if (defaultCoreName == null) return null;
EmbeddedSolrConnector esc = this.solr0 == null ? null : new EmbeddedSolrConnector(this.solr0, defaultCoreName);
RemoteSolrConnector rsc = this.solr1 == null ? null : new RemoteSolrConnector(this.solr1, true, defaultCoreName);
this.defaultConnector = new ConcurrentUpdateSolrConnector(new MirrorSolrConnector(esc, rsc), 100, 100000, Runtime.getRuntime().availableProcessors());
this.connectorCache.put(defaultCoreName, this.defaultConnector);
return this.defaultConnector;
}
public SolrConnector getMirrorConnector(String corename) {
SolrConnector msc = this.connectorCache.get(corename);
String coreName = this.getDefaultCoreName();
if (coreName == null) return null;
return getGenericMirrorConnector(coreName);
}
public SolrConnector getGenericMirrorConnector(String corename) {
SolrConnector msc = this.mirrorConnectorCache.get(corename);
if (msc != null) return msc;
EmbeddedSolrConnector esc = this.solr0 == null ? null : new EmbeddedSolrConnector(this.solr0, corename);
RemoteSolrConnector rsc = this.solr1 == null ? null : new RemoteSolrConnector(this.solr1, true, corename);
EmbeddedSolrConnector esc = getEmbeddedConnector(corename);
RemoteSolrConnector rsc = getRemoteConnector(corename);
msc = new ConcurrentUpdateSolrConnector(new MirrorSolrConnector(esc, rsc), 100, 100000, Runtime.getRuntime().availableProcessors());
this.connectorCache.put(corename, msc);
this.mirrorConnectorCache.put(corename, msc);
return msc;
}
public int bufferSize() {
int b = 0;
for (SolrConnector csc: this.connectorCache.values()) b += csc.bufferSize();
for (EmbeddedSolrConnector ssc: this.embeddedCache.values()) b += ssc.bufferSize();
for (SolrConnector sc: this.mirrorConnectorCache.values()) b += sc.bufferSize();
for (EmbeddedSolrConnector esc: this.embeddedConnectorCache.values()) b += esc.bufferSize();
for (RemoteSolrConnector rsc: this.remoteConnectorCache.values()) b += rsc.bufferSize();
return b;
}
public void clearCaches() {
for (SolrConnector csc: this.connectorCache.values()) csc.clearCaches();
for (EmbeddedSolrConnector ssc: this.embeddedCache.values()) ssc.commit(true);
for (SolrConnector csc: this.mirrorConnectorCache.values()) csc.clearCaches();
for (EmbeddedSolrConnector esc: this.embeddedConnectorCache.values()) esc.clearCaches();
for (RemoteSolrConnector rsc: this.remoteConnectorCache.values()) rsc.clearCaches();
}
}

30
source/net/yacy/search/index/Fulltext.java
Unescape View File

@ -121,7 +121,7 @@ public final class Fulltext {
}
public boolean connectedLocalSolr() {
return this.solrInstances.isConnected0();
return this.solrInstances.isConnectedEmbedded();
}
public void connectLocalSolr() throws IOException {
@ -141,23 +141,23 @@ public final class Fulltext {
int p = lvn.indexOf('_');
assert SOLR_PATH.endsWith(lvn.substring(p)) : "luceneVersion = " + lvn + ", solrPath = " + SOLR_PATH + ", p = " + p + ", check defaults/solr/solrconfig.xml";
ConcurrentLog.info("Fulltext", "connected solr in " + solrLocation.toString() + ", lucene version " + lvn + ", default core size: " + localCollectionConnector.getSize());
this.solrInstances.connect0(localCollectionInstance);
this.solrInstances.connectEmbedded(localCollectionInstance);
}
public void disconnectLocalSolr() {
this.solrInstances.disconnect0();
this.solrInstances.disconnectEmbedded();
}
public boolean connectedRemoteSolr() {
return this.solrInstances.isConnected1();
return this.solrInstances.isConnectedRemote();
}
public void connectRemoteSolr(final ArrayList<RemoteInstance> instances, final boolean writeEnabled) {
this.solrInstances.connect1(new ShardInstance(instances, ShardSelection.Method.MODULO_HOST_MD5, writeEnabled));
this.solrInstances.connectRemote(new ShardInstance(instances, ShardSelection.Method.MODULO_HOST_MD5, writeEnabled));
}
public void disconnectRemoteSolr() {
this.solrInstances.disconnect1();
this.solrInstances.disconnectRemote();
}
public EmbeddedSolrConnector getDefaultEmbeddedConnector() {
@ -169,9 +169,9 @@ public final class Fulltext {
}
public RemoteSolrConnector getDefaultRemoteSolrConnector() {
if (this.solrInstances.getSolr1() == null) return null;
if (this.solrInstances.getRemote() == null) return null;
try {
return new RemoteSolrConnector(this.solrInstances.getSolr1(), true);
return new RemoteSolrConnector(this.solrInstances.getRemote(), true);
} catch (final IOException e) {
return null;
}
@ -179,7 +179,7 @@ public final class Fulltext {
public EmbeddedInstance getEmbeddedInstance() {
synchronized (this.solrInstances) {
if (this.solrInstances.isConnected0()) return this.solrInstances.getSolr0();
if (this.solrInstances.isConnectedEmbedded()) return this.solrInstances.getEmbedded();
return null;
}
}
@ -193,7 +193,7 @@ public final class Fulltext {
public SolrConnector getWebgraphConnector() {
if (!this.writeWebgraph) return null;
synchronized (this.solrInstances) {
return this.solrInstances.getMirrorConnector(WebgraphSchema.CORE_NAME);
return this.solrInstances.getGenericMirrorConnector(WebgraphSchema.CORE_NAME);
}
}
@ -215,7 +215,7 @@ public final class Fulltext {
public void clearLocalSolr() throws IOException {
synchronized (this.solrInstances) {
EmbeddedInstance instance = this.solrInstances.getSolr0();
EmbeddedInstance instance = this.solrInstances.getEmbedded();
if (instance != null) {
for (String name: instance.getCoreNames()) new EmbeddedSolrConnector(instance, name).clear();
}
@ -226,7 +226,7 @@ public final class Fulltext {
public void clearRemoteSolr() throws IOException {
synchronized (this.solrInstances) {
ShardInstance instance = this.solrInstances.getSolr1();
ShardInstance instance = this.solrInstances.getRemote();
if (instance != null) {
for (String name: instance.getCoreNames()) new RemoteSolrConnector(instance, true, name).clear();
}
@ -552,7 +552,7 @@ public final class Fulltext {
}
public List<File> dumpFiles() {
EmbeddedInstance esc = this.solrInstances.getSolr0();
EmbeddedInstance esc = this.solrInstances.getEmbedded();
ArrayList<File> zips = new ArrayList<File>();
if (esc == null) {
ConcurrentLog.warn("Fulltext", "HOT DUMP selected solr0 == NULL, no dump list!");
@ -578,7 +578,7 @@ public final class Fulltext {
* @return
*/
public File dumpSolr() {
EmbeddedInstance esc = this.solrInstances.getSolr0();
EmbeddedInstance esc = this.solrInstances.getEmbedded();
File storagePath = esc.getContainerPath();
File zipOut = new File(this.archivePath, storagePath.getName() + "_" + GenericFormatter.SHORT_DAY_FORMATTER.format() + ".zip");
synchronized (this.solrInstances) {
@ -605,7 +605,7 @@ public final class Fulltext {
* @param solrDumpZipFile
*/
public void restoreSolr(File solrDumpZipFile) {
EmbeddedInstance esc = this.solrInstances.getSolr0();
EmbeddedInstance esc = this.solrInstances.getEmbedded();
File storagePath = esc.getContainerPath();
synchronized (this.solrInstances) {
this.disconnectLocalSolr();

Write Preview
Loading…
Cancel
Save