boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

clear solr caches in case that an exception occurrs. The reason behind

Browse Source 
this hack is the occurrence of Exceptions like:
W 2014/02/11 18:51:33 ConcurrentLog GC overhead limit exceeded
java.io.IOException: GC overhead limit exceeded
        at
net.yacy.cora.federate.solr.connector.AbstractSolrConnector.getDocumentById(AbstractSolrConnector.java:334)
        at
net.yacy.cora.federate.solr.connector.MirrorSolrConnector.getDocumentById(MirrorSolrConnector.java:173)
        at
net.yacy.cora.federate.solr.connector.ConcurrentUpdateSolrConnector.getDocumentById(ConcurrentUpdateSolrConnector.java:415)
        at net.yacy.search.index.Fulltext.getMetadata(Fulltext.java:331)
        at net.yacy.search.index.Fulltext.getMetadata(Fulltext.java:317)
        at
net.yacy.search.query.SearchEvent.pullOneRWI(SearchEvent.java:1024)
        at
net.yacy.search.query.SearchEvent.pullOneFilteredFromRWI(SearchEvent.java:1047)
        at
net.yacy.search.query.SearchEvent$3.run(SearchEvent.java:1263)
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
        at java.util.Arrays.copyOfRange(Arrays.java:3077)
        at java.lang.StringCoding.decode(StringCoding.java:196)
        at java.lang.String.<init>(String.java:491)
        at java.lang.String.<init>(String.java:547)
        at
org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader.readField(CompressingStoredFieldsReader.java:187)
        at
org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader.visitDocument(CompressingStoredFieldsReader.java:351)
        at
org.apache.lucene.index.SegmentReader.document(SegmentReader.java:276)
        at
org.apache.lucene.index.BaseCompositeReader.document(BaseCompositeReader.java:110)
        at
org.apache.lucene.index.IndexReader.document(IndexReader.java:436)
        at
org.apache.solr.search.SolrIndexSearcher.doc(SolrIndexSearcher.java:657)
        at
net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector.SolrQueryResponse2SolrDocumentList(EmbeddedSolrConnector.java:230)
        at
net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector.getDocumentListByParams(EmbeddedSolrConnector.java:320)
        at
net.yacy.cora.federate.solr.connector.AbstractSolrConnector.getDocumentById(AbstractSolrConnector.java:330)
        ... 7 more
        
This problem was analysed with the Eclipse Memory Analyser after a heap
dump, where the following problem was reported as the main Problem
Suspect:

One instance of "org.apache.solr.util.ConcurrentLRUCache" loaded by
"sun.misc.Launcher$AppClassLoader @ 0x42e940a0" occupies 902.898.256
(61,80%) bytes. The memory is accumulated in one instance of
"java.util.concurrent.ConcurrentHashMap$Segment[]" loaded by "<system
class loader>".

This memory is part of the result cache of Solr. Flushing this cache
appears the most appropriate solution to that problem.
pull/1/head
orbiter 11 years ago
parent bc28247089
commit 14764632b5
2 changed files with 14 additions and 0 deletions
Show Stats Download Patch File Download Diff File

1
source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java
Unescape View File

@ -331,6 +331,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
if (docs == null || docs.isEmpty()) return null; if (docs == null || docs.isEmpty()) return null;
return docs.get(0); return docs.get(0);
} catch (final Throwable e) { } catch (final Throwable e) {
clearCaches(); // we clear the in case that this is caused by OOM
throw new IOException(e.getMessage(), e); throw new IOException(e.getMessage(), e);
} }
} }

13
source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java
Unescape View File

@ -77,6 +77,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
this.server.commit(true, true, softCommit); this.server.commit(true, true, softCommit);
//if (this.server instanceof HttpSolrServer) ((HttpSolrServer) this.server).shutdown(); //if (this.server instanceof HttpSolrServer) ((HttpSolrServer) this.server).shutdown();
} catch (final Throwable e) { } catch (final Throwable e) {
clearCaches(); // prevent further OOM if this was caused by OOM
//Log.logException(e); //Log.logException(e);
} }
} }
@ -93,6 +94,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
try { try {
this.server.optimize(true, true, maxSegments); this.server.optimize(true, true, maxSegments);
} catch (final Throwable e) { } catch (final Throwable e) {
clearCaches(); // prevent further OOM if this was caused by OOM
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
} }
} }
@ -112,6 +114,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
if (segmentCount == null) return 1; if (segmentCount == null) return 1;
return segmentCount.intValue(); return segmentCount.intValue();
} catch (final Throwable e) { } catch (final Throwable e) {
clearCaches(); // prevent further OOM if this was caused by OOM
log.warn(e); log.warn(e);
return 0; return 0;
} }
@ -139,6 +142,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
if (numDocs == null) return 0; if (numDocs == null) return 0;
return numDocs.longValue(); return numDocs.longValue();
} catch (final Throwable e) { } catch (final Throwable e) {
clearCaches(); // prevent further OOM if this was caused by OOM
log.warn(e); log.warn(e);
return 0; return 0;
} }
@ -156,6 +160,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
this.server.deleteByQuery(AbstractSolrConnector.CATCHALL_TERM); this.server.deleteByQuery(AbstractSolrConnector.CATCHALL_TERM);
this.server.commit(true, true, false); this.server.commit(true, true, false);
} catch (final Throwable e) { } catch (final Throwable e) {
clearCaches(); // prevent further OOM if this was caused by OOM
throw new IOException(e); throw new IOException(e);
} }
} }
@ -168,6 +173,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
try { try {
this.server.deleteById(id, -1); this.server.deleteById(id, -1);
} catch (final Throwable e) { } catch (final Throwable e) {
clearCaches(); // prevent further OOM if this was caused by OOM
throw new IOException(e); throw new IOException(e);
} }
} }
@ -182,6 +188,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
try { try {
this.server.deleteById(l, -1); this.server.deleteById(l, -1);
} catch (final Throwable e) { } catch (final Throwable e) {
clearCaches(); // prevent further OOM if this was caused by OOM
throw new IOException(e); throw new IOException(e);
} }
} }
@ -199,6 +206,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
try { try {
this.server.deleteByQuery(querystring, -1); this.server.deleteByQuery(querystring, -1);
} catch (final Throwable e) { } catch (final Throwable e) {
clearCaches(); // prevent further OOM if this was caused by OOM
throw new IOException(e); throw new IOException(e);
} }
} }
@ -215,6 +223,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
try { try {
this.server.request(up); this.server.request(up);
} catch (final Throwable e) { } catch (final Throwable e) {
clearCaches(); // prevent further OOM if this was caused by OOM
throw new IOException(e); throw new IOException(e);
} }
} }
@ -227,6 +236,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
if (solrdoc.containsKey("_version_")) solrdoc.setField("_version_",0L); // prevent Solr "version conflict" if (solrdoc.containsKey("_version_")) solrdoc.setField("_version_",0L); // prevent Solr "version conflict"
this.server.add(solrdoc, -1); this.server.add(solrdoc, -1);
} catch (final Throwable e) { } catch (final Throwable e) {
clearCaches(); // prevent further OOM if this was caused by OOM
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
// catches "version conflict for": try this again and delete the document in advance // catches "version conflict for": try this again and delete the document in advance
try { try {
@ -265,6 +275,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
} }
this.server.add(solrdocs, -1); this.server.add(solrdocs, -1);
} catch (final Throwable e) { } catch (final Throwable e) {
clearCaches(); // prevent further OOM if this was caused by OOM
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
// catches "version conflict for": try this again and delete the document in advance // catches "version conflict for": try this again and delete the document in advance
List<String> ids = new ArrayList<String>(); List<String> ids = new ArrayList<String>();
@ -314,8 +325,10 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
if (rsp != null) if (log.isFine()) log.fine(rsp.getResults().getNumFound() + " results for q=" + q); if (rsp != null) if (log.isFine()) log.fine(rsp.getResults().getNumFound() + " results for q=" + q);
return rsp.getResults(); return rsp.getResults();
} catch (final SolrServerException e) { } catch (final SolrServerException e) {
clearCaches(); // prevent further OOM if this was caused by OOM
throw new SolrException(ErrorCode.UNKNOWN, e); throw new SolrException(ErrorCode.UNKNOWN, e);
} catch (final Throwable e) { } catch (final Throwable e) {
clearCaches(); // prevent further OOM if this was caused by OOM
throw new IOException("Error executing query", e); throw new IOException("Error executing query", e);
} }
} }

Write Preview
Loading…
Cancel
Save