From 14764632b5091a164da1ea07b4ef8a2a46b51f1e Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 11 Feb 2014 20:56:40 +0100 Subject: [PATCH] clear solr caches in case that an exception occurrs. The reason behind this hack is the occurrence of Exceptions like: W 2014/02/11 18:51:33 ConcurrentLog GC overhead limit exceeded java.io.IOException: GC overhead limit exceeded at net.yacy.cora.federate.solr.connector.AbstractSolrConnector.getDocumentById(AbstractSolrConnector.java:334) at net.yacy.cora.federate.solr.connector.MirrorSolrConnector.getDocumentById(MirrorSolrConnector.java:173) at net.yacy.cora.federate.solr.connector.ConcurrentUpdateSolrConnector.getDocumentById(ConcurrentUpdateSolrConnector.java:415) at net.yacy.search.index.Fulltext.getMetadata(Fulltext.java:331) at net.yacy.search.index.Fulltext.getMetadata(Fulltext.java:317) at net.yacy.search.query.SearchEvent.pullOneRWI(SearchEvent.java:1024) at net.yacy.search.query.SearchEvent.pullOneFilteredFromRWI(SearchEvent.java:1047) at net.yacy.search.query.SearchEvent$3.run(SearchEvent.java:1263) Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded at java.util.Arrays.copyOfRange(Arrays.java:3077) at java.lang.StringCoding.decode(StringCoding.java:196) at java.lang.String.(String.java:491) at java.lang.String.(String.java:547) at org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader.readField(CompressingStoredFieldsReader.java:187) at org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader.visitDocument(CompressingStoredFieldsReader.java:351) at org.apache.lucene.index.SegmentReader.document(SegmentReader.java:276) at org.apache.lucene.index.BaseCompositeReader.document(BaseCompositeReader.java:110) at org.apache.lucene.index.IndexReader.document(IndexReader.java:436) at org.apache.solr.search.SolrIndexSearcher.doc(SolrIndexSearcher.java:657) at net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector.SolrQueryResponse2SolrDocumentList(EmbeddedSolrConnector.java:230) at net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector.getDocumentListByParams(EmbeddedSolrConnector.java:320) at net.yacy.cora.federate.solr.connector.AbstractSolrConnector.getDocumentById(AbstractSolrConnector.java:330) ... 7 more This problem was analysed with the Eclipse Memory Analyser after a heap dump, where the following problem was reported as the main Problem Suspect: One instance of "org.apache.solr.util.ConcurrentLRUCache" loaded by "sun.misc.Launcher$AppClassLoader @ 0x42e940a0" occupies 902.898.256 (61,80%) bytes. The memory is accumulated in one instance of "java.util.concurrent.ConcurrentHashMap$Segment[]" loaded by "". This memory is part of the result cache of Solr. Flushing this cache appears the most appropriate solution to that problem. --- .../solr/connector/AbstractSolrConnector.java | 1 + .../solr/connector/SolrServerConnector.java | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java index 036204fbe..60a5b6508 100644 --- a/source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java @@ -331,6 +331,7 @@ public abstract class AbstractSolrConnector implements SolrConnector { if (docs == null || docs.isEmpty()) return null; return docs.get(0); } catch (final Throwable e) { + clearCaches(); // we clear the in case that this is caused by OOM throw new IOException(e.getMessage(), e); } } diff --git a/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java b/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java index e8b670ee2..463b96e3e 100644 --- a/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java @@ -77,6 +77,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen this.server.commit(true, true, softCommit); //if (this.server instanceof HttpSolrServer) ((HttpSolrServer) this.server).shutdown(); } catch (final Throwable e) { + clearCaches(); // prevent further OOM if this was caused by OOM //Log.logException(e); } } @@ -93,6 +94,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen try { this.server.optimize(true, true, maxSegments); } catch (final Throwable e) { + clearCaches(); // prevent further OOM if this was caused by OOM ConcurrentLog.logException(e); } } @@ -112,6 +114,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen if (segmentCount == null) return 1; return segmentCount.intValue(); } catch (final Throwable e) { + clearCaches(); // prevent further OOM if this was caused by OOM log.warn(e); return 0; } @@ -139,6 +142,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen if (numDocs == null) return 0; return numDocs.longValue(); } catch (final Throwable e) { + clearCaches(); // prevent further OOM if this was caused by OOM log.warn(e); return 0; } @@ -156,6 +160,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen this.server.deleteByQuery(AbstractSolrConnector.CATCHALL_TERM); this.server.commit(true, true, false); } catch (final Throwable e) { + clearCaches(); // prevent further OOM if this was caused by OOM throw new IOException(e); } } @@ -168,6 +173,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen try { this.server.deleteById(id, -1); } catch (final Throwable e) { + clearCaches(); // prevent further OOM if this was caused by OOM throw new IOException(e); } } @@ -182,6 +188,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen try { this.server.deleteById(l, -1); } catch (final Throwable e) { + clearCaches(); // prevent further OOM if this was caused by OOM throw new IOException(e); } } @@ -199,6 +206,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen try { this.server.deleteByQuery(querystring, -1); } catch (final Throwable e) { + clearCaches(); // prevent further OOM if this was caused by OOM throw new IOException(e); } } @@ -215,6 +223,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen try { this.server.request(up); } catch (final Throwable e) { + clearCaches(); // prevent further OOM if this was caused by OOM throw new IOException(e); } } @@ -227,6 +236,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen if (solrdoc.containsKey("_version_")) solrdoc.setField("_version_",0L); // prevent Solr "version conflict" this.server.add(solrdoc, -1); } catch (final Throwable e) { + clearCaches(); // prevent further OOM if this was caused by OOM ConcurrentLog.logException(e); // catches "version conflict for": try this again and delete the document in advance try { @@ -265,6 +275,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen } this.server.add(solrdocs, -1); } catch (final Throwable e) { + clearCaches(); // prevent further OOM if this was caused by OOM ConcurrentLog.logException(e); // catches "version conflict for": try this again and delete the document in advance List ids = new ArrayList(); @@ -314,8 +325,10 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen if (rsp != null) if (log.isFine()) log.fine(rsp.getResults().getNumFound() + " results for q=" + q); return rsp.getResults(); } catch (final SolrServerException e) { + clearCaches(); // prevent further OOM if this was caused by OOM throw new SolrException(ErrorCode.UNKNOWN, e); } catch (final Throwable e) { + clearCaches(); // prevent further OOM if this was caused by OOM throw new IOException("Error executing query", e); } }