try to recover from an OOM during citation index reading and fail-over

to second solr core in case of unrecoverable OOM.

source/net/yacy/kelondro/blob/HeapReader.java

@@ -539,7 +539,13 @@ public class HeapReader {
             try {
                 blob = new byte[len];
             } catch (final OutOfMemoryError e) {
-                throw new SpaceExceededException(len, "HeapReader.get()/blob");
+                // try once again after GC
+                MemoryControl.gc(1000, "HeapReader.get()/blob");
+                try {
+                    blob = new byte[len];
+                } catch (final OutOfMemoryError ee) {
+                    throw new SpaceExceededException(len, "HeapReader.get()/blob");
+                }
             }
             this.file.readFully(blob, 0, blob.length);
 

source/net/yacy/search/index/Segment.java

@@ -360,24 +360,29 @@ public class Segment {
             this.externalHosts = new RowHandleSet(6, Base64Order.enhancedCoder, 0);
             this.internalIDs = new RowHandleSet(12, Base64Order.enhancedCoder, 0);
             this.externalIDs = new RowHandleSet(12, Base64Order.enhancedCoder, 0);
-            if (connectedCitation()) {
+            try {
-                // read the references from the citation index
+                if (connectedCitation()) {
-                ReferenceContainer<CitationReference> references;
+                    // read the references from the citation index
-                references = urlCitation().get(id, null);
+                    ReferenceContainer<CitationReference> references;
-                if (references == null) return; // no references at all
+                    references = urlCitation().get(id, null);
-                Iterator<CitationReference> ri = references.entries();
+                    if (references == null) return; // no references at all
-                while (ri.hasNext()) {
+                    Iterator<CitationReference> ri = references.entries();
-                    CitationReference ref = ri.next();
+                    while (ri.hasNext()) {
-                    byte[] hh = ref.hosthash(); // host hash
+                        CitationReference ref = ri.next();
-                    if (ByteBuffer.equals(hh, 0, id, 6, 6)) {
+                        byte[] hh = ref.hosthash(); // host hash
-                        internalIDs.put(ref.urlhash());
+                        if (ByteBuffer.equals(hh, 0, id, 6, 6)) {
-                        internal++;
+                            internalIDs.put(ref.urlhash());
-                    } else {
+                            internal++;
-                        externalHosts.put(hh);
+                        } else {
-                        externalIDs.put(ref.urlhash());
+                            externalHosts.put(hh);
-                        external++;
+                            externalIDs.put(ref.urlhash());
+                            external++;
+                        }
                     }
                 }
+            } catch (SpaceExceededException e) {
+                // the Citation Index got too large, we ignore the problem and hope that a second solr index is attached which will take over now
+                if (Segment.this.fulltext.writeToWebgraph()) internalIDs.clear();
             }
             if ((internalIDs.size() == 0 || !connectedCitation()) && Segment.this.fulltext.writeToWebgraph()) {
                 // reqd the references from the webgraph