healing for some OOM problems

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7502 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/server/serverCore.java

@@ -726,8 +726,22 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
                                 parameter[0] = this.request.trim();
                             }
                         }
-                        //long commandStart = System.currentTimeMillis();
-                        Object result = commandMethod.invoke(this.commandObj, parameter);
+                        
+                        Object result = null;
+                        try {
+                            result = commandMethod.invoke(this.commandObj, parameter);
+                        } catch (OutOfMemoryError e) {
+                            // try again
+                            terminateOldSessions(2000);
+                            try {
+                                result = commandMethod.invoke(this.commandObj, parameter);
+                            } catch (OutOfMemoryError e2) {
+                                // try again
+                                Thread.sleep(1000);
+                                result = commandMethod.invoke(this.commandObj, parameter);
+                            }
+                        }
+                        
                         //announceMoreExecTime(commandStart - System.currentTimeMillis()); // shall be negative!
                         //this.log.logDebug("* session " + handle + " completed command '" + request + "'. time = " + (System.currentTimeMillis() - handle));
                         this.out.flush();

source/net/yacy/cora/protocol/http/HTTPClient.java

@@ -445,8 +445,12 @@ public class HTTPClient {
             // get the response body
             final HttpEntity httpEntity = httpResponse.getEntity();
             if (httpEntity != null) {
-                if (getStatusCode() == 200 && httpEntity.getContentLength()  < maxBytes) {
+                if (getStatusCode() == 200 && httpEntity.getContentLength() < maxBytes) {
+                    try {
                         content = EntityUtils.toByteArray(httpEntity);
+                    } catch (OutOfMemoryError e) {
+                        throw new IOException(e.toString());
+                    }
                 } 
                 // Ensures that the entity content is fully consumed and the content stream, if exists, is closed.
             	EntityUtils.consume(httpEntity);

source/net/yacy/kelondro/blob/BLOB.java

@@ -189,8 +189,9 @@ public interface BLOB {
          * of the input
          * @param b
          * @return an array that is equal or smaller in size than b
+         * @throws RowSpaceExceededException 
          */
-        public byte[] rewrite(byte[] b);
+        public byte[] rewrite(byte[] b) throws RowSpaceExceededException;
         
     }
     

source/net/yacy/kelondro/blob/HeapModifier.java

@@ -28,6 +28,7 @@ import java.io.File;
 import java.io.IOException;
 import java.util.SortedMap;
 
+import net.yacy.kelondro.index.RowSpaceExceededException;
 import net.yacy.kelondro.io.CachedFileWriter;
 import net.yacy.kelondro.logging.Log;
 import net.yacy.kelondro.order.ByteOrder;
@@ -242,7 +243,7 @@ public class HeapModifier extends HeapReader implements BLOB {
 	    throw new UnsupportedOperationException();
     }
 	
-	public int reduce(byte[] key, final Reducer reducer) throws IOException {
+	public int reduce(byte[] key, final Reducer reducer) throws IOException, RowSpaceExceededException {
         key = normalizeKey(key);
         assert key.length == this.keylength;
      

source/net/yacy/kelondro/index/HandleSet.java

@@ -64,7 +64,7 @@ public final class HandleSet implements Iterable<byte[]>, Cloneable {
         this.index = index;
     }
 
-    public HandleSet(Row rowdef, byte[] b) {
+    public HandleSet(Row rowdef, byte[] b) throws RowSpaceExceededException {
         this.rowdef = rowdef;
         this.index = RowSet.importRowSet(b, this.rowdef);
     }

source/net/yacy/kelondro/index/RowSet.java

@@ -32,6 +32,7 @@ import net.yacy.kelondro.logging.Log;
 import net.yacy.kelondro.order.Base64Order;
 import net.yacy.kelondro.order.CloneableIterator;
 import net.yacy.kelondro.order.NaturalOrder;
+import net.yacy.kelondro.util.MemoryControl;
 
 
 public class RowSet extends RowCollection implements Index, Iterable<Row.Entry> {
@@ -68,7 +69,7 @@ public class RowSet extends RowCollection implements Index, Iterable<Row.Entry>
         assert rowdef.objectOrder != null;
     }
     
-    public final static RowSet importRowSet(final byte[] b, final Row rowdef) {
+    public final static RowSet importRowSet(final byte[] b, final Row rowdef) throws RowSpaceExceededException {
     	assert b.length >= exportOverheadSize : "b.length = " + b.length;
     	if (b.length < exportOverheadSize) return new RowSet(rowdef);
         final int size = (int) NaturalOrder.decodeLong(b, 0, 4);
@@ -80,7 +81,13 @@ public class RowSet extends RowCollection implements Index, Iterable<Row.Entry>
         long alloc = ((long) size) * ((long) rowdef.objectsize);
         assert alloc <= Integer.MAX_VALUE : "alloc = " + alloc;
         assert alloc == b.length - exportOverheadSize;
-        final byte[] chunkcache = new byte[(int) alloc];
+        MemoryControl.request((int) alloc, true);
+        final byte[] chunkcache;
+        try {
+            chunkcache = new byte[(int) alloc];
+        } catch (OutOfMemoryError e) {
+            throw new RowSpaceExceededException((int) alloc, "importRowSet");
+        }
         //assert b.length - exportOverheadSize == size * rowdef.objectsize : "b.length = " + b.length + ", size * rowdef.objectsize = " + size * rowdef.objectsize;
         if (b.length - exportOverheadSize != alloc) {
             Log.logSevere("RowSet", "exportOverheadSize wrong: b.length = " + b.length + ", size * rowdef.objectsize = " + size * rowdef.objectsize);

source/net/yacy/kelondro/rwi/ReferenceContainerArray.java

@@ -291,7 +291,7 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
             this.wordHash = wordHash;
         }
         
-        public byte[] rewrite(byte[] b) {
+        public byte[] rewrite(byte[] b) throws RowSpaceExceededException {
             if (b == null) return null;
             ReferenceContainer<ReferenceType> c = rewriter.reduce(new ReferenceContainer<ReferenceType>(factory, this.wordHash, RowSet.importRowSet(b, payloadrow)));
             if (c == null) return null;

source/net/yacy/kelondro/rwi/ReferenceIterator.java

@@ -34,6 +34,7 @@ import java.util.Map;
 import net.yacy.kelondro.blob.HeapReader;
 import net.yacy.kelondro.index.Row;
 import net.yacy.kelondro.index.RowSet;
+import net.yacy.kelondro.index.RowSpaceExceededException;
 import net.yacy.kelondro.logging.Log;
 import net.yacy.kelondro.order.CloneableIterator;
 
@@ -67,7 +68,12 @@ public class ReferenceIterator <ReferenceType extends Reference> implements Clon
     public ReferenceContainer<ReferenceType> next() {
         Map.Entry<byte[], byte[]> entry = blobs.next();
         byte[] payload = entry.getValue();
-        return new ReferenceContainer<ReferenceType>(factory, entry.getKey(), RowSet.importRowSet(payload, payloadrow));
+        try {
+            return new ReferenceContainer<ReferenceType>(factory, entry.getKey(), RowSet.importRowSet(payload, payloadrow));
+        } catch (RowSpaceExceededException e) {
+            Log.logSevere("ReferenceIterator", "lost entry '" + entry.getKey() + "' because of too low memory: " + e.toString());
+            return null;
+        }
     }
     
     public void remove() {