more patches for a better out-of-memory management

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7555 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/http/server/HTTPDFileHandler.java

@@ -1220,7 +1220,12 @@ public final class HTTPDFileHandler {
     }
     
     private static final Object invokeServlet(final File targetClass, final RequestHeader request, final serverObjects args) throws IllegalArgumentException, IllegalAccessException, InvocationTargetException {
+        try {
             return rewriteMethod(targetClass).invoke(null, new Object[] {request, args, switchboard});
+        } catch (OutOfMemoryError e) {
+            Log.logException(e);
+            return null;
+        }
     }
 
     /**

source/de/anomic/search/Switchboard.java

@@ -1173,7 +1173,7 @@ public final class Switchboard extends serverSwitch {
         blogDB.close();
         blogCommentDB.close();
         userDB.close();
-        bookmarksDB.close();
+        if (bookmarksDB != null) bookmarksDB.close(); // may null if concurrent initialization was not finished
         messageDB.close();
         webStructure.close();
         crawlQueues.close();

source/de/anomic/yacy/graphics/WebStructureGraph.java

@@ -66,7 +66,12 @@ public class WebStructureGraph {
         this.structureFile = structureFile;
         
         // load web structure
-        final Map<String, String> loadedStructure = (this.structureFile.exists()) ? FileUtils.loadMap(this.structureFile) : new TreeMap<String, String>();
+        Map<String, String> loadedStructure;
+        try {
+            loadedStructure = (this.structureFile.exists()) ? FileUtils.loadMap(this.structureFile) : new TreeMap<String, String>();
+        } catch (OutOfMemoryError e) {
+            loadedStructure = new TreeMap<String, String>();
+        }
         if (loadedStructure != null) this.structure_old.putAll(loadedStructure);
         
         // delete out-dated entries in case the structure is too big

source/de/anomic/yacy/yacySeedDB.java

@@ -1017,7 +1017,12 @@ public final class yacySeedDB implements AlternativeDomainNames {
                 Map<String, String> dna0;
                 ConcurrentHashMap<String, String> dna;
                 while (it.hasNext()) {
+                    try {
                         dna0 = it.next();
+                    } catch (OutOfMemoryError e) {
+                        Log.logException(e);
+                        dna0 = null;
+                    }
                     assert dna0 != null;
                     if (dna0 == null) continue;
                     if (dna0 instanceof ConcurrentHashMap) {

source/net/yacy/cora/protocol/Domains.java

@@ -39,6 +39,7 @@ import java.util.regex.Pattern;
 
 import net.yacy.cora.storage.ARC;
 import net.yacy.cora.storage.ConcurrentARC;
+import net.yacy.kelondro.util.MemoryControl;
 
 public class Domains {
 
@@ -484,6 +485,11 @@ public class Domains {
         InetAddress ip = parseInetAddress(host);
         if (ip != null) return ip;
         
+        if (MemoryControl.shortStatus()) {
+            NAME_CACHE_HIT.clear();
+            NAME_CACHE_MISS.clear();
+        }
+        
         // try to resolve host by doing a name cache lookup
         ip = NAME_CACHE_HIT.get(host);
         if (ip != null) return ip;

source/net/yacy/document/parser/sitemapParser.java

@@ -157,6 +157,7 @@ public class sitemapParser extends AbstractParser implements Parser {
             catch (ParserConfigurationException e) { throw new IOException (e); }
             catch (SAXParseException e) { throw new IOException (e); }
             catch (SAXException e) { throw new IOException (e); }
+            catch (OutOfMemoryError e) { throw new IOException (e); }
             NodeList sitemapNodes = doc.getElementsByTagName("sitemap");
             for (int i = 0; i < sitemapNodes.getLength(); i++) {
                 String url = new SitemapEntry((Element) sitemapNodes.item(i)).url();

source/net/yacy/kelondro/blob/Gap.java

@@ -61,7 +61,12 @@ public class Gap extends TreeMap<Long, Integer> {
     public Gap(final File file) throws IOException {
         super();
         // read the index dump and fill the index
-        DataInputStream is = new DataInputStream(new BufferedInputStream(new FileInputStream(file), 1024 * 1024));
+        DataInputStream is;
+        try {
+            is = new DataInputStream(new BufferedInputStream(new FileInputStream(file), 1024 * 1024));
+        } catch (OutOfMemoryError e) {
+            is = new DataInputStream(new FileInputStream(file));
+        }
         long p;
         int l;
         while (true) {

source/net/yacy/kelondro/blob/Heap.java

@@ -40,6 +40,7 @@ import net.yacy.kelondro.logging.Log;
 import net.yacy.kelondro.order.ByteOrder;
 import net.yacy.kelondro.order.CloneableIterator;
 import net.yacy.kelondro.order.NaturalOrder;
+import net.yacy.kelondro.util.MemoryControl;
 
 
 public final class Heap extends HeapModifier implements BLOB {
@@ -349,7 +350,7 @@ public final class Heap extends HeapModifier implements BLOB {
             assert this.buffer != null;
             
             // if there is not enough space in the buffer, flush all
-            if (this.buffersize + b.length > buffermax) {
+            if (this.buffersize + b.length > buffermax || MemoryControl.shortStatus()) {
                 // this is too big. Flush everything
                 super.shrinkWithGapsAtEnd();
                 flushBuffer();

source/net/yacy/kelondro/blob/MapHeap.java

@@ -50,6 +50,7 @@ import net.yacy.kelondro.order.CloneableIterator;
 import net.yacy.kelondro.order.NaturalOrder;
 import net.yacy.kelondro.order.RotateIterator;
 import net.yacy.kelondro.util.FileUtils;
+import net.yacy.kelondro.util.MemoryControl;
 import net.yacy.kelondro.util.kelondroException;
 
 public class MapHeap implements Map<byte[], Map<String, String>> {
@@ -155,10 +156,14 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
                 if (blob != null) blob.insert(key, sb);
     
                 // write map to cache
+                if (MemoryControl.shortStatus()) {
+                    cache.clear();
+                } else {
                     cache.put(key, newMap);
                 }
             }
         }
+    }
     
     public Map<String, String> put(byte[] key, final Map<String, String> newMap) {
         Map<String, String> v = null;
@@ -283,9 +288,13 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
                     throw new IOException(e.getMessage());
                 }
         
+                if (MemoryControl.shortStatus()) {
+                    cache.clear();
+                } else {
                     // write map to cache
                     cache.put(key, map);
                 }
+            }
             
             // return value
             return map;

source/net/yacy/kelondro/data/word/Word.java

@@ -56,7 +56,14 @@ public class Word {
     public static final int commonHashLength = 12;
     
     private static final int hashCacheSize = Math.max(100000, Math.min(10000000, (int) (MemoryControl.available() / 20000L)));
-    private static final ARC<String, byte[]> hashCache = new ConcurrentARC<String, byte[]>(hashCacheSize, 2 * Runtime.getRuntime().availableProcessors());
+    private static ARC<String, byte[]> hashCache = null;
+    static {
+        try {
+            hashCache = new ConcurrentARC<String, byte[]>(hashCacheSize, 2 * Runtime.getRuntime().availableProcessors());
+        } catch (OutOfMemoryError e) {
+            hashCache = new ConcurrentARC<String, byte[]>(1000, Runtime.getRuntime().availableProcessors());
+        }
+    }
     
     // object carries statistics for words and sentences
     public  int      count;       // number of occurrences
@@ -112,7 +119,11 @@ public class Word {
         // calculate the hash
     	h = Base64Order.enhancedCoder.encodeSubstring(Digest.encodeMD5Raw(wordlc), commonHashLength);
         assert h[2] != '@';
+        if (MemoryControl.shortStatus()) {
+            hashCache.clear();
+        } else {
             hashCache.put(wordlc, h); // prevent expensive MD5 computation and encoding
+        }
         return h;
     }
     

source/net/yacy/kelondro/index/HandleMap.java

@@ -79,7 +79,12 @@ public final class HandleMap implements Iterable<Row.Entry> {
     public HandleMap(final int keylength, final ByteOrder objectOrder, final int idxbytes, final File file) throws IOException, RowSpaceExceededException {
         this(keylength, objectOrder, idxbytes, (int) (file.length() / (keylength + idxbytes)), file.getAbsolutePath());
         // read the index dump and fill the index
-        InputStream is = new BufferedInputStream(new FileInputStream(file), 1024 * 1024);
+        InputStream is;
+        try {
+            is = new BufferedInputStream(new FileInputStream(file), 1024 * 1024);
+        } catch (OutOfMemoryError e) {
+            is = new FileInputStream(file);
+        }
         if (file.getName().endsWith(".gz")) is = new GZIPInputStream(is);
         final byte[] a = new byte[keylength + idxbytes];
         int c;

source/net/yacy/kelondro/io/ByteCountInputStream.java

@@ -31,6 +31,8 @@ import java.io.IOException;
 import java.io.InputStream;
 //import java.util.HashMap;
 
+import net.yacy.kelondro.logging.Log;
+
 public final class ByteCountInputStream extends FilterInputStream {
     
 //    private final static Object syncObject = new Object();
@@ -115,7 +117,11 @@ public final class ByteCountInputStream extends FilterInputStream {
 //    }
     
     public final void close() throws IOException {
+        try {
             super.close();
+        } catch (OutOfMemoryError e) {
+            Log.logException(e);
+        }
         this.finish();
     }
     

source/net/yacy/kelondro/logging/Log.java

@@ -22,10 +22,12 @@
 
 package net.yacy.kelondro.logging;
 
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.PrintStream;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.logging.FileHandler;
@@ -381,7 +383,11 @@ public final class Log {
             Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler(){
                 public void uncaughtException(final Thread t, final Throwable e) {
                     String msg = String.format("Thread %s: %s",t.getName(), e.getMessage());
-                    exceptionLog.logSevere(msg, e);
+                    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+                    PrintStream ps = new PrintStream(baos);
+                    e.printStackTrace(ps);
+                    ps.close();
+                    exceptionLog.logSevere(msg + "\n" + baos.toString(), e);
                     //System.err.print("Exception in thread \"" + t.getName() + "\" ");
                     //e.printStackTrace(System.err);
                 }

source/net/yacy/kelondro/util/MemoryControl.java

@@ -35,18 +35,16 @@ public class MemoryControl {
 	
     private static final Runtime runtime = Runtime.getRuntime();
     public static long maxMemory = runtime.maxMemory(); // this value does never change during runtime
-    
 	private static final Log log = new Log("MEMORY");
     
     private static final long[] gcs = new long[5];
     private static int gcs_pos = 0;
-
     private static long lastGC = 0l;
-    
     private static long DHTMbyte = 0L;
     private static long prevDHTtreshold = 0L;
     private static int DHTtresholdCount = 0;
     private static boolean allowDHT = true;
+    private static boolean shortStatus = false;
 
     /**
      * Runs the garbage collector if last garbage collection is more than last millis ago
@@ -139,6 +137,11 @@ public class MemoryControl {
      * @return whether enough memory could be freed (or is free) or not
      */
     public static boolean request(final long size, final boolean force) {
+        boolean r = request0(size, force);
+        shortStatus = !r;
+        return r;
+    }
+    private static boolean request0(final long size, final boolean force) {
     	final long avg = getAverageGCFree();
     	if (avg >= size) return true;
         long avail = available();
@@ -168,6 +171,10 @@ public class MemoryControl {
         }
     }
     
+    public static boolean shortStatus() {
+        return shortStatus;
+    }
+    
     /**
      * memory that is currently bound in objects
      * @return used bytes

source/net/yacy/visualization/RasterPlotter.java

@@ -68,7 +68,7 @@ public class RasterPlotter {
     
     protected final int            width, height;
     private final   int[]          cc;
-    private final   BufferedImage  image;
+    private   BufferedImage  image;
     private final   WritableRaster grid;
     private         int            defaultColR, defaultColG, defaultColB;
     private final   long           backgroundCol;
@@ -79,9 +79,6 @@ public class RasterPlotter {
     }
     
     public RasterPlotter(final int width, final int height, final DrawMode drawMode, final long backgroundColor) {
-        if (!(MemoryControl.request(1024 * 1024 + 3 * width * height, false))) {
-            throw new RuntimeException(RasterPlotter.class.getSimpleName() + ": not enough memory (" + MemoryControl.available() + ") available");
-        }
         this.cc = new int[3];
         this.width = width;
         this.height = height;
@@ -90,8 +87,12 @@ public class RasterPlotter {
         this.defaultColG = 0xFF;
         this.defaultColB = 0xFF;
         this.defaultMode = drawMode;
-        
+        try {
             this.image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
+        } catch (OutOfMemoryError e) {
+            this.image = new BufferedImage(1, 1, BufferedImage.TYPE_INT_RGB);
+            //throw new RuntimeException(RasterPlotter.class.getSimpleName() + ": not enough memory (" + MemoryControl.available() + ") available");
+        }
         this.clear();
         this.grid = image.getRaster();
     }