diff --git a/source/de/anomic/http/server/HTTPDFileHandler.java b/source/de/anomic/http/server/HTTPDFileHandler.java index 13f5738fa..9cbf66b8b 100644 --- a/source/de/anomic/http/server/HTTPDFileHandler.java +++ b/source/de/anomic/http/server/HTTPDFileHandler.java @@ -1220,7 +1220,12 @@ public final class HTTPDFileHandler { } private static final Object invokeServlet(final File targetClass, final RequestHeader request, final serverObjects args) throws IllegalArgumentException, IllegalAccessException, InvocationTargetException { - return rewriteMethod(targetClass).invoke(null, new Object[] {request, args, switchboard}); + try { + return rewriteMethod(targetClass).invoke(null, new Object[] {request, args, switchboard}); + } catch (OutOfMemoryError e) { + Log.logException(e); + return null; + } } /** diff --git a/source/de/anomic/search/Switchboard.java b/source/de/anomic/search/Switchboard.java index 5a0d6958b..f82609116 100644 --- a/source/de/anomic/search/Switchboard.java +++ b/source/de/anomic/search/Switchboard.java @@ -1173,7 +1173,7 @@ public final class Switchboard extends serverSwitch { blogDB.close(); blogCommentDB.close(); userDB.close(); - bookmarksDB.close(); + if (bookmarksDB != null) bookmarksDB.close(); // may null if concurrent initialization was not finished messageDB.close(); webStructure.close(); crawlQueues.close(); diff --git a/source/de/anomic/yacy/graphics/WebStructureGraph.java b/source/de/anomic/yacy/graphics/WebStructureGraph.java index 18dfa9af2..521dfdee5 100644 --- a/source/de/anomic/yacy/graphics/WebStructureGraph.java +++ b/source/de/anomic/yacy/graphics/WebStructureGraph.java @@ -66,7 +66,12 @@ public class WebStructureGraph { this.structureFile = structureFile; // load web structure - final Map loadedStructure = (this.structureFile.exists()) ? FileUtils.loadMap(this.structureFile) : new TreeMap(); + Map loadedStructure; + try { + loadedStructure = (this.structureFile.exists()) ? FileUtils.loadMap(this.structureFile) : new TreeMap(); + } catch (OutOfMemoryError e) { + loadedStructure = new TreeMap(); + } if (loadedStructure != null) this.structure_old.putAll(loadedStructure); // delete out-dated entries in case the structure is too big diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index 1ed8e58f4..413e695be 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -1017,7 +1017,12 @@ public final class yacySeedDB implements AlternativeDomainNames { Map dna0; ConcurrentHashMap dna; while (it.hasNext()) { - dna0 = it.next(); + try { + dna0 = it.next(); + } catch (OutOfMemoryError e) { + Log.logException(e); + dna0 = null; + } assert dna0 != null; if (dna0 == null) continue; if (dna0 instanceof ConcurrentHashMap) { diff --git a/source/net/yacy/cora/protocol/Domains.java b/source/net/yacy/cora/protocol/Domains.java index cc14b9258..85fc641d1 100644 --- a/source/net/yacy/cora/protocol/Domains.java +++ b/source/net/yacy/cora/protocol/Domains.java @@ -39,6 +39,7 @@ import java.util.regex.Pattern; import net.yacy.cora.storage.ARC; import net.yacy.cora.storage.ConcurrentARC; +import net.yacy.kelondro.util.MemoryControl; public class Domains { @@ -484,6 +485,11 @@ public class Domains { InetAddress ip = parseInetAddress(host); if (ip != null) return ip; + if (MemoryControl.shortStatus()) { + NAME_CACHE_HIT.clear(); + NAME_CACHE_MISS.clear(); + } + // try to resolve host by doing a name cache lookup ip = NAME_CACHE_HIT.get(host); if (ip != null) return ip; diff --git a/source/net/yacy/document/parser/sitemapParser.java b/source/net/yacy/document/parser/sitemapParser.java index d88f53122..d000a1401 100644 --- a/source/net/yacy/document/parser/sitemapParser.java +++ b/source/net/yacy/document/parser/sitemapParser.java @@ -157,6 +157,7 @@ public class sitemapParser extends AbstractParser implements Parser { catch (ParserConfigurationException e) { throw new IOException (e); } catch (SAXParseException e) { throw new IOException (e); } catch (SAXException e) { throw new IOException (e); } + catch (OutOfMemoryError e) { throw new IOException (e); } NodeList sitemapNodes = doc.getElementsByTagName("sitemap"); for (int i = 0; i < sitemapNodes.getLength(); i++) { String url = new SitemapEntry((Element) sitemapNodes.item(i)).url(); diff --git a/source/net/yacy/kelondro/blob/Gap.java b/source/net/yacy/kelondro/blob/Gap.java index 9f46eb85c..8a3cf6f6f 100644 --- a/source/net/yacy/kelondro/blob/Gap.java +++ b/source/net/yacy/kelondro/blob/Gap.java @@ -61,7 +61,12 @@ public class Gap extends TreeMap { public Gap(final File file) throws IOException { super(); // read the index dump and fill the index - DataInputStream is = new DataInputStream(new BufferedInputStream(new FileInputStream(file), 1024 * 1024)); + DataInputStream is; + try { + is = new DataInputStream(new BufferedInputStream(new FileInputStream(file), 1024 * 1024)); + } catch (OutOfMemoryError e) { + is = new DataInputStream(new FileInputStream(file)); + } long p; int l; while (true) { diff --git a/source/net/yacy/kelondro/blob/Heap.java b/source/net/yacy/kelondro/blob/Heap.java index 47e73b5eb..1b5b2b470 100755 --- a/source/net/yacy/kelondro/blob/Heap.java +++ b/source/net/yacy/kelondro/blob/Heap.java @@ -40,6 +40,7 @@ import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.ByteOrder; import net.yacy.kelondro.order.CloneableIterator; import net.yacy.kelondro.order.NaturalOrder; +import net.yacy.kelondro.util.MemoryControl; public final class Heap extends HeapModifier implements BLOB { @@ -349,7 +350,7 @@ public final class Heap extends HeapModifier implements BLOB { assert this.buffer != null; // if there is not enough space in the buffer, flush all - if (this.buffersize + b.length > buffermax) { + if (this.buffersize + b.length > buffermax || MemoryControl.shortStatus()) { // this is too big. Flush everything super.shrinkWithGapsAtEnd(); flushBuffer(); diff --git a/source/net/yacy/kelondro/blob/MapHeap.java b/source/net/yacy/kelondro/blob/MapHeap.java index 63663343d..789d34a00 100644 --- a/source/net/yacy/kelondro/blob/MapHeap.java +++ b/source/net/yacy/kelondro/blob/MapHeap.java @@ -50,6 +50,7 @@ import net.yacy.kelondro.order.CloneableIterator; import net.yacy.kelondro.order.NaturalOrder; import net.yacy.kelondro.order.RotateIterator; import net.yacy.kelondro.util.FileUtils; +import net.yacy.kelondro.util.MemoryControl; import net.yacy.kelondro.util.kelondroException; public class MapHeap implements Map> { @@ -155,7 +156,11 @@ public class MapHeap implements Map> { if (blob != null) blob.insert(key, sb); // write map to cache - cache.put(key, newMap); + if (MemoryControl.shortStatus()) { + cache.clear(); + } else { + cache.put(key, newMap); + } } } } @@ -283,8 +288,12 @@ public class MapHeap implements Map> { throw new IOException(e.getMessage()); } - // write map to cache - cache.put(key, map); + if (MemoryControl.shortStatus()) { + cache.clear(); + } else { + // write map to cache + cache.put(key, map); + } } // return value diff --git a/source/net/yacy/kelondro/data/word/Word.java b/source/net/yacy/kelondro/data/word/Word.java index 334c95ea4..1db7ec555 100644 --- a/source/net/yacy/kelondro/data/word/Word.java +++ b/source/net/yacy/kelondro/data/word/Word.java @@ -56,7 +56,14 @@ public class Word { public static final int commonHashLength = 12; private static final int hashCacheSize = Math.max(100000, Math.min(10000000, (int) (MemoryControl.available() / 20000L))); - private static final ARC hashCache = new ConcurrentARC(hashCacheSize, 2 * Runtime.getRuntime().availableProcessors()); + private static ARC hashCache = null; + static { + try { + hashCache = new ConcurrentARC(hashCacheSize, 2 * Runtime.getRuntime().availableProcessors()); + } catch (OutOfMemoryError e) { + hashCache = new ConcurrentARC(1000, Runtime.getRuntime().availableProcessors()); + } + } // object carries statistics for words and sentences public int count; // number of occurrences @@ -112,7 +119,11 @@ public class Word { // calculate the hash h = Base64Order.enhancedCoder.encodeSubstring(Digest.encodeMD5Raw(wordlc), commonHashLength); assert h[2] != '@'; - hashCache.put(wordlc, h); // prevent expensive MD5 computation and encoding + if (MemoryControl.shortStatus()) { + hashCache.clear(); + } else { + hashCache.put(wordlc, h); // prevent expensive MD5 computation and encoding + } return h; } diff --git a/source/net/yacy/kelondro/index/HandleMap.java b/source/net/yacy/kelondro/index/HandleMap.java index 1f1194752..3db2588f9 100644 --- a/source/net/yacy/kelondro/index/HandleMap.java +++ b/source/net/yacy/kelondro/index/HandleMap.java @@ -79,7 +79,12 @@ public final class HandleMap implements Iterable { public HandleMap(final int keylength, final ByteOrder objectOrder, final int idxbytes, final File file) throws IOException, RowSpaceExceededException { this(keylength, objectOrder, idxbytes, (int) (file.length() / (keylength + idxbytes)), file.getAbsolutePath()); // read the index dump and fill the index - InputStream is = new BufferedInputStream(new FileInputStream(file), 1024 * 1024); + InputStream is; + try { + is = new BufferedInputStream(new FileInputStream(file), 1024 * 1024); + } catch (OutOfMemoryError e) { + is = new FileInputStream(file); + } if (file.getName().endsWith(".gz")) is = new GZIPInputStream(is); final byte[] a = new byte[keylength + idxbytes]; int c; diff --git a/source/net/yacy/kelondro/io/ByteCountInputStream.java b/source/net/yacy/kelondro/io/ByteCountInputStream.java index 39e807a8e..faa2e0aba 100644 --- a/source/net/yacy/kelondro/io/ByteCountInputStream.java +++ b/source/net/yacy/kelondro/io/ByteCountInputStream.java @@ -31,6 +31,8 @@ import java.io.IOException; import java.io.InputStream; //import java.util.HashMap; +import net.yacy.kelondro.logging.Log; + public final class ByteCountInputStream extends FilterInputStream { // private final static Object syncObject = new Object(); @@ -115,7 +117,11 @@ public final class ByteCountInputStream extends FilterInputStream { // } public final void close() throws IOException { - super.close(); + try { + super.close(); + } catch (OutOfMemoryError e) { + Log.logException(e); + } this.finish(); } diff --git a/source/net/yacy/kelondro/logging/Log.java b/source/net/yacy/kelondro/logging/Log.java index 1151604a9..9bfb7a63c 100644 --- a/source/net/yacy/kelondro/logging/Log.java +++ b/source/net/yacy/kelondro/logging/Log.java @@ -22,10 +22,12 @@ package net.yacy.kelondro.logging; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.PrintStream; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.logging.FileHandler; @@ -381,7 +383,11 @@ public final class Log { Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler(){ public void uncaughtException(final Thread t, final Throwable e) { String msg = String.format("Thread %s: %s",t.getName(), e.getMessage()); - exceptionLog.logSevere(msg, e); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(baos); + e.printStackTrace(ps); + ps.close(); + exceptionLog.logSevere(msg + "\n" + baos.toString(), e); //System.err.print("Exception in thread \"" + t.getName() + "\" "); //e.printStackTrace(System.err); } diff --git a/source/net/yacy/kelondro/util/MemoryControl.java b/source/net/yacy/kelondro/util/MemoryControl.java index 3525deb9e..25aa18a3c 100644 --- a/source/net/yacy/kelondro/util/MemoryControl.java +++ b/source/net/yacy/kelondro/util/MemoryControl.java @@ -35,18 +35,16 @@ public class MemoryControl { private static final Runtime runtime = Runtime.getRuntime(); public static long maxMemory = runtime.maxMemory(); // this value does never change during runtime - private static final Log log = new Log("MEMORY"); private static final long[] gcs = new long[5]; private static int gcs_pos = 0; - private static long lastGC = 0l; - private static long DHTMbyte = 0L; private static long prevDHTtreshold = 0L; private static int DHTtresholdCount = 0; private static boolean allowDHT = true; + private static boolean shortStatus = false; /** * Runs the garbage collector if last garbage collection is more than last millis ago @@ -139,6 +137,11 @@ public class MemoryControl { * @return whether enough memory could be freed (or is free) or not */ public static boolean request(final long size, final boolean force) { + boolean r = request0(size, force); + shortStatus = !r; + return r; + } + private static boolean request0(final long size, final boolean force) { final long avg = getAverageGCFree(); if (avg >= size) return true; long avail = available(); @@ -168,6 +171,10 @@ public class MemoryControl { } } + public static boolean shortStatus() { + return shortStatus; + } + /** * memory that is currently bound in objects * @return used bytes diff --git a/source/net/yacy/visualization/RasterPlotter.java b/source/net/yacy/visualization/RasterPlotter.java index 949a69976..ff3869ce2 100644 --- a/source/net/yacy/visualization/RasterPlotter.java +++ b/source/net/yacy/visualization/RasterPlotter.java @@ -68,7 +68,7 @@ public class RasterPlotter { protected final int width, height; private final int[] cc; - private final BufferedImage image; + private BufferedImage image; private final WritableRaster grid; private int defaultColR, defaultColG, defaultColB; private final long backgroundCol; @@ -79,9 +79,6 @@ public class RasterPlotter { } public RasterPlotter(final int width, final int height, final DrawMode drawMode, final long backgroundColor) { - if (!(MemoryControl.request(1024 * 1024 + 3 * width * height, false))) { - throw new RuntimeException(RasterPlotter.class.getSimpleName() + ": not enough memory (" + MemoryControl.available() + ") available"); - } this.cc = new int[3]; this.width = width; this.height = height; @@ -90,8 +87,12 @@ public class RasterPlotter { this.defaultColG = 0xFF; this.defaultColB = 0xFF; this.defaultMode = drawMode; - - this.image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB); + try { + this.image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB); + } catch (OutOfMemoryError e) { + this.image = new BufferedImage(1, 1, BufferedImage.TYPE_INT_RGB); + //throw new RuntimeException(RasterPlotter.class.getSimpleName() + ": not enough memory (" + MemoryControl.available() + ") available"); + } this.clear(); this.grid = image.getRaster(); }