diff --git a/htroot/PerformanceMemory_p.java b/htroot/PerformanceMemory_p.java index f04599f13..35e4783b6 100644 --- a/htroot/PerformanceMemory_p.java +++ b/htroot/PerformanceMemory_p.java @@ -50,6 +50,7 @@ import java.io.File; import de.anomic.http.httpc; import de.anomic.http.httpHeader; import de.anomic.plasma.plasmaSwitchboard; +import de.anomic.server.serverMemory; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.server.serverFileUtils; @@ -138,7 +139,7 @@ public class PerformanceMemory_p { long memoryTotalAfterInitBGC = Long.parseLong(env.getConfig("memoryTotalAfterInitBGC", "0")); long memoryTotalAfterInitAGC = Long.parseLong(env.getConfig("memoryTotalAfterInitAGC", "0")); long memoryTotalAfterStartup = Long.parseLong(env.getConfig("memoryTotalAfterStartup", "0")); - long memoryMax = Runtime.getRuntime().maxMemory(); + long memoryMax = serverMemory.max; prop.put("memoryMax", memoryMax / MB); prop.put("memoryAvailAfterStartup", (memoryMax - memoryTotalAfterStartup + memoryFreeAfterStartup) / MB); diff --git a/htroot/Status.java b/htroot/Status.java index 24ff245b1..3a9e71b2b 100644 --- a/htroot/Status.java +++ b/htroot/Status.java @@ -56,6 +56,7 @@ import de.anomic.http.httpdByteCountOutputStream; import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverCore; import de.anomic.server.serverDate; +import de.anomic.server.serverMemory; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.yacy.yacyCore; @@ -270,7 +271,7 @@ public class Status { // memory usage and system attributes prop.put("freeMemory", bytesToString(rt.freeMemory())); prop.put("totalMemory", bytesToString(rt.totalMemory())); - prop.put("maxMemory", bytesToString(rt.maxMemory())); + prop.put("maxMemory", bytesToString(serverMemory.max)); prop.put("processors", rt.availableProcessors()); // proxy traffic diff --git a/source/de/anomic/kelondro/kelondroAttrSeq.java b/source/de/anomic/kelondro/kelondroAttrSeq.java index 180619be0..45ebe89c7 100644 --- a/source/de/anomic/kelondro/kelondroAttrSeq.java +++ b/source/de/anomic/kelondro/kelondroAttrSeq.java @@ -64,6 +64,7 @@ import java.util.logging.Logger; import de.anomic.server.serverFileUtils; +import de.anomic.server.serverMemory; public class kelondroAttrSeq { @@ -448,12 +449,11 @@ public class kelondroAttrSeq { } } - private static final Runtime runtime = Runtime.getRuntime(); private static final long cc = 0; private static boolean shortmemstate = false; private static boolean shortmem() { if ((cc % 300) == 0) { - shortmemstate = (runtime.freeMemory() < 20000000L); + shortmemstate = (serverMemory.available() < 20000000L); } return shortmemstate; } diff --git a/source/de/anomic/kelondro/kelondroObjectCache.java b/source/de/anomic/kelondro/kelondroObjectCache.java index 5ac8a232b..38f529eba 100644 --- a/source/de/anomic/kelondro/kelondroObjectCache.java +++ b/source/de/anomic/kelondro/kelondroObjectCache.java @@ -58,6 +58,8 @@ package de.anomic.kelondro; import java.util.TreeMap; +import de.anomic.server.serverMemory; + public class kelondroObjectCache { private final TreeMap cache; @@ -288,7 +290,7 @@ public class kelondroObjectCache { ((k = (String) ages.getMinObject()) != null) && ((ages.size() > maxSize) || (((System.currentTimeMillis() - longEmit(ages.getScore(k))) > maxAge) && - (Runtime.getRuntime().freeMemory() < minMem))) + (serverMemory.available() < minMem))) ) { cache.remove(k); ages.deleteScore(k); @@ -304,7 +306,7 @@ public class kelondroObjectCache { ((k = (String) hasnot.getMinObject()) != null) && ((hasnot.size() > maxSize) || (((System.currentTimeMillis() - longEmit(hasnot.getScore(k))) > maxAge) && - (Runtime.getRuntime().freeMemory() < minMem))) + (serverMemory.available() < minMem))) ) { hasnot.deleteScore(k); hasnotFlush++; diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java index 2f962dfc6..bc93a5981 100644 --- a/source/de/anomic/kelondro/kelondroRecords.java +++ b/source/de/anomic/kelondro/kelondroRecords.java @@ -78,6 +78,8 @@ import java.util.Iterator; import java.util.TreeSet; import java.util.logging.Logger; +import de.anomic.server.serverMemory; + public class kelondroRecords { // constants @@ -449,14 +451,6 @@ public class kelondroRecords { } } - private static final long max = Runtime.getRuntime().maxMemory(); - private static final Runtime runtime = Runtime.getRuntime(); - - public static long availableMemory() { - // memory that is available including increasing total memory up to maximum - return max - runtime.totalMemory() + runtime.freeMemory(); - } - public File file() { if (filename == null) return null; return new File(filename); @@ -837,7 +831,7 @@ public class kelondroRecords { // returns false if the cache is considered to be full if (cacheSize == 0) return false; // no caching if (cacheHeaders.size() == 0) return true; // nothing there to flush - if ((cacheHeaders.size() < cacheSize) && (availableMemory() >= memBlock)) return true; // no need to flush cache space + if ((cacheHeaders.size() < cacheSize) && (serverMemory.available() >= memBlock)) return true; // no need to flush cache space // just delete any of the entries cacheHeaders.removeOne(); diff --git a/source/de/anomic/kelondro/kelondroRowBufferedSet.java b/source/de/anomic/kelondro/kelondroRowBufferedSet.java index 9703869c4..9e4667aed 100644 --- a/source/de/anomic/kelondro/kelondroRowBufferedSet.java +++ b/source/de/anomic/kelondro/kelondroRowBufferedSet.java @@ -28,6 +28,8 @@ import java.util.Map; import java.util.Iterator; import java.util.TreeMap; +import de.anomic.server.serverMemory; + public class kelondroRowBufferedSet extends kelondroRowSet { private static final long memBlockLimit = 2000000; // do not fill cache further if the amount of available memory is less that this @@ -136,7 +138,7 @@ public class kelondroRowBufferedSet extends kelondroRowSet { if (oldentry == null) { // this was not anywhere buffer.put(key, newentry); - if (((buffer.size() > bufferFlushMinimum) && (kelondroRecords.availableMemory() > memBlockLimit)) || + if (((buffer.size() > bufferFlushMinimum) && (serverMemory.available() > memBlockLimit)) || (buffer.size() > bufferFlushLimit)) flush(); } else { // replace old entry diff --git a/source/de/anomic/plasma/plasmaRankingCRProcess.java b/source/de/anomic/plasma/plasmaRankingCRProcess.java index df6b37c0a..983b9365c 100644 --- a/source/de/anomic/plasma/plasmaRankingCRProcess.java +++ b/source/de/anomic/plasma/plasmaRankingCRProcess.java @@ -59,6 +59,7 @@ import de.anomic.kelondro.kelondroRow; import de.anomic.kelondro.kelondroRowSet; import de.anomic.server.serverFileUtils; import de.anomic.server.serverDate; +import de.anomic.server.serverMemory; import de.anomic.tools.bitfield; public class plasmaRankingCRProcess { @@ -376,7 +377,7 @@ public class plasmaRankingCRProcess { count++; if ((count % 1000) == 0) { l = java.lang.Math.max(1, (System.currentTimeMillis() - start) / 1000); - System.out.println("processed " + count + " citations, " + (count / l) + " per second, rci.size = " + rci.size() + ", " + ((size - count) / (count / l)) + " seconds remaining; mem = " + Runtime.getRuntime().freeMemory()); + System.out.println("processed " + count + " citations, " + (count / l) + " per second, rci.size = " + rci.size() + ", " + ((size - count) / (count / l)) + " seconds remaining; mem = " + serverMemory.available()); } i.remove(); } diff --git a/source/de/anomic/server/serverAbstractThread.java b/source/de/anomic/server/serverAbstractThread.java index f8f009983..bf03ede45 100644 --- a/source/de/anomic/server/serverAbstractThread.java +++ b/source/de/anomic/server/serverAbstractThread.java @@ -245,7 +245,7 @@ public abstract class serverAbstractThread extends Thread implements serverThrea long timestamp; long memstamp0, memstamp1; boolean isBusy; - Runtime rt = Runtime.getRuntime(); + //Runtime rt = Runtime.getRuntime(); while (running) { if ((this.intermissionObedient) && (this.intermission > 0) && (this.intermission != Long.MAX_VALUE)) { @@ -266,7 +266,7 @@ public abstract class serverAbstractThread extends Thread implements serverThrea timestamp = System.currentTimeMillis(); ratz(this.idlePause); idletime += System.currentTimeMillis() - timestamp; - } else if (rt.freeMemory() > memprereq) try { + } else if (serverMemory.available() > memprereq) try { // do job timestamp = System.currentTimeMillis(); memstamp0 = serverMemory.used(); @@ -305,7 +305,6 @@ public abstract class serverAbstractThread extends Thread implements serverThrea ratz(this.idlePause); idletime += System.currentTimeMillis() - timestamp; outofmemoryCycles++; - if (rt.freeMemory() <= memprereq) System.gc(); // give next loop a chance } } this.close(); diff --git a/source/de/anomic/server/serverMemory.java b/source/de/anomic/server/serverMemory.java index ab33cdfa1..939ed7ff4 100644 --- a/source/de/anomic/server/serverMemory.java +++ b/source/de/anomic/server/serverMemory.java @@ -49,7 +49,8 @@ import java.text.DecimalFormat; public class serverMemory { - public static final long max = Runtime.getRuntime().maxMemory(); + public static boolean vm15 = System.getProperty("java.vm.version").startsWith("1.5"); + public static final long max = (vm15) ? Runtime.getRuntime().maxMemory() : computedMaxMemory(); // patch for maxMemory bug in Java 1.4.2 private static final Runtime runtime = Runtime.getRuntime(); public static long free() { @@ -90,5 +91,44 @@ public class serverMemory { } catch (Exception e) { return "unknown"; } - } + } + + private static int computedMaxMemory() { + // there is a bug in java 1.4.2 for maxMemory() + // see for bug description: + // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4686462 + // to get the correct maxMemory, we force a OutOfMemoryError here to measure the 'real' maxMemory() + int mb = 1024 * 1024; + byte[][] x = new byte[2048][]; + for (int i = 0; i < x.length; i++) { + try { + x[i] = new byte[mb]; + } catch (OutOfMemoryError e) { + x = null; // free memory + //System.out.println("* computed maxMemory = " + i + " mb"); + return (int) Math.max(i * mb, Runtime.getRuntime().totalMemory()); + } + } + return 2048 * mb; + } + + public static void main(String[] args) { + // try this with a jvm 1.4.2 and with a jvm 1.5 and compare results + int mb = 1024 * 1024; + System.out.println("vm: " + System.getProperty("java.vm.version")); + System.out.println("computed max = " + (computedMaxMemory() / mb) + " mb"); + int alloc = 10000; + Runtime rt = Runtime.getRuntime(); + byte[][] x = new byte[100000][]; + for (int i = 0; i < 100000; i++) { + x[i] = new byte[alloc]; + if (i % 100 == 0) System.out.println("used = " + (i * alloc / mb) + + ", total = " + (rt.totalMemory() / mb) + + ", free = " + (rt.freeMemory() / mb) + + ", max = " + (rt.maxMemory() / mb) + + ", avail = " + ((rt.maxMemory() - rt.totalMemory() + rt.freeMemory()) / mb)); + } + + } + } diff --git a/source/yacy.java b/source/yacy.java index a5c3963f6..121217e07 100644 --- a/source/yacy.java +++ b/source/yacy.java @@ -90,6 +90,7 @@ import de.anomic.plasma.plasmaWordIndexFile; import de.anomic.server.serverCore; import de.anomic.server.serverDate; import de.anomic.server.serverFileUtils; +import de.anomic.server.serverMemory; import de.anomic.server.serverPlainSwitch; import de.anomic.server.serverSwitch; import de.anomic.server.serverSystem; @@ -705,7 +706,7 @@ public final class yacy { plasmaCrawlLURL minimizedUrlDB = new plasmaCrawlLURL(new File(dbroot, "urlHash.temp.db"), cache, 10000); Runtime rt = Runtime.getRuntime(); - int cacheMem = (int)((rt.maxMemory()-rt.totalMemory())/1024)-(2*cache + 8*1024); + int cacheMem = (int)((serverMemory.max-rt.totalMemory())/1024)-(2*cache + 8*1024); if (cacheMem < 2048) throw new OutOfMemoryError("Not enough memory available to start clean up."); plasmaWordIndex wordIndex = new plasmaWordIndex(dbroot, indexRoot, cacheMem, 10000, log, sps.getConfigBool("useCollectionIndex", false)); @@ -1256,15 +1257,10 @@ public final class yacy { // check memory amount System.gc(); - long startupMemFree = Runtime.getRuntime().freeMemory(); // the - // amount of - // free - // memory in - // the Java - // Virtual - // Machine + long startupMemFree = Runtime.getRuntime().freeMemory(); // the amount of free memory in the Java Virtual Machine long startupMemTotal = Runtime.getRuntime().totalMemory(); // the total amount of memory in the Java virtual machine; may vary over time - + serverMemory.available(); // force initialization of class serverMemory + // go into headless awt mode System.setProperty("java.awt.headless", "true"); //which XML Parser?