because of a bug in the JRE 1.4.2 there was no memory protection

see http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4686462
this commit fixes the bug by using a memory-computation patch.
All uses of Runtime.maxMemory had been replaced by serverMemory.max
The bug is not present any more in Java 1.5

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2419 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/PerformanceMemory_p.java

@@ -50,6 +50,7 @@ import java.io.File;
 import de.anomic.http.httpc;
 import de.anomic.http.httpHeader;
 import de.anomic.plasma.plasmaSwitchboard;
+import de.anomic.server.serverMemory;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
 import de.anomic.server.serverFileUtils;
@@ -138,7 +139,7 @@ public class PerformanceMemory_p {
         long memoryTotalAfterInitBGC = Long.parseLong(env.getConfig("memoryTotalAfterInitBGC", "0"));
         long memoryTotalAfterInitAGC = Long.parseLong(env.getConfig("memoryTotalAfterInitAGC", "0"));
         long memoryTotalAfterStartup = Long.parseLong(env.getConfig("memoryTotalAfterStartup", "0"));
-        long memoryMax = Runtime.getRuntime().maxMemory();
+        long memoryMax = serverMemory.max;
         
         prop.put("memoryMax", memoryMax / MB);
         prop.put("memoryAvailAfterStartup", (memoryMax - memoryTotalAfterStartup + memoryFreeAfterStartup) / MB);

htroot/Status.java

@@ -56,6 +56,7 @@ import de.anomic.http.httpdByteCountOutputStream;
 import de.anomic.plasma.plasmaSwitchboard;
 import de.anomic.server.serverCore;
 import de.anomic.server.serverDate;
+import de.anomic.server.serverMemory;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
 import de.anomic.yacy.yacyCore;
@@ -270,7 +271,7 @@ public class Status {
         // memory usage and system attributes
         prop.put("freeMemory", bytesToString(rt.freeMemory()));
         prop.put("totalMemory", bytesToString(rt.totalMemory()));
-        prop.put("maxMemory", bytesToString(rt.maxMemory()));
+        prop.put("maxMemory", bytesToString(serverMemory.max));
         prop.put("processors", rt.availableProcessors());
 
         // proxy traffic

source/de/anomic/kelondro/kelondroAttrSeq.java

@@ -64,6 +64,7 @@ import java.util.logging.Logger;
 
 
 import de.anomic.server.serverFileUtils;
+import de.anomic.server.serverMemory;
 
 public class kelondroAttrSeq {
     
@@ -448,12 +449,11 @@ public class kelondroAttrSeq {
         }
     }
     
-    private static final Runtime runtime = Runtime.getRuntime();
     private static final long cc = 0;
     private static boolean shortmemstate = false;
     private static boolean shortmem() {
         if ((cc % 300) == 0) {
-            shortmemstate = (runtime.freeMemory() < 20000000L);
+            shortmemstate = (serverMemory.available() < 20000000L);
         }
         return shortmemstate;
     }

source/de/anomic/kelondro/kelondroObjectCache.java

@@ -58,6 +58,8 @@ package de.anomic.kelondro;
 
 import java.util.TreeMap;
 
+import de.anomic.server.serverMemory;
+
 public class kelondroObjectCache {
 
     private final  TreeMap cache;
@@ -288,7 +290,7 @@ public class kelondroObjectCache {
                    ((k = (String) ages.getMinObject()) != null) &&
                    ((ages.size() > maxSize) ||
                     (((System.currentTimeMillis() - longEmit(ages.getScore(k))) > maxAge) &&
-                     (Runtime.getRuntime().freeMemory() < minMem)))
+                     (serverMemory.available() < minMem)))
                   ) {
                 cache.remove(k);
                 ages.deleteScore(k);
@@ -304,7 +306,7 @@ public class kelondroObjectCache {
                     ((k = (String) hasnot.getMinObject()) != null) &&
                     ((hasnot.size() > maxSize) ||
                       (((System.currentTimeMillis() - longEmit(hasnot.getScore(k))) > maxAge) &&
-                       (Runtime.getRuntime().freeMemory() < minMem)))
+                       (serverMemory.available() < minMem)))
                    ) {
                  hasnot.deleteScore(k);
                  hasnotFlush++;

source/de/anomic/kelondro/kelondroRecords.java

@@ -78,6 +78,8 @@ import java.util.Iterator;
 import java.util.TreeSet;
 import java.util.logging.Logger;
 
+import de.anomic.server.serverMemory;
+
 public class kelondroRecords {
 
     // constants
@@ -449,14 +451,6 @@ public class kelondroRecords {
         }
     }
 
-    private static final long max = Runtime.getRuntime().maxMemory();
-    private static final Runtime runtime = Runtime.getRuntime();
-    
-    public static long availableMemory() {
-        // memory that is available including increasing total memory up to maximum
-        return max - runtime.totalMemory() + runtime.freeMemory();
-    }
-    
     public File file() {
         if (filename == null) return null;
         return new File(filename);
@@ -837,7 +831,7 @@ public class kelondroRecords {
             // returns false if the cache is considered to be full
             if (cacheSize == 0) return false; // no caching
             if (cacheHeaders.size() == 0) return true; // nothing there to flush
-            if ((cacheHeaders.size() < cacheSize) && (availableMemory() >= memBlock)) return true; // no need to flush cache space
+            if ((cacheHeaders.size() < cacheSize) && (serverMemory.available() >= memBlock)) return true; // no need to flush cache space
             
             // just delete any of the entries
             cacheHeaders.removeOne();

source/de/anomic/kelondro/kelondroRowBufferedSet.java

@@ -28,6 +28,8 @@ import java.util.Map;
 import java.util.Iterator;
 import java.util.TreeMap;
 
+import de.anomic.server.serverMemory;
+
 public class kelondroRowBufferedSet extends kelondroRowSet {
 
     private static final long memBlockLimit = 2000000;      // do not fill cache further if the amount of available memory is less that this
@@ -136,7 +138,7 @@ public class kelondroRowBufferedSet extends kelondroRowSet {
                     if (oldentry == null) {
                         // this was not anywhere
                         buffer.put(key, newentry);
-                        if (((buffer.size() > bufferFlushMinimum) &&  (kelondroRecords.availableMemory() > memBlockLimit)) ||
+                        if (((buffer.size() > bufferFlushMinimum) &&  (serverMemory.available() > memBlockLimit)) ||
                             (buffer.size() > bufferFlushLimit)) flush();
                     } else {
                         // replace old entry

source/de/anomic/plasma/plasmaRankingCRProcess.java

@@ -59,6 +59,7 @@ import de.anomic.kelondro.kelondroRow;
 import de.anomic.kelondro.kelondroRowSet;
 import de.anomic.server.serverFileUtils;
 import de.anomic.server.serverDate;
+import de.anomic.server.serverMemory;
 import de.anomic.tools.bitfield;
 
 public class plasmaRankingCRProcess {
@@ -376,7 +377,7 @@ public class plasmaRankingCRProcess {
             count++;
             if ((count % 1000) == 0) {
                 l = java.lang.Math.max(1, (System.currentTimeMillis() - start) / 1000);
-                System.out.println("processed " + count + " citations, " + (count / l) + " per second, rci.size = " + rci.size() + ", " + ((size - count) / (count / l)) + " seconds remaining; mem = " + Runtime.getRuntime().freeMemory());  
+                System.out.println("processed " + count + " citations, " + (count / l) + " per second, rci.size = " + rci.size() + ", " + ((size - count) / (count / l)) + " seconds remaining; mem = " + serverMemory.available());  
             }
             i.remove();
         }

source/de/anomic/server/serverAbstractThread.java

@@ -245,7 +245,7 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
         long timestamp;
         long memstamp0, memstamp1;
         boolean isBusy;
-        Runtime rt = Runtime.getRuntime();
+        //Runtime rt = Runtime.getRuntime();
                 
         while (running) {
             if ((this.intermissionObedient) && (this.intermission > 0) && (this.intermission != Long.MAX_VALUE)) {
@@ -266,7 +266,7 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
                 timestamp = System.currentTimeMillis();
                 ratz(this.idlePause);
                 idletime += System.currentTimeMillis() - timestamp;
-            } else if (rt.freeMemory() > memprereq) try {
+            } else if (serverMemory.available() > memprereq) try {
                 // do job
                 timestamp = System.currentTimeMillis();
                 memstamp0 = serverMemory.used();
@@ -305,7 +305,6 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
                 ratz(this.idlePause);
                 idletime += System.currentTimeMillis() - timestamp;
                 outofmemoryCycles++;
-                if (rt.freeMemory() <= memprereq) System.gc(); // give next loop a chance
             }
         }
         this.close();

source/de/anomic/server/serverMemory.java

@@ -49,7 +49,8 @@ import java.text.DecimalFormat;
 
 public class serverMemory {
 
-    public static final long max = Runtime.getRuntime().maxMemory();
+    public static boolean vm15 = System.getProperty("java.vm.version").startsWith("1.5");
+    public static final long max = (vm15) ? Runtime.getRuntime().maxMemory() : computedMaxMemory(); // patch for maxMemory bug in Java 1.4.2
     private static final Runtime runtime = Runtime.getRuntime();
     
     public static long free() {
@@ -91,4 +92,43 @@ public class serverMemory {
             return "unknown";
         }
     }
+    
+    private static int computedMaxMemory() {
+        // there is a bug in java 1.4.2 for maxMemory()
+        // see for bug description:
+        // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4686462
+        // to get the correct maxMemory, we force a OutOfMemoryError here to measure the 'real' maxMemory()
+        int mb = 1024 * 1024;
+        byte[][] x = new byte[2048][];
+        for (int i = 0; i < x.length; i++) {
+            try {
+                x[i] = new byte[mb];
+            } catch (OutOfMemoryError e) {
+                x = null; // free memory
+                //System.out.println("* computed maxMemory = " + i + " mb");
+                return (int) Math.max(i * mb, Runtime.getRuntime().totalMemory());
+            }
+        }
+        return 2048 * mb;
+    }
+    
+    public static void main(String[] args) {
+        // try this with a jvm 1.4.2 and with a jvm 1.5 and compare results
+        int mb = 1024 * 1024;
+        System.out.println("vm: " + System.getProperty("java.vm.version"));
+        System.out.println("computed max = " + (computedMaxMemory() / mb) + " mb");
+        int alloc = 10000;
+        Runtime rt = Runtime.getRuntime();
+        byte[][] x = new byte[100000][];
+        for (int i = 0; i < 100000; i++) {
+            x[i] = new byte[alloc];
+            if (i % 100 == 0) System.out.println("used = " + (i * alloc / mb) +
+                    ", total = " + (rt.totalMemory() / mb) +
+                    ", free = " + (rt.freeMemory() / mb) +
+                    ", max = " + (rt.maxMemory() / mb) +
+                    ", avail = " + ((rt.maxMemory() - rt.totalMemory() + rt.freeMemory()) / mb));
+        }
+
+    }
+    
 }

source/yacy.java

@@ -90,6 +90,7 @@ import de.anomic.plasma.plasmaWordIndexFile;
 import de.anomic.server.serverCore;
 import de.anomic.server.serverDate;
 import de.anomic.server.serverFileUtils;
+import de.anomic.server.serverMemory;
 import de.anomic.server.serverPlainSwitch;
 import de.anomic.server.serverSwitch;
 import de.anomic.server.serverSystem;
@@ -705,7 +706,7 @@ public final class yacy {
             plasmaCrawlLURL minimizedUrlDB = new plasmaCrawlLURL(new File(dbroot, "urlHash.temp.db"), cache, 10000);
             
             Runtime rt = Runtime.getRuntime();
-            int cacheMem = (int)((rt.maxMemory()-rt.totalMemory())/1024)-(2*cache + 8*1024);
+            int cacheMem = (int)((serverMemory.max-rt.totalMemory())/1024)-(2*cache + 8*1024);
             if (cacheMem < 2048) throw new OutOfMemoryError("Not enough memory available to start clean up.");
                 
             plasmaWordIndex wordIndex = new plasmaWordIndex(dbroot, indexRoot, cacheMem, 10000, log, sps.getConfigBool("useCollectionIndex", false));
@@ -1256,14 +1257,9 @@ public final class yacy {
 
         // check memory amount
         System.gc();
-        long startupMemFree  = Runtime.getRuntime().freeMemory(); // the
+        long startupMemFree  = Runtime.getRuntime().freeMemory(); // the amount of free memory in the Java Virtual Machine
-                                                                    // amount of
-                                                                    // free
-                                                                    // memory in
-                                                                    // the Java
-                                                                    // Virtual
-                                                                    // Machine
         long startupMemTotal = Runtime.getRuntime().totalMemory(); // the total amount of memory in the Java virtual machine; may vary over time
+        serverMemory.available(); // force initialization of class serverMemory
         
         // go into headless awt mode
         System.setProperty("java.awt.headless", "true");