boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

because of a bug in the JRE 1.4.2 there was no memory protection

Browse Source 
see http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4686462
this commit fixes the bug by using a memory-computation patch.
All uses of Runtime.maxMemory had been replaced by serverMemory.max
The bug is not present any more in Java 1.5

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2419 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent 4eca0f8830
commit 0187c60010
10 changed files with 67 additions and 31 deletions
Show Stats Download Patch File Download Diff File

3
htroot/PerformanceMemory_p.java
Unescape View File

@ -50,6 +50,7 @@ import java.io.File;
import de.anomic.http.httpc;
import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverMemory;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.server.serverFileUtils;
@ -138,7 +139,7 @@ public class PerformanceMemory_p {
long memoryTotalAfterInitBGC = Long.parseLong(env.getConfig("memoryTotalAfterInitBGC", "0"));
long memoryTotalAfterInitAGC = Long.parseLong(env.getConfig("memoryTotalAfterInitAGC", "0"));
long memoryTotalAfterStartup = Long.parseLong(env.getConfig("memoryTotalAfterStartup", "0"));
long memoryMax = Runtime.getRuntime().maxMemory();
long memoryMax = serverMemory.max;
prop.put("memoryMax", memoryMax / MB);
prop.put("memoryAvailAfterStartup", (memoryMax - memoryTotalAfterStartup + memoryFreeAfterStartup) / MB);

3
htroot/Status.java
Unescape View File

@ -56,6 +56,7 @@ import de.anomic.http.httpdByteCountOutputStream;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore;
import de.anomic.server.serverDate;
import de.anomic.server.serverMemory;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
@ -270,7 +271,7 @@ public class Status {
// memory usage and system attributes
prop.put("freeMemory", bytesToString(rt.freeMemory()));
prop.put("totalMemory", bytesToString(rt.totalMemory()));
prop.put("maxMemory", bytesToString(rt.maxMemory()));
prop.put("maxMemory", bytesToString(serverMemory.max));
prop.put("processors", rt.availableProcessors());
// proxy traffic

4
source/de/anomic/kelondro/kelondroAttrSeq.java
Unescape View File

@ -64,6 +64,7 @@ import java.util.logging.Logger;
import de.anomic.server.serverFileUtils;
import de.anomic.server.serverMemory;
public class kelondroAttrSeq {
@ -448,12 +449,11 @@ public class kelondroAttrSeq {
}
}
private static final Runtime runtime = Runtime.getRuntime();
private static final long cc = 0;
private static boolean shortmemstate = false;
private static boolean shortmem() {
if ((cc % 300) == 0) {
shortmemstate = (runtime.freeMemory() < 20000000L);
shortmemstate = (serverMemory.available() < 20000000L);
}
return shortmemstate;
}

6
source/de/anomic/kelondro/kelondroObjectCache.java
Unescape View File

@ -58,6 +58,8 @@ package de.anomic.kelondro;
import java.util.TreeMap;
import de.anomic.server.serverMemory;
public class kelondroObjectCache {
private final TreeMap cache;
@ -288,7 +290,7 @@ public class kelondroObjectCache {
((k = (String) ages.getMinObject()) != null) &&
((ages.size() > maxSize) ||
(((System.currentTimeMillis() - longEmit(ages.getScore(k))) > maxAge) &&
(Runtime.getRuntime().freeMemory() < minMem)))
(serverMemory.available() < minMem)))
) {
cache.remove(k);
ages.deleteScore(k);
@ -304,7 +306,7 @@ public class kelondroObjectCache {
((k = (String) hasnot.getMinObject()) != null) &&
((hasnot.size() > maxSize) ||
(((System.currentTimeMillis() - longEmit(hasnot.getScore(k))) > maxAge) &&
(Runtime.getRuntime().freeMemory() < minMem)))
(serverMemory.available() < minMem)))
) {
hasnot.deleteScore(k);
hasnotFlush++;

12
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -78,6 +78,8 @@ import java.util.Iterator;
import java.util.TreeSet;
import java.util.logging.Logger;
import de.anomic.server.serverMemory;
public class kelondroRecords {
// constants
@ -449,14 +451,6 @@ public class kelondroRecords {
}
}
private static final long max = Runtime.getRuntime().maxMemory();
private static final Runtime runtime = Runtime.getRuntime();
public static long availableMemory() {
// memory that is available including increasing total memory up to maximum
return max - runtime.totalMemory() + runtime.freeMemory();
}
public File file() {
if (filename == null) return null;
return new File(filename);
@ -837,7 +831,7 @@ public class kelondroRecords {
// returns false if the cache is considered to be full
if (cacheSize == 0) return false; // no caching
if (cacheHeaders.size() == 0) return true; // nothing there to flush
if ((cacheHeaders.size() < cacheSize) && (availableMemory() >= memBlock)) return true; // no need to flush cache space
if ((cacheHeaders.size() < cacheSize) && (serverMemory.available() >= memBlock)) return true; // no need to flush cache space
// just delete any of the entries
cacheHeaders.removeOne();

4
source/de/anomic/kelondro/kelondroRowBufferedSet.java
Unescape View File

@ -28,6 +28,8 @@ import java.util.Map;
import java.util.Iterator;
import java.util.TreeMap;
import de.anomic.server.serverMemory;
public class kelondroRowBufferedSet extends kelondroRowSet {
private static final long memBlockLimit = 2000000; // do not fill cache further if the amount of available memory is less that this
@ -136,7 +138,7 @@ public class kelondroRowBufferedSet extends kelondroRowSet {
if (oldentry == null) {
// this was not anywhere
buffer.put(key, newentry);
if (((buffer.size() > bufferFlushMinimum) && (kelondroRecords.availableMemory() > memBlockLimit)) ||
if (((buffer.size() > bufferFlushMinimum) && (serverMemory.available() > memBlockLimit)) ||
(buffer.size() > bufferFlushLimit)) flush();
} else {
// replace old entry

3
source/de/anomic/plasma/plasmaRankingCRProcess.java
Unescape View File

@ -59,6 +59,7 @@ import de.anomic.kelondro.kelondroRow;
import de.anomic.kelondro.kelondroRowSet;
import de.anomic.server.serverFileUtils;
import de.anomic.server.serverDate;
import de.anomic.server.serverMemory;
import de.anomic.tools.bitfield;
public class plasmaRankingCRProcess {
@ -376,7 +377,7 @@ public class plasmaRankingCRProcess {
count++;
if ((count % 1000) == 0) {
l = java.lang.Math.max(1, (System.currentTimeMillis() - start) / 1000);
System.out.println("processed " + count + " citations, " + (count / l) + " per second, rci.size = " + rci.size() + ", " + ((size - count) / (count / l)) + " seconds remaining; mem = " + Runtime.getRuntime().freeMemory());
System.out.println("processed " + count + " citations, " + (count / l) + " per second, rci.size = " + rci.size() + ", " + ((size - count) / (count / l)) + " seconds remaining; mem = " + serverMemory.available());
}
i.remove();
}

5
source/de/anomic/server/serverAbstractThread.java
Unescape View File

@ -245,7 +245,7 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
long timestamp;
long memstamp0, memstamp1;
boolean isBusy;
Runtime rt = Runtime.getRuntime();
//Runtime rt = Runtime.getRuntime();
while (running) {
if ((this.intermissionObedient) && (this.intermission > 0) && (this.intermission != Long.MAX_VALUE)) {
@ -266,7 +266,7 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
timestamp = System.currentTimeMillis();
ratz(this.idlePause);
idletime += System.currentTimeMillis() - timestamp;
} else if (rt.freeMemory() > memprereq) try {
} else if (serverMemory.available() > memprereq) try {
// do job
timestamp = System.currentTimeMillis();
memstamp0 = serverMemory.used();
@ -305,7 +305,6 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
ratz(this.idlePause);
idletime += System.currentTimeMillis() - timestamp;
outofmemoryCycles++;
if (rt.freeMemory() <= memprereq) System.gc(); // give next loop a chance
}
}
this.close();

44
source/de/anomic/server/serverMemory.java
Unescape View File

@ -49,7 +49,8 @@ import java.text.DecimalFormat;
public class serverMemory {
public static final long max = Runtime.getRuntime().maxMemory();
public static boolean vm15 = System.getProperty("java.vm.version").startsWith("1.5");
public static final long max = (vm15) ? Runtime.getRuntime().maxMemory() : computedMaxMemory(); // patch for maxMemory bug in Java 1.4.2
private static final Runtime runtime = Runtime.getRuntime();
public static long free() {
@ -90,5 +91,44 @@ public class serverMemory {
} catch (Exception e) {
return "unknown";
}
}
}
private static int computedMaxMemory() {
// there is a bug in java 1.4.2 for maxMemory()
// see for bug description:
// http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4686462
// to get the correct maxMemory, we force a OutOfMemoryError here to measure the 'real' maxMemory()
int mb = 1024 * 1024;
byte[][] x = new byte[2048][];
for (int i = 0; i < x.length; i++) {
try {
x[i] = new byte[mb];
} catch (OutOfMemoryError e) {
x = null; // free memory
//System.out.println("* computed maxMemory = " + i + " mb");
return (int) Math.max(i * mb, Runtime.getRuntime().totalMemory());
}
}
return 2048 * mb;
}
public static void main(String[] args) {
// try this with a jvm 1.4.2 and with a jvm 1.5 and compare results
int mb = 1024 * 1024;
System.out.println("vm: " + System.getProperty("java.vm.version"));
System.out.println("computed max = " + (computedMaxMemory() / mb) + " mb");
int alloc = 10000;
Runtime rt = Runtime.getRuntime();
byte[][] x = new byte[100000][];
for (int i = 0; i < 100000; i++) {
x[i] = new byte[alloc];
if (i % 100 == 0) System.out.println("used = " + (i * alloc / mb) +
", total = " + (rt.totalMemory() / mb) +
", free = " + (rt.freeMemory() / mb) +
", max = " + (rt.maxMemory() / mb) +
", avail = " + ((rt.maxMemory() - rt.totalMemory() + rt.freeMemory()) / mb));
}
}
}

14
source/yacy.java
Unescape View File

@ -90,6 +90,7 @@ import de.anomic.plasma.plasmaWordIndexFile;
import de.anomic.server.serverCore;
import de.anomic.server.serverDate;
import de.anomic.server.serverFileUtils;
import de.anomic.server.serverMemory;
import de.anomic.server.serverPlainSwitch;
import de.anomic.server.serverSwitch;
import de.anomic.server.serverSystem;
@ -705,7 +706,7 @@ public final class yacy {
plasmaCrawlLURL minimizedUrlDB = new plasmaCrawlLURL(new File(dbroot, "urlHash.temp.db"), cache, 10000);
Runtime rt = Runtime.getRuntime();
int cacheMem = (int)((rt.maxMemory()-rt.totalMemory())/1024)-(2*cache + 8*1024);
int cacheMem = (int)((serverMemory.max-rt.totalMemory())/1024)-(2*cache + 8*1024);
if (cacheMem < 2048) throw new OutOfMemoryError("Not enough memory available to start clean up.");
plasmaWordIndex wordIndex = new plasmaWordIndex(dbroot, indexRoot, cacheMem, 10000, log, sps.getConfigBool("useCollectionIndex", false));
@ -1256,15 +1257,10 @@ public final class yacy {
// check memory amount
System.gc();
long startupMemFree = Runtime.getRuntime().freeMemory(); // the
// amount of
// free
// memory in
// the Java
// Virtual
// Machine
long startupMemFree = Runtime.getRuntime().freeMemory(); // the amount of free memory in the Java Virtual Machine
long startupMemTotal = Runtime.getRuntime().totalMemory(); // the total amount of memory in the Java virtual machine; may vary over time
serverMemory.available(); // force initialization of class serverMemory
// go into headless awt mode
System.setProperty("java.awt.headless", "true");
//which XML Parser?

Write Preview
Loading…
Cancel
Save