consistent use of de.anomic.server.serverMemory to get information about memory statistics

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4522 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
danielr 17 years ago
parent ce8adf9907
commit fbe335db73

@ -87,15 +87,15 @@ public class PerformanceMemory_p {
}
}
long memoryFreeNow = Runtime.getRuntime().freeMemory();
long memoryFreeNow = serverMemory.free();
long memoryFreeAfterInitBGC = Long.parseLong(env.getConfig("memoryFreeAfterInitBGC", "0"));
long memoryFreeAfterInitAGC = Long.parseLong(env.getConfig("memoryFreeAfterInitAGC", "0"));
long memoryFreeAfterStartup = Long.parseLong(env.getConfig("memoryFreeAfterStartup", "0"));
long memoryTotalNow = Runtime.getRuntime().totalMemory();
long memoryTotalNow = serverMemory.total();
long memoryTotalAfterInitBGC = Long.parseLong(env.getConfig("memoryTotalAfterInitBGC", "0"));
long memoryTotalAfterInitAGC = Long.parseLong(env.getConfig("memoryTotalAfterInitAGC", "0"));
long memoryTotalAfterStartup = Long.parseLong(env.getConfig("memoryTotalAfterStartup", "0"));
long memoryMax = Runtime.getRuntime().maxMemory();
long memoryMax = serverMemory.max();
prop.putNum("memoryMax", memoryMax / MB);
prop.putNum("memoryAvailAfterStartup", (memoryMax - memoryTotalAfterStartup + memoryFreeAfterStartup) / MB);

@ -288,12 +288,10 @@ public class Status {
prop.put("omode", "2");
}
final Runtime rt = Runtime.getRuntime();
// memory usage and system attributes
prop.put("freeMemory", serverMemory.bytesToString(rt.freeMemory()));
prop.put("totalMemory", serverMemory.bytesToString(rt.totalMemory()));
prop.put("maxMemory", serverMemory.bytesToString(rt.maxMemory()));
prop.put("freeMemory", serverMemory.bytesToString(serverMemory.free()));
prop.put("totalMemory", serverMemory.bytesToString(serverMemory.total()));
prop.put("maxMemory", serverMemory.bytesToString(serverMemory.max()));
prop.put("processors", serverProcessor.availableCPU);
// proxy traffic

@ -43,6 +43,7 @@ import de.anomic.http.httpHeader;
import de.anomic.http.httpdByteCountInputStream;
import de.anomic.http.httpdByteCountOutputStream;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverMemory;
import de.anomic.server.serverObjects;
import de.anomic.server.serverProcessor;
import de.anomic.server.serverSwitch;
@ -72,10 +73,9 @@ public class status_p {
//
// memory usage and system attributes
final Runtime rt = Runtime.getRuntime();
prop.putNum("freeMemory", rt.freeMemory());
prop.putNum("totalMemory", rt.totalMemory());
prop.putNum("maxMemory", rt.maxMemory());
prop.putNum("freeMemory", serverMemory.free());
prop.putNum("totalMemory", serverMemory.total());
prop.putNum("maxMemory", serverMemory.max());
prop.putNum("processors", serverProcessor.availableCPU);
// proxy traffic

@ -270,8 +270,7 @@ public class dbtest {
boolean assertionenabled = false;
assert assertionenabled = true;
if (assertionenabled) System.out.println("*** Asserts are enabled"); else System.out.println("*** HINT: YOU SHOULD ENABLE ASSERTS! (include -ea in start arguments");
Runtime runtime = Runtime.getRuntime();
long mb = (runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory()) / 1024 / 1024;
long mb = serverMemory.available() / 1024 / 1024;
System.out.println("*** RAM = " + mb + " MB");
System.out.print(">java " +
((assertionenabled) ? "-ea " : "") +
@ -398,14 +397,14 @@ public class dbtest {
System.out.println("Loop " + loop + ": Write = " + write + ", Remove = " + remove);
System.out.println(" bevore GC: " +
"free = " + Runtime.getRuntime().freeMemory() +
", max = " + Runtime.getRuntime().maxMemory() +
", total = " + Runtime.getRuntime().totalMemory());
"free = " + serverMemory.free() +
", max = " + serverMemory.max() +
", total = " + serverMemory.total());
System.gc();
System.out.println(" after GC: " +
"free = " + Runtime.getRuntime().freeMemory() +
", max = " + Runtime.getRuntime().maxMemory() +
", total = " + Runtime.getRuntime().totalMemory());
"free = " + serverMemory.free() +
", max = " + serverMemory.max() +
", total = " + serverMemory.total());
loop++;
}
}

@ -169,7 +169,7 @@ public final class indexRAMRI implements indexRI {
+ " words done, "
+ (cache.size() / (wordsPerSecond + 1))
+ " seconds remaining, free mem = "
+ (Runtime.getRuntime().freeMemory() / 1024 / 1024)
+ (serverMemory.free() / 1024 / 1024)
+ "MB");
messageTime = System.currentTimeMillis() + 5000;
}
@ -209,7 +209,6 @@ public final class indexRAMRI implements indexRI {
//long creationTime;
indexRWIRowEntry wordEntry;
kelondroRow.EntryIndex row;
//Runtime rt = Runtime.getRuntime();
while (i.hasNext()) {
// get out one entry
row = i.next();
@ -221,12 +220,12 @@ public final class indexRAMRI implements indexRI {
addEntry(wordHash, wordEntry, startTime, false);
urlCount++;
// protect against memory shortage
//while (rt.freeMemory() < 1000000) {flushFromMem(); java.lang.System.gc();}
//while (serverMemory.free() < 1000000) {flushFromMem(); java.lang.System.gc();}
// write a log
if (System.currentTimeMillis() > messageTime) {
serverMemory.gc(1000, "indexRAMRI, for better statistic-2"); // for better statistic - thq
urlsPerSecond = 1 + urlCount * 1000 / (1 + System.currentTimeMillis() - startTime);
log.logInfo("restoring status: " + urlCount + " urls done, " + ((dumpArray.size() - urlCount) / urlsPerSecond) + " seconds remaining, free mem = " + (Runtime.getRuntime().freeMemory() / 1024 / 1024) + "MB");
log.logInfo("restoring status: " + urlCount + " urls done, " + ((dumpArray.size() - urlCount) / urlsPerSecond) + " seconds remaining, free mem = " + (serverMemory.free() / 1024 / 1024) + "MB");
messageTime = System.currentTimeMillis() + 5000;
}
}
@ -363,7 +362,7 @@ public final class indexRAMRI implements indexRI {
return hash;
}
// cases with respect to memory situation
if (Runtime.getRuntime().freeMemory() < 100000) {
if (serverMemory.free() < 100000) {
// urgent low-memory case
hash = (String) hashScore.getMaxObject(); // flush high-score entries (saves RAM)
} else {

@ -100,7 +100,7 @@ public class kelondroEcoTable implements kelondroIndex {
long neededRAM4table = ((long) records) * (((long) rowdef.objectsize) + 4L) * 3L;
table = ((neededRAM4table < maxarraylength) &&
((useTailCache == tailCacheForceUsage) ||
((useTailCache == tailCacheUsageAuto) && (Runtime.getRuntime().freeMemory() > neededRAM4table + 200 * 1024 * 1024)))) ?
((useTailCache == tailCacheUsageAuto) && (serverMemory.free() > neededRAM4table + 200 * 1024 * 1024)))) ?
new kelondroRowSet(taildef, records) : null;
System.out.println("*** DEBUG " + tablefile + ": available RAM: " + (serverMemory.available() / 1024 / 1024) + "MB, allocating space for " + records + " entries");
long neededRAM4index = 2 * 1024 * 1024 + records * (rowdef.primaryKeyLength + 4) * 3 / 2;

@ -428,7 +428,7 @@ public class plasmaRankingCRProcess {
count++;
if ((count % 1000) == 0) {
l = java.lang.Math.max(1, (System.currentTimeMillis() - start) / 1000);
System.out.println("processed " + count + " citations, " + (count / l) + " per second, rci.size = " + rci.size() + ", " + ((size - count) / (count / l) / 60) + " minutes remaining; mem = " + Runtime.getRuntime().freeMemory());
System.out.println("processed " + count + " citations, " + (count / l) + " per second, rci.size = " + rci.size() + ", " + ((size - count) / (count / l) / 60) + " minutes remaining; mem = " + serverMemory.free());
}
}

@ -28,6 +28,9 @@ package de.anomic.server;
import de.anomic.server.logging.serverLog;
import de.anomic.tools.yFormatter;
/**
* Use this to get information about memory usage or try to free some memory
*/
public class serverMemory {
private static final Runtime runtime = Runtime.getRuntime();
@ -38,6 +41,11 @@ public class serverMemory {
private static long lastGC;
/**
* Runs the garbage collector if last garbage collection is more than last millis ago
* @param last time which must be passed since lased gc
* @param info additional info for log
*/
public final synchronized static void gc(int last, String info) { // thq
long elapsed = System.currentTimeMillis() - lastGC;
if (elapsed > last) {
@ -50,11 +58,15 @@ public class serverMemory {
}
}
/** @return the amount of freed bytes by a forced GC this method performes */
/**
* Tries to free count bytes
* @param count bytes
* @return the amount of freed bytes by a forced GC this method performes
*/
private static long runGC(final boolean count) {
final long memnow = runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory();
final long memBefore = available();
gc(1000, "serverMemory.runGC(...)");
final long freed = runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory() - memnow;
final long freed = available() - memBefore;
if (count) {
gcs[gcs_pos] = freed;
gcs_pos = (gcs_pos + 1) % gcs.length;
@ -78,29 +90,41 @@ public class serverMemory {
return (y == 0) ? 0 : x / y;
}
public static long free() {
// memory that is free without increasing of total memory taken from os
/**
* memory that is free without increasing of total memory taken from os
* @return bytes
*/
public static final long free() {
return runtime.freeMemory();
}
/**
* Tries to free a specified amount of bytes. If the currently available memory is enough, the
* method returns <code>true</code> without performing additional steps. Otherwise - if
* <code>gciffail</code> is set - a Full GC is run, if <code>gciffail</code> is set to
* <code>false</code> and not enough memory is available, this method returns <code>false</code>.
*
* @see serverMemory#request(long, boolean) for another implementation
* @param memory amount of bytes to be assured to be free
* @param gciffail if not enough memory is available, this parameter specifies whether to perform
* a Full GC to free enough RAM
* @return whether enough RAM is available
* memory that is available including increasing total memory up to maximum
* @return bytes
*/
public static final long available() {
// memory that is available including increasing total memory up to maximum
return runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory();
return max() - total() + free();
}
/**
* maximum memory which the vm can use
* @return bytes
*/
public static final long max()
{
return runtime.maxMemory();
}
/**
* currently allocated memory in the Java virtual machine; may vary over time
* @return bytes
*/
public static final long total()
{
return runtime.totalMemory();
}
/**
* <p>Tries to free a specified amount of bytes.</p>
* <p>
* If the currently available memory is enough, the method returns <code>true</code> without
@ -120,7 +144,7 @@ public class serverMemory {
* @return whether enough memory could be freed (or is free) or not
*/
public static boolean request(final long size, final boolean force) {
long avail = runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory();
long avail = available();
if (avail >= size) return true;
if (log.isFine()) {
String t = new Throwable("Stack trace").getStackTrace()[1].toString();
@ -130,7 +154,7 @@ public class serverMemory {
if (force || avg == 0 || avg + avail >= size) {
// this is only called if we expect that an allocation of <size> bytes would cause the jvm to call the GC anyway
final long freed = runGC(!force);
avail = runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory();
avail = available();
log.logInfo("performed " + ((force) ? "explicit" : "necessary") + " GC, freed " + (freed >> 10)
+ " KB (requested/available/average: "
+ (size >> 10) + " / " + (avail >> 10) + " / " + (avg >> 10) + " KB)");
@ -142,11 +166,19 @@ public class serverMemory {
}
}
/**
* memory that is currently bound in objects
* @return used bytes
*/
public static long used() {
// memory that is currently bound in objects
return runtime.totalMemory() - runtime.freeMemory();
return total() - free();
}
/**
* Formats a number if it are bytes to greatest unit (1024 based)
* @param byteCount
* @return formatted String with unit
*/
public static String bytesToString(long byteCount) {
try {
final StringBuffer byteString = new StringBuffer();
@ -171,21 +203,24 @@ public class serverMemory {
}
}
/**
* main
* @param args
*/
public static void main(String[] args) {
// try this with a jvm 1.4.2 and with a jvm 1.5 and compare results
int mb = 1024 * 1024;
System.out.println("vm: " + System.getProperty("java.vm.version"));
System.out.println("computed max = " + (Runtime.getRuntime().maxMemory() / mb) + " mb");
System.out.println("computed max = " + (max() / mb) + " mb");
int alloc = 10000;
Runtime rt = Runtime.getRuntime();
byte[][] x = new byte[100000][];
for (int i = 0; i < 100000; i++) {
x[i] = new byte[alloc];
if (i % 100 == 0) System.out.println("used = " + (i * alloc / mb) +
", total = " + (rt.totalMemory() / mb) +
", free = " + (rt.freeMemory() / mb) +
", max = " + (rt.maxMemory() / mb) +
", avail = " + ((rt.maxMemory() - rt.totalMemory() + rt.freeMemory()) / mb));
", total = " + (total() / mb) +
", free = " + (free() / mb) +
", max = " + (max() / mb) +
", avail = " + (available() / mb));
}
}

@ -83,6 +83,7 @@ import de.anomic.plasma.plasmaWordIndex;
import de.anomic.server.serverCore;
import de.anomic.server.serverDate;
import de.anomic.server.serverFileUtils;
import de.anomic.server.serverMemory;
import de.anomic.server.serverSemaphore;
import de.anomic.server.serverSystem;
import de.anomic.server.logging.serverLog;
@ -392,11 +393,11 @@ public final class yacy {
// save information about available memory after all initializations
//try {
sb.setConfig("memoryFreeAfterInitBGC", Runtime.getRuntime().freeMemory());
sb.setConfig("memoryTotalAfterInitBGC", Runtime.getRuntime().totalMemory());
sb.setConfig("memoryFreeAfterInitBGC", serverMemory.free());
sb.setConfig("memoryTotalAfterInitBGC", serverMemory.total());
System.gc();
sb.setConfig("memoryFreeAfterInitAGC", Runtime.getRuntime().freeMemory());
sb.setConfig("memoryTotalAfterInitAGC", Runtime.getRuntime().totalMemory());
sb.setConfig("memoryFreeAfterInitAGC", serverMemory.free());
sb.setConfig("memoryTotalAfterInitAGC", serverMemory.total());
//} catch (ConcurrentModificationException e) {}
// signal finished startup
@ -605,8 +606,7 @@ public final class yacy {
// db used to hold all neede urls
plasmaCrawlLURL minimizedUrlDB = new plasmaCrawlLURL(indexRoot2);
Runtime rt = Runtime.getRuntime();
int cacheMem = (int)(rt.maxMemory() - rt.totalMemory());
int cacheMem = (int)(serverMemory.max() - serverMemory.total());
if (cacheMem < 2048000) throw new OutOfMemoryError("Not enough memory available to start clean up.");
plasmaWordIndex wordIndex = new plasmaWordIndex(indexPrimaryRoot, indexSecondaryRoot, log);
@ -645,8 +645,8 @@ public final class yacy {
log.logInfo(wordCounter + " words scanned " +
"[" + wordChunkStartHash + " .. " + wordChunkEndHash + "]\n" +
"Duration: "+ 500*1000/duration + " words/s" +
" | Free memory: " + rt.freeMemory() +
" | Total memory: " + rt.totalMemory());
" | Free memory: " + serverMemory.free() +
" | Total memory: " + serverMemory.total());
wordChunkStart = wordChunkEnd;
wordChunkStartHash = wordChunkEndHash;
}
@ -901,8 +901,8 @@ public final class yacy {
// check memory amount
System.gc();
long startupMemFree = Runtime.getRuntime().freeMemory(); // the amount of free memory in the Java Virtual Machine
long startupMemTotal = Runtime.getRuntime().totalMemory(); // the total amount of memory in the Java virtual machine; may vary over time
long startupMemFree = serverMemory.free();
long startupMemTotal = serverMemory.total();
// go into headless awt mode
System.setProperty("java.awt.headless", "true");

Loading…
Cancel
Save