You can toggle between previous (standard) and new (generation) strategy at PerformanceMemory_p.html. The generation memory strategy is implemented with the objective of running more robust but with the cost of early stopping some tasks (eg. dht) while running low on memory. This new strategy does respect the generational way a heap is organized on most used jvms. These changes run fine on my 3 peers for weeks now, but as I'm human, I may fail. Please be carefull using generation memory strategy and report errors by naming OS, jvm and java_args. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7886 6c8d7289-2bf4-0310-a012-ef5d649a1542pull/1/head
sixcooler
14 years ago
parent
63a375b801
commit
4fec99115b
@ -0,0 +1,206 @@
|
||||
// MemoryControl.java
|
||||
// -------------------------------------------
|
||||
// (C) 2011 by Sebastian Gaebel
|
||||
// first published 22.08.2011 on http://yacy.net
|
||||
//
|
||||
// $LastChangedDate: 2011-08-18 00:24:17 +0200 (Do, 18. Aug 2011) $
|
||||
// $LastChangedRevision: 7883 $
|
||||
// $LastChangedBy: orbiter $
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package net.yacy.kelondro.util;
|
||||
|
||||
import java.lang.management.ManagementFactory;
|
||||
import java.lang.management.MemoryMXBean;
|
||||
import java.lang.management.MemoryPoolMXBean;
|
||||
import java.lang.management.MemoryUsage;
|
||||
|
||||
public class GenerationMemoryStrategy extends MemoryStrategy {
|
||||
|
||||
private MemoryPoolMXBean eden, survivor, old;
|
||||
private MemoryMXBean heap;
|
||||
|
||||
public GenerationMemoryStrategy() {
|
||||
name = "Generation Memory Strategy";
|
||||
error = initPoolBeans();
|
||||
heap = ManagementFactory.getMemoryMXBean();
|
||||
if (lastGC == 0l) gc(10000, "initial gc - to get proper results"); // this is necessary on some GCs / vm
|
||||
if (error) log.logWarning(name + ": not a generational heap");
|
||||
}
|
||||
|
||||
/**
|
||||
* memory that is free without increasing of total memory taken from os
|
||||
* @return bytes
|
||||
*/
|
||||
protected final long free() {
|
||||
return youngUsage(false).getCommitted() - youngUsage(false).getUsed();
|
||||
}
|
||||
|
||||
/**
|
||||
* memory that is available including increasing total memory up to maximum
|
||||
* Smallest of both old and young
|
||||
* @return bytes
|
||||
*/
|
||||
protected final long available() {
|
||||
return available(false);
|
||||
}
|
||||
|
||||
/**
|
||||
* memory that is available including increasing total memory up to maximum
|
||||
* @param force specifies whether ignoring prefered size
|
||||
* @return bytes
|
||||
*/
|
||||
private final long available(final boolean force) {
|
||||
return force & properState() ? Math.max(youngAvailable(), oldAvailable()) : Math.min(youngAvailable(), oldAvailable());
|
||||
}
|
||||
|
||||
/**
|
||||
* memory that is currently bound in objects
|
||||
* @return used bytes
|
||||
*/
|
||||
protected final long used() {
|
||||
return heap.getHeapMemoryUsage().getUsed();
|
||||
}
|
||||
|
||||
/**
|
||||
* currently allocated memory in the Java virtual machine; may vary over time
|
||||
* @return bytes
|
||||
*/
|
||||
protected final long total() {
|
||||
return heap.getHeapMemoryUsage().getCommitted();
|
||||
}
|
||||
|
||||
/**
|
||||
* maximum memory the Java virtual will allocate machine; may vary over time in some cases
|
||||
* @return bytes
|
||||
*/
|
||||
protected final long maxMemory() {
|
||||
return heap.getHeapMemoryUsage().getMax();
|
||||
}
|
||||
|
||||
/**
|
||||
* checks if a specified amount of bytes are available
|
||||
* after the jvm recycled unused objects
|
||||
*
|
||||
* @param size the requested amount of free memory in bytes
|
||||
* @param force specifies whether ignoring prefered size
|
||||
* @return whether enough memory could be freed (or is free) or not
|
||||
*/
|
||||
protected final boolean request(final long size, final boolean force, boolean shortStatus) {
|
||||
if (size == 0l) return true; // does not make sense to check - returning true without setting shortStatus (which also doesn't make sense to me)
|
||||
final boolean unknown = size < 0l; // size < 0 indicate an unknown size - maybe from gziped streams
|
||||
final boolean r = unknown? properState() : size < available(force);
|
||||
shortStatus = !r;
|
||||
return r;
|
||||
}
|
||||
|
||||
/**
|
||||
* use this to check for temporary space
|
||||
* @return bytes available to allocate in Eden Space (Young Generation)
|
||||
*/
|
||||
protected final long youngAvailable() {
|
||||
return youngUsage(true).getCommitted() - youngUsage(true).getUsed();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return bytes available to allocate in Tenured Space (Old Generation)
|
||||
*/
|
||||
protected final long oldAvailable() {
|
||||
return oldUsage(true).getCommitted() - oldUsage(true).getUsed();
|
||||
}
|
||||
|
||||
/**
|
||||
* alive objects get 'moved' on gc from eden space to survior and from survior to old gen
|
||||
* in a worse case (all objects of survivor alive) all objects get 'moved' from suvivor to old gen
|
||||
* this method checks if there is is space left in old gen for that
|
||||
*
|
||||
* @return Memory is in proper state
|
||||
*/
|
||||
protected boolean properState() {
|
||||
return (oldUsage(true).getUsed() + survivorUsage(false).getCommitted()) < oldUsage(false).getCommitted();
|
||||
}
|
||||
|
||||
/**
|
||||
* based on my research for a proper running jvm, this gives a guidance value for the heap size
|
||||
*
|
||||
* @return bytes recommend for heap size
|
||||
*/
|
||||
protected long recommendHeapSize() {
|
||||
// the heap/old-ration is jvm-specific and can be changed by parameter - using this + 20% buffer
|
||||
final double factor = 1.2 * (double)heap.getHeapMemoryUsage().getMax() / (double)oldUsage(false).getMax();
|
||||
// current needed space in old gen
|
||||
final long neededOld = oldUsage(true).getUsed() + survivorUsage(false).getMax();
|
||||
return (long) (neededOld * factor);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param collected specifies whether trying to get the memory usage after the jvm recycled unused objects
|
||||
* @return MemoryUsage of Eden Space aka Young Gen
|
||||
*/
|
||||
private MemoryUsage youngUsage(final boolean collected) {
|
||||
if (collected) {
|
||||
final MemoryUsage usage = eden.getCollectionUsage();
|
||||
if (usage != null) return usage;
|
||||
error = true;
|
||||
log.logWarning(name + ": no young colletion usage available");
|
||||
}
|
||||
return eden.getUsage();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param collected specifies whether trying to get the memory usage after the jvm recycled unused objects
|
||||
* @return MemoryUsage of Survivor
|
||||
*/
|
||||
private MemoryUsage survivorUsage(final boolean collected) {
|
||||
if (collected) {
|
||||
final MemoryUsage usage = survivor.getCollectionUsage();
|
||||
if (usage != null) return usage;
|
||||
error = true;
|
||||
log.logWarning(name + ": no survivior colletion usage available");
|
||||
}
|
||||
return survivor.getUsage();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param collected specifies whether trying to get the memory usage after the jvm recycled unused objects
|
||||
* @return MemoryUsage of Old Gen
|
||||
*/
|
||||
private MemoryUsage oldUsage(final boolean collected) {
|
||||
if (collected) {
|
||||
final MemoryUsage usage = old.getCollectionUsage();
|
||||
if (usage != null) return usage;
|
||||
error = true;
|
||||
log.logWarning(name + ": no old colletion usage available");
|
||||
}
|
||||
return old.getUsage();
|
||||
}
|
||||
|
||||
private boolean initPoolBeans() {
|
||||
for (final MemoryPoolMXBean bean : ManagementFactory.getMemoryPoolMXBeans()) {
|
||||
if (bean.getName().startsWith("G1")) break; //this strategy will not run on G1
|
||||
if (bean.getName().contains("Eden")) {
|
||||
eden = bean;
|
||||
} else if (bean.getName().contains("Survivor")) {
|
||||
survivor = bean;
|
||||
} else if (bean.getName().contains("Old") || bean.getName().contains("Tenured")) {
|
||||
old = bean;
|
||||
}
|
||||
}
|
||||
return eden == null || survivor == null || old == null;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,138 @@
|
||||
// MemoryControl.java
|
||||
// -------------------------------------------
|
||||
// (C) 2011 by Sebastian Gaebel
|
||||
// first published 22.08.2011 on http://yacy.net
|
||||
//
|
||||
// $LastChangedDate: 2011-08-18 00:24:17 +0200 (Do, 18. Aug 2011) $
|
||||
// $LastChangedRevision: 7883 $
|
||||
// $LastChangedBy: orbiter $
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package net.yacy.kelondro.util;
|
||||
|
||||
import java.lang.management.ManagementFactory;
|
||||
|
||||
import net.yacy.kelondro.logging.Log;
|
||||
|
||||
public abstract class MemoryStrategy {
|
||||
|
||||
protected final static Log log = new Log("MEMORY");
|
||||
protected static long lastGC = 0l;
|
||||
protected static boolean error = true;
|
||||
protected static String name;
|
||||
|
||||
/**
|
||||
* @return if an error has been detected
|
||||
*/
|
||||
protected final boolean hasError() {
|
||||
return error;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return an identifying name
|
||||
*/
|
||||
protected final String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs the garbage collector if last garbage collection is more than last millis ago
|
||||
* @param last time which must be passed since lased gc
|
||||
* @param info additional info for log
|
||||
*/
|
||||
protected synchronized boolean gc(final int last, final String info) { // thq
|
||||
assert last >= 10000; // too many forced GCs will cause bad execution performance
|
||||
final long elapsed = System.currentTimeMillis() - lastGC;
|
||||
if (elapsed > last) {
|
||||
ManagementFactory.getMemoryMXBean().gc();
|
||||
lastGC = System.currentTimeMillis();
|
||||
return true;
|
||||
}
|
||||
|
||||
if (log.isFinest()) log.logFinest("[gc] no execute, last run: " + (elapsed / 1000) + " seconds ago, call: " + info);
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* memory that is free without increasing of total memory taken from os
|
||||
* @return bytes
|
||||
*/
|
||||
protected abstract long free();
|
||||
|
||||
/**
|
||||
* memory that is available including increasing total memory up to maximum
|
||||
* @return bytes
|
||||
*/
|
||||
protected abstract long available();
|
||||
|
||||
/**
|
||||
* memory that is currently bound in objects
|
||||
* @return used bytes
|
||||
*/
|
||||
protected abstract long used();
|
||||
|
||||
/**
|
||||
* currently allocated memory in the Java virtual machine; may vary over time
|
||||
* @return bytes
|
||||
*/
|
||||
protected abstract long total();
|
||||
|
||||
/**
|
||||
* maximum memory the Java virtual will allocate machine; may vary over time in some cases
|
||||
* @return bytes
|
||||
*/
|
||||
protected abstract long maxMemory();
|
||||
|
||||
/**
|
||||
* <p>Tries to free a specified amount of bytes.</p>
|
||||
* <p>
|
||||
* If the currently available memory is enough, the method returns <code>true</code> without
|
||||
* performing additional steps. If not, the behaviour depends on the parameter <code>force</code>.
|
||||
* If <code>false</code>, a Full GC is only performed if former GCs indicated that a GC should
|
||||
* provide enough free memory. If former GCs didn't but <code>force</code> is set to <code>true</code>
|
||||
* a Full GC is performed nevertheless.
|
||||
* </p>
|
||||
* <p>
|
||||
* Setting the <code>force</code> parameter to false doesn't necessarily mean, that no GC may be
|
||||
* performed by this method, if the currently available memory doesn't suffice!
|
||||
* </p>
|
||||
* <p><em>Be careful with this method as GCs should always be the last measure to take</em></p>
|
||||
*
|
||||
* @param size the requested amount of free memory in bytes
|
||||
* @param force specifies whether a GC should be run even in case former GCs didn't provide enough memory
|
||||
* @return whether enough memory could be freed (or is free) or not
|
||||
*/
|
||||
protected abstract boolean request(final long size, final boolean force, boolean shortStatus);
|
||||
|
||||
/**
|
||||
* @return if Memory seams to be in a proper state
|
||||
*/
|
||||
protected abstract boolean properState();
|
||||
|
||||
/**
|
||||
* forced enable properState - StandardMemoryStrategy only
|
||||
*/
|
||||
protected void resetProperState() {
|
||||
}
|
||||
|
||||
/**
|
||||
* set the memory to be available for properState - StandardMemoryStrategy only
|
||||
*/
|
||||
protected void setProperMbyte(final long mbyte) {
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,223 @@
|
||||
// MemoryControl.java
|
||||
// -------------------------------------------
|
||||
// (C) 2005 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||
// first published 22.09.2005 on http://yacy.net
|
||||
//
|
||||
// $LastChangedDate: 2011-08-18 00:24:17 +0200 (Do, 18. Aug 2011) $
|
||||
// $LastChangedRevision: 7883 $
|
||||
// $LastChangedBy: orbiter $
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package net.yacy.kelondro.util;
|
||||
|
||||
/**
|
||||
* Standard implementation to get information about memory usage or try to free some memory
|
||||
*/
|
||||
public class StandardMemoryStrategy extends MemoryStrategy {
|
||||
|
||||
|
||||
private final Runtime runtime = Runtime.getRuntime();
|
||||
private final long[] gcs = new long[5];
|
||||
private int gcs_pos = 0;
|
||||
private long properMbyte = 0L;
|
||||
private long prevTreshold = 0L;
|
||||
private int tresholdCount = 0;
|
||||
private boolean proper = true;
|
||||
|
||||
public StandardMemoryStrategy() {
|
||||
name = "Standard Memory Strategy";
|
||||
error= false; //since this is the standard implementation we assume always false here
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs the garbage collector if last garbage collection is more than last millis ago
|
||||
* @param last time which must be passed since lased gc
|
||||
* @param info additional info for log
|
||||
*/
|
||||
protected final synchronized boolean gc(final int last, final String info) { // thq
|
||||
assert last >= 10000; // too many forced GCs will cause bad execution performance
|
||||
final long elapsed = System.currentTimeMillis() - lastGC;
|
||||
if (elapsed > last) {
|
||||
final long before = free();
|
||||
final long start = System.currentTimeMillis();
|
||||
System.gc();
|
||||
lastGC = System.currentTimeMillis();
|
||||
final long after = free();
|
||||
gcs[gcs_pos++] = after - before;
|
||||
if (gcs_pos >= gcs.length) gcs_pos = 0;
|
||||
|
||||
if (log.isFine()) log.logInfo("[gc] before: " + Formatter.bytesToString(before) +
|
||||
", after: " + Formatter.bytesToString(after) +
|
||||
", freed: " + Formatter.bytesToString(after - before) +
|
||||
", rt: " + (lastGC - start) + " ms, call: " + info);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (log.isFinest()) log.logFinest("[gc] no execute, last run: " + (elapsed / 1000) + " seconds ago, call: " + info);
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* This method calculates the average amount of bytes freed by the last GCs forced by this class
|
||||
* @return the average amount of freed bytes of the last forced GCs or <code>0</code> if no
|
||||
* GC has been run yet
|
||||
*/
|
||||
protected final long getAverageGCFree() {
|
||||
long x = 0;
|
||||
int y = 0;
|
||||
for (final long gc : gcs)
|
||||
if (gc != 0) {
|
||||
x += gc;
|
||||
y++;
|
||||
}
|
||||
return (y == 0) ? 0 : x / y;
|
||||
}
|
||||
|
||||
/**
|
||||
* memory that is free without increasing of total memory taken from os
|
||||
* @return bytes
|
||||
*/
|
||||
protected final long free() {
|
||||
return runtime.freeMemory();
|
||||
}
|
||||
|
||||
/**
|
||||
* memory that is available including increasing total memory up to maximum
|
||||
* @return bytes
|
||||
*/
|
||||
protected final long available() {
|
||||
return maxMemory() - total() + free();
|
||||
}
|
||||
|
||||
/**
|
||||
* maximum memory the Java virtual will allocate machine; may vary over time in some cases
|
||||
* @return bytes
|
||||
*/
|
||||
protected final long maxMemory()
|
||||
{
|
||||
return runtime.maxMemory();
|
||||
}
|
||||
|
||||
/**
|
||||
* currently allocated memory in the Java virtual machine; may vary over time
|
||||
* @return bytes
|
||||
*/
|
||||
protected final long total()
|
||||
{
|
||||
return runtime.totalMemory();
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>Tries to free a specified amount of bytes.</p>
|
||||
* <p>
|
||||
* If the currently available memory is enough, the method returns <code>true</code> without
|
||||
* performing additional steps. If not, the behaviour depends on the parameter <code>force</code>.
|
||||
* If <code>false</code>, a Full GC is only performed if former GCs indicated that a GC should
|
||||
* provide enough free memory. If former GCs didn't but <code>force</code> is set to <code>true</code>
|
||||
* a Full GC is performed nevertheless.
|
||||
* </p>
|
||||
* <p>
|
||||
* Setting the <code>force</code> parameter to false doesn't necessarily mean, that no GC may be
|
||||
* performed by this method, if the currently available memory doesn't suffice!
|
||||
* </p>
|
||||
* <p><em>Be careful with this method as GCs should always be the last measure to take</em></p>
|
||||
*
|
||||
* @param size the requested amount of free memory in bytes
|
||||
* @param force specifies whether a GC should be run even in case former GCs didn't provide enough memory
|
||||
* @return whether enough memory could be freed (or is free) or not
|
||||
*/
|
||||
protected boolean request(final long size, final boolean force, boolean shortStatus) {
|
||||
if (size <= 0) return true;
|
||||
final boolean r = request0(size, force);
|
||||
shortStatus = !r;
|
||||
return r;
|
||||
}
|
||||
private boolean request0(final long size, final boolean force) {
|
||||
final long avg = getAverageGCFree();
|
||||
if (avg >= size) return true;
|
||||
long avail = available();
|
||||
if (avail >= size) return true;
|
||||
if (log.isFine()) {
|
||||
final String t = new Throwable("Stack trace").getStackTrace()[1].toString();
|
||||
log.logFine(t + " requested " + (size >> 10) + " KB, got " + (avail >> 10) + " KB");
|
||||
}
|
||||
if (force || avg == 0 || avg + avail >= size) {
|
||||
// this is only called if we expect that an allocation of <size> bytes would cause the jvm to call the GC anyway
|
||||
|
||||
final long memBefore = avail;
|
||||
final boolean performedGC = gc(10000, "serverMemory.runGC(...)");
|
||||
avail = available();
|
||||
if (performedGC) {
|
||||
final long freed = avail - memBefore;
|
||||
log.logInfo("performed " + ((force) ? "explicit" : "necessary") + " GC, freed " + (freed >> 10)
|
||||
+ " KB (requested/available/average: "
|
||||
+ (size >> 10) + " / " + (avail >> 10) + " / " + (avg >> 10) + " KB)");
|
||||
}
|
||||
checkProper(avail);
|
||||
return avail >= size;
|
||||
} else {
|
||||
if (log.isFine()) log.logFine("former GCs indicate to not be able to free enough memory (requested/available/average: "
|
||||
+ (size >> 10) + " / " + (avail >> 10) + " / " + (avg >> 10) + " KB)");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* memory that is currently bound in objects
|
||||
* @return used bytes
|
||||
*/
|
||||
protected final long used() {
|
||||
return total() - free();
|
||||
}
|
||||
|
||||
protected boolean properState() {
|
||||
return proper;
|
||||
}
|
||||
|
||||
protected void resetProperState() {
|
||||
proper = true;
|
||||
tresholdCount = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* set the memory to be available
|
||||
*/
|
||||
protected void setProperMbyte(final long mbyte) {
|
||||
properMbyte = mbyte;
|
||||
tresholdCount = 0;
|
||||
}
|
||||
|
||||
private void checkProper(final long available) {
|
||||
// disable proper state if memory is less than treshold - 4 times, maximum 11 minutes between each detection
|
||||
if ((available >> 20) < properMbyte) {
|
||||
final long t = System.currentTimeMillis();
|
||||
if(prevTreshold + 11L /* minutes */ * 60000L > t) {
|
||||
tresholdCount++;
|
||||
if(tresholdCount > 3 /* occurencies - 1 */) proper = false;
|
||||
}
|
||||
else tresholdCount = 1;
|
||||
|
||||
prevTreshold = t;
|
||||
|
||||
log.logInfo("checkProper: below treshold; tresholdCount: " + tresholdCount + "; proper: " + proper);
|
||||
}
|
||||
else if (!proper && (available >> 20) > (properMbyte * 2L)) // we were wrong!
|
||||
resetProperState();
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in new issue