defaultSettings = ((post == null) || (!(post.containsKey("submitdefault")))) ? null : FileUtils.loadMap(defaultSettingsFile);
@@ -350,7 +350,7 @@ public class PerformanceQueues_p {
final long diskFree = sb.getConfigLong(SwitchboardConstants.DISK_FREE, 3000L);
final long diskFreeHardlimit = sb.getConfigLong(SwitchboardConstants.DISK_FREE_HARDLIMIT, 1000L);
final long memoryAcceptDHT = sb.getConfigLong(SwitchboardConstants.MEMORY_ACCEPTDHT, 50000L);
- final boolean observerTrigger = !MemoryControl.getDHTallowed();
+ final boolean observerTrigger = !MemoryControl.properState();
prop.put("diskFree", diskFree);
prop.put("diskFreeHardlimit", diskFreeHardlimit);
prop.put("memoryAcceptDHT", memoryAcceptDHT);
diff --git a/source/de/anomic/crawler/ResourceObserver.java b/source/de/anomic/crawler/ResourceObserver.java
index c9b6cf641..3e30e1dc6 100644
--- a/source/de/anomic/crawler/ResourceObserver.java
+++ b/source/de/anomic/crawler/ResourceObserver.java
@@ -62,7 +62,7 @@ public class ResourceObserver {
* checks the resources and pauses crawls if necessary
*/
public void resourceObserverJob() {
- MemoryControl.setDHTMbyte(getMinFreeMemory());
+ MemoryControl.setProperMbyte(getMinFreeMemory());
normalizedDiskFree = getNormalizedDiskFree();
normalizedMemoryFree = getNormalizedMemoryFree();
@@ -124,7 +124,7 @@ public class ResourceObserver {
}
private Space getNormalizedMemoryFree() {
- if(!MemoryControl.getDHTallowed()) return Space.LOW;
+ if(!MemoryControl.properState()) return Space.LOW;
return Space.HIGH;
}
diff --git a/source/net/yacy/kelondro/util/GenerationMemoryStrategy.java b/source/net/yacy/kelondro/util/GenerationMemoryStrategy.java
new file mode 100644
index 000000000..a2324086a
--- /dev/null
+++ b/source/net/yacy/kelondro/util/GenerationMemoryStrategy.java
@@ -0,0 +1,206 @@
+// MemoryControl.java
+// -------------------------------------------
+// (C) 2011 by Sebastian Gaebel
+// first published 22.08.2011 on http://yacy.net
+//
+// $LastChangedDate: 2011-08-18 00:24:17 +0200 (Do, 18. Aug 2011) $
+// $LastChangedRevision: 7883 $
+// $LastChangedBy: orbiter $
+//
+// LICENSE
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+package net.yacy.kelondro.util;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
+import java.lang.management.MemoryPoolMXBean;
+import java.lang.management.MemoryUsage;
+
+public class GenerationMemoryStrategy extends MemoryStrategy {
+
+ private MemoryPoolMXBean eden, survivor, old;
+ private MemoryMXBean heap;
+
+ public GenerationMemoryStrategy() {
+ name = "Generation Memory Strategy";
+ error = initPoolBeans();
+ heap = ManagementFactory.getMemoryMXBean();
+ if (lastGC == 0l) gc(10000, "initial gc - to get proper results"); // this is necessary on some GCs / vm
+ if (error) log.logWarning(name + ": not a generational heap");
+ }
+
+ /**
+ * memory that is free without increasing of total memory taken from os
+ * @return bytes
+ */
+ protected final long free() {
+ return youngUsage(false).getCommitted() - youngUsage(false).getUsed();
+ }
+
+ /**
+ * memory that is available including increasing total memory up to maximum
+ * Smallest of both old and young
+ * @return bytes
+ */
+ protected final long available() {
+ return available(false);
+ }
+
+ /**
+ * memory that is available including increasing total memory up to maximum
+ * @param force specifies whether ignoring prefered size
+ * @return bytes
+ */
+ private final long available(final boolean force) {
+ return force & properState() ? Math.max(youngAvailable(), oldAvailable()) : Math.min(youngAvailable(), oldAvailable());
+ }
+
+ /**
+ * memory that is currently bound in objects
+ * @return used bytes
+ */
+ protected final long used() {
+ return heap.getHeapMemoryUsage().getUsed();
+ }
+
+ /**
+ * currently allocated memory in the Java virtual machine; may vary over time
+ * @return bytes
+ */
+ protected final long total() {
+ return heap.getHeapMemoryUsage().getCommitted();
+ }
+
+ /**
+ * maximum memory the Java virtual will allocate machine; may vary over time in some cases
+ * @return bytes
+ */
+ protected final long maxMemory() {
+ return heap.getHeapMemoryUsage().getMax();
+ }
+
+ /**
+ * checks if a specified amount of bytes are available
+ * after the jvm recycled unused objects
+ *
+ * @param size the requested amount of free memory in bytes
+ * @param force specifies whether ignoring prefered size
+ * @return whether enough memory could be freed (or is free) or not
+ */
+ protected final boolean request(final long size, final boolean force, boolean shortStatus) {
+ if (size == 0l) return true; // does not make sense to check - returning true without setting shortStatus (which also doesn't make sense to me)
+ final boolean unknown = size < 0l; // size < 0 indicate an unknown size - maybe from gziped streams
+ final boolean r = unknown? properState() : size < available(force);
+ shortStatus = !r;
+ return r;
+ }
+
+ /**
+ * use this to check for temporary space
+ * @return bytes available to allocate in Eden Space (Young Generation)
+ */
+ protected final long youngAvailable() {
+ return youngUsage(true).getCommitted() - youngUsage(true).getUsed();
+ }
+
+ /**
+ * @return bytes available to allocate in Tenured Space (Old Generation)
+ */
+ protected final long oldAvailable() {
+ return oldUsage(true).getCommitted() - oldUsage(true).getUsed();
+ }
+
+ /**
+ * alive objects get 'moved' on gc from eden space to survior and from survior to old gen
+ * in a worse case (all objects of survivor alive) all objects get 'moved' from suvivor to old gen
+ * this method checks if there is is space left in old gen for that
+ *
+ * @return Memory is in proper state
+ */
+ protected boolean properState() {
+ return (oldUsage(true).getUsed() + survivorUsage(false).getCommitted()) < oldUsage(false).getCommitted();
+ }
+
+ /**
+ * based on my research for a proper running jvm, this gives a guidance value for the heap size
+ *
+ * @return bytes recommend for heap size
+ */
+ protected long recommendHeapSize() {
+ // the heap/old-ration is jvm-specific and can be changed by parameter - using this + 20% buffer
+ final double factor = 1.2 * (double)heap.getHeapMemoryUsage().getMax() / (double)oldUsage(false).getMax();
+ // current needed space in old gen
+ final long neededOld = oldUsage(true).getUsed() + survivorUsage(false).getMax();
+ return (long) (neededOld * factor);
+ }
+
+ /**
+ * @param collected specifies whether trying to get the memory usage after the jvm recycled unused objects
+ * @return MemoryUsage of Eden Space aka Young Gen
+ */
+ private MemoryUsage youngUsage(final boolean collected) {
+ if (collected) {
+ final MemoryUsage usage = eden.getCollectionUsage();
+ if (usage != null) return usage;
+ error = true;
+ log.logWarning(name + ": no young colletion usage available");
+ }
+ return eden.getUsage();
+ }
+
+ /**
+ * @param collected specifies whether trying to get the memory usage after the jvm recycled unused objects
+ * @return MemoryUsage of Survivor
+ */
+ private MemoryUsage survivorUsage(final boolean collected) {
+ if (collected) {
+ final MemoryUsage usage = survivor.getCollectionUsage();
+ if (usage != null) return usage;
+ error = true;
+ log.logWarning(name + ": no survivior colletion usage available");
+ }
+ return survivor.getUsage();
+ }
+
+ /**
+ * @param collected specifies whether trying to get the memory usage after the jvm recycled unused objects
+ * @return MemoryUsage of Old Gen
+ */
+ private MemoryUsage oldUsage(final boolean collected) {
+ if (collected) {
+ final MemoryUsage usage = old.getCollectionUsage();
+ if (usage != null) return usage;
+ error = true;
+ log.logWarning(name + ": no old colletion usage available");
+ }
+ return old.getUsage();
+ }
+
+ private boolean initPoolBeans() {
+ for (final MemoryPoolMXBean bean : ManagementFactory.getMemoryPoolMXBeans()) {
+ if (bean.getName().startsWith("G1")) break; //this strategy will not run on G1
+ if (bean.getName().contains("Eden")) {
+ eden = bean;
+ } else if (bean.getName().contains("Survivor")) {
+ survivor = bean;
+ } else if (bean.getName().contains("Old") || bean.getName().contains("Tenured")) {
+ old = bean;
+ }
+ }
+ return eden == null || survivor == null || old == null;
+ }
+}
diff --git a/source/net/yacy/kelondro/util/MemoryControl.java b/source/net/yacy/kelondro/util/MemoryControl.java
index 2657bd375..55f2e4a8c 100644
--- a/source/net/yacy/kelondro/util/MemoryControl.java
+++ b/source/net/yacy/kelondro/util/MemoryControl.java
@@ -25,25 +25,43 @@
package net.yacy.kelondro.util;
-import net.yacy.kelondro.logging.Log;
-
/**
* Use this to get information about memory usage or try to free some memory
*/
public class MemoryControl {
-
- private static final Runtime runtime = Runtime.getRuntime();
- private static final Log log = new Log("MEMORY");
-
- private static final long[] gcs = new long[5];
- private static int gcs_pos = 0;
- private static long lastGC = 0l;
- private static long DHTMbyte = 0L;
- private static long prevDHTtreshold = 0L;
- private static int DHTtresholdCount = 0;
- private static boolean allowDHT = true;
- private static boolean shortStatus = false, simulatedShortStatus = false;
+ private static boolean shortStatus = false, simulatedShortStatus = false, usingStandardStrategy = true;
+ private static MemoryStrategy strategy;
+
+ private static MemoryStrategy getStrategy() {
+ if (strategy == null || strategy.hasError()) {
+ if (!usingStandardStrategy) {
+ strategy = new GenerationMemoryStrategy();
+ // fall back if error detected
+ if (strategy.hasError()) {
+ usingStandardStrategy = true;
+ strategy = new StandardMemoryStrategy();
+ }
+ } else {
+ strategy = new StandardMemoryStrategy();
+ }
+ }
+ return strategy;
+ }
+
+ public final static void setStandardStrategy(final boolean std) {
+ if (usingStandardStrategy != std) {
+ usingStandardStrategy = std;
+ strategy = null;
+ }
+ }
+
+ /**
+ * @return the name of the used strategy
+ */
+ public final static String getStrategyName() {
+ return getStrategy().getName();
+ }
/**
* Runs the garbage collector if last garbage collection is more than last millis ago
@@ -51,42 +69,7 @@ public class MemoryControl {
* @param info additional info for log
*/
public final synchronized static boolean gc(final int last, final String info) { // thq
- assert last >= 10000; // too many forced GCs will cause bad execution performance
- final long elapsed = System.currentTimeMillis() - lastGC;
- if (elapsed > last) {
- final long before = free();
- final long start = System.currentTimeMillis();
- System.gc();
- lastGC = System.currentTimeMillis();
- final long after = free();
- gcs[gcs_pos++] = after - before;
- if (gcs_pos >= gcs.length) gcs_pos = 0;
-
- if (log.isFine()) log.logInfo("[gc] before: " + Formatter.bytesToString(before) +
- ", after: " + Formatter.bytesToString(after) +
- ", freed: " + Formatter.bytesToString(after - before) +
- ", rt: " + (lastGC - start) + " ms, call: " + info);
- return true;
- }
-
- if (log.isFinest()) log.logFinest("[gc] no execute, last run: " + (elapsed / 1000) + " seconds ago, call: " + info);
- return false;
- }
-
- /**
- * This method calculates the average amount of bytes freed by the last GCs forced by this class
- * @return the average amount of freed bytes of the last forced GCs or 0 if no
- * GC has been run yet
- */
- public static long getAverageGCFree() {
- long x = 0;
- int y = 0;
- for (final long gc : gcs)
- if (gc != 0) {
- x += gc;
- y++;
- }
- return (y == 0) ? 0 : x / y;
+ return getStrategy().gc(last, info);
}
/**
@@ -94,7 +77,7 @@ public class MemoryControl {
* @return bytes
*/
public static final long free() {
- return runtime.freeMemory();
+ return getStrategy().free();
}
/**
@@ -102,7 +85,7 @@ public class MemoryControl {
* @return bytes
*/
public static final long available() {
- return maxMemory() - total() + free();
+ return getStrategy().available();
}
/**
@@ -111,7 +94,7 @@ public class MemoryControl {
*/
public static final long maxMemory()
{
- return runtime.maxMemory();
+ return getStrategy().maxMemory();
}
/**
@@ -120,62 +103,18 @@ public class MemoryControl {
*/
public static final long total()
{
- return runtime.totalMemory();
+ return getStrategy().total();
}
/**
- * Tries to free a specified amount of bytes.
- *
- * If the currently available memory is enough, the method returns true without
- * performing additional steps. If not, the behaviour depends on the parameter force.
- * If false, a Full GC is only performed if former GCs indicated that a GC should
- * provide enough free memory. If former GCs didn't but force is set to true
- * a Full GC is performed nevertheless.
- *
- *
- * Setting the force parameter to false doesn't necessarily mean, that no GC may be
- * performed by this method, if the currently available memory doesn't suffice!
- *
- * Be careful with this method as GCs should always be the last measure to take
- *
+ * check for a specified amount of bytes
+ *
* @param size the requested amount of free memory in bytes
- * @param force specifies whether a GC should be run even in case former GCs didn't provide enough memory
+ * @param force specifies whether risk an expensive GC
* @return whether enough memory could be freed (or is free) or not
*/
public static boolean request(final long size, final boolean force) {
- if (size <= 0) return true;
- final boolean r = request0(size, force);
- shortStatus = !r;
- return r;
- }
- private static boolean request0(final long size, final boolean force) {
- final long avg = getAverageGCFree();
- if (avg >= size) return true;
- long avail = available();
- if (avail >= size) return true;
- if (log.isFine()) {
- final String t = new Throwable("Stack trace").getStackTrace()[1].toString();
- log.logFine(t + " requested " + (size >> 10) + " KB, got " + (avail >> 10) + " KB");
- }
- if (force || avg == 0 || avg + avail >= size) {
- // this is only called if we expect that an allocation of bytes would cause the jvm to call the GC anyway
-
- final long memBefore = avail;
- final boolean performedGC = gc(10000, "serverMemory.runGC(...)");
- avail = available();
- if (performedGC) {
- final long freed = avail - memBefore;
- log.logInfo("performed " + ((force) ? "explicit" : "necessary") + " GC, freed " + (freed >> 10)
- + " KB (requested/available/average: "
- + (size >> 10) + " / " + (avail >> 10) + " / " + (avg >> 10) + " KB)");
- }
- checkDHTrule(avail);
- return avail >= size;
- } else {
- if (log.isFine()) log.logFine("former GCs indicate to not be able to free enough memory (requested/available/average: "
- + (size >> 10) + " / " + (avail >> 10) + " / " + (avg >> 10) + " KB)");
- return false;
- }
+ return getStrategy().request(size, force, shortStatus);
}
/**
@@ -194,6 +133,9 @@ public class MemoryControl {
return simulatedShortStatus;
}
+ /**
+ * @return if last request failed
+ */
public static boolean shortStatus() {
//if (shortStatus) System.out.println("**** SHORT MEMORY ****");
return simulatedShortStatus || shortStatus;
@@ -204,56 +146,50 @@ public class MemoryControl {
* @return used bytes
*/
public static long used() {
- return total() - free();
- }
-
- public static boolean getDHTallowed() {
- return allowDHT;
+ return getStrategy().used();
}
- public static void setDHTallowed() {
- allowDHT = true;
- DHTtresholdCount = 0;
+ /**
+ * @return if Memory seams to be in a proper state
+ */
+ public static boolean properState() {
+ return getStrategy().properState();
}
/**
- * set the memory to be available
+ * forced enable properState - StandardMemoryStrategy only
*/
- public static void setDHTMbyte(final long mbyte) {
- DHTMbyte = mbyte;
- DHTtresholdCount = 0;
+ public static void resetProperState() {
+ getStrategy().resetProperState();
}
- private static void checkDHTrule(final long available) {
- // disable dht if memory is less than treshold - 4 times, maximum 11 minutes between each detection
- if ((available >> 20) < DHTMbyte) {
- final long t = System.currentTimeMillis();
- if(prevDHTtreshold + 11L /* minutes */ * 60000L > t) {
- DHTtresholdCount++;
- if(DHTtresholdCount > 3 /* occurencies - 1 */) allowDHT = false;
- }
- else DHTtresholdCount = 1;
-
- prevDHTtreshold = t;
-
- log.logInfo("checkDHTrule: below treshold; tresholdCount: " + DHTtresholdCount + "; allowDHT: " + allowDHT);
- }
- else if (!allowDHT && (available >> 20) > (DHTMbyte * 2L)) // we were wrong!
- setDHTallowed();
+ /**
+ * set the memory to be available for properState - StandardMemoryStrategy only
+ */
+ public static void setProperMbyte(final long mbyte) {
+ getStrategy().setProperMbyte(mbyte);
}
/**
* main
- * @param args
+ * @param args use 'force' to request by force, use 'std' / 'gen' to specify strategy
*/
public static void main(final String[] args) {
- // try this with a jvm 1.4.2 and with a jvm 1.5 and compare results
+ // try this with different strategy and compare results
final int mb = 1024 * 1024;
+ boolean force = false;
+ for (final String arg : args) {
+ if (arg.equals("force")) force = true;
+ if (arg.equalsIgnoreCase("gen")) usingStandardStrategy = false;
+ if (arg.equalsIgnoreCase("std")) usingStandardStrategy = true;
+ }
System.out.println("vm: " + System.getProperty("java.vm.version"));
System.out.println("computed max = " + (maxMemory() / mb) + " mb");
+ System.out.println("using " + getStrategyName());
final byte[][] x = new byte[100000][];
+
for (int i = 0; i < 100000; i++) {
- if (request(mb, false))
+ if (request(mb, force))
{
x[i] = new byte[mb];
System.out.println("used = " + i + " / " + (used() /mb) +
@@ -261,8 +197,8 @@ public class MemoryControl {
", free = " + (free() / mb) +
", max = " + (maxMemory() / mb) +
", avail = " + (available() / mb) +
- ", averageGC = " + getAverageGCFree());
- }
+ (usingStandardStrategy? ", averageGC = " + ((StandardMemoryStrategy)getStrategy()).getAverageGCFree() : ""));
+ } else System.exit(0);
}
}
diff --git a/source/net/yacy/kelondro/util/MemoryStrategy.java b/source/net/yacy/kelondro/util/MemoryStrategy.java
new file mode 100644
index 000000000..3df8b2dfa
--- /dev/null
+++ b/source/net/yacy/kelondro/util/MemoryStrategy.java
@@ -0,0 +1,138 @@
+// MemoryControl.java
+// -------------------------------------------
+// (C) 2011 by Sebastian Gaebel
+// first published 22.08.2011 on http://yacy.net
+//
+// $LastChangedDate: 2011-08-18 00:24:17 +0200 (Do, 18. Aug 2011) $
+// $LastChangedRevision: 7883 $
+// $LastChangedBy: orbiter $
+//
+// LICENSE
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+package net.yacy.kelondro.util;
+
+import java.lang.management.ManagementFactory;
+
+import net.yacy.kelondro.logging.Log;
+
+public abstract class MemoryStrategy {
+
+ protected final static Log log = new Log("MEMORY");
+ protected static long lastGC = 0l;
+ protected static boolean error = true;
+ protected static String name;
+
+ /**
+ * @return if an error has been detected
+ */
+ protected final boolean hasError() {
+ return error;
+ }
+
+ /**
+ * @return an identifying name
+ */
+ protected final String getName() {
+ return name;
+ }
+
+ /**
+ * Runs the garbage collector if last garbage collection is more than last millis ago
+ * @param last time which must be passed since lased gc
+ * @param info additional info for log
+ */
+ protected synchronized boolean gc(final int last, final String info) { // thq
+ assert last >= 10000; // too many forced GCs will cause bad execution performance
+ final long elapsed = System.currentTimeMillis() - lastGC;
+ if (elapsed > last) {
+ ManagementFactory.getMemoryMXBean().gc();
+ lastGC = System.currentTimeMillis();
+ return true;
+ }
+
+ if (log.isFinest()) log.logFinest("[gc] no execute, last run: " + (elapsed / 1000) + " seconds ago, call: " + info);
+ return false;
+ }
+
+ /**
+ * memory that is free without increasing of total memory taken from os
+ * @return bytes
+ */
+ protected abstract long free();
+
+ /**
+ * memory that is available including increasing total memory up to maximum
+ * @return bytes
+ */
+ protected abstract long available();
+
+ /**
+ * memory that is currently bound in objects
+ * @return used bytes
+ */
+ protected abstract long used();
+
+ /**
+ * currently allocated memory in the Java virtual machine; may vary over time
+ * @return bytes
+ */
+ protected abstract long total();
+
+ /**
+ * maximum memory the Java virtual will allocate machine; may vary over time in some cases
+ * @return bytes
+ */
+ protected abstract long maxMemory();
+
+ /**
+ * Tries to free a specified amount of bytes.
+ *
+ * If the currently available memory is enough, the method returns true without
+ * performing additional steps. If not, the behaviour depends on the parameter force.
+ * If false, a Full GC is only performed if former GCs indicated that a GC should
+ * provide enough free memory. If former GCs didn't but force is set to true
+ * a Full GC is performed nevertheless.
+ *
+ *
+ * Setting the force parameter to false doesn't necessarily mean, that no GC may be
+ * performed by this method, if the currently available memory doesn't suffice!
+ *
+ * Be careful with this method as GCs should always be the last measure to take
+ *
+ * @param size the requested amount of free memory in bytes
+ * @param force specifies whether a GC should be run even in case former GCs didn't provide enough memory
+ * @return whether enough memory could be freed (or is free) or not
+ */
+ protected abstract boolean request(final long size, final boolean force, boolean shortStatus);
+
+ /**
+ * @return if Memory seams to be in a proper state
+ */
+ protected abstract boolean properState();
+
+ /**
+ * forced enable properState - StandardMemoryStrategy only
+ */
+ protected void resetProperState() {
+ }
+
+ /**
+ * set the memory to be available for properState - StandardMemoryStrategy only
+ */
+ protected void setProperMbyte(final long mbyte) {
+ }
+}
diff --git a/source/net/yacy/kelondro/util/StandardMemoryStrategy.java b/source/net/yacy/kelondro/util/StandardMemoryStrategy.java
new file mode 100644
index 000000000..581a21d8d
--- /dev/null
+++ b/source/net/yacy/kelondro/util/StandardMemoryStrategy.java
@@ -0,0 +1,223 @@
+// MemoryControl.java
+// -------------------------------------------
+// (C) 2005 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
+// first published 22.09.2005 on http://yacy.net
+//
+// $LastChangedDate: 2011-08-18 00:24:17 +0200 (Do, 18. Aug 2011) $
+// $LastChangedRevision: 7883 $
+// $LastChangedBy: orbiter $
+//
+// LICENSE
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+package net.yacy.kelondro.util;
+
+/**
+ * Standard implementation to get information about memory usage or try to free some memory
+ */
+public class StandardMemoryStrategy extends MemoryStrategy {
+
+
+ private final Runtime runtime = Runtime.getRuntime();
+ private final long[] gcs = new long[5];
+ private int gcs_pos = 0;
+ private long properMbyte = 0L;
+ private long prevTreshold = 0L;
+ private int tresholdCount = 0;
+ private boolean proper = true;
+
+ public StandardMemoryStrategy() {
+ name = "Standard Memory Strategy";
+ error= false; //since this is the standard implementation we assume always false here
+ }
+
+ /**
+ * Runs the garbage collector if last garbage collection is more than last millis ago
+ * @param last time which must be passed since lased gc
+ * @param info additional info for log
+ */
+ protected final synchronized boolean gc(final int last, final String info) { // thq
+ assert last >= 10000; // too many forced GCs will cause bad execution performance
+ final long elapsed = System.currentTimeMillis() - lastGC;
+ if (elapsed > last) {
+ final long before = free();
+ final long start = System.currentTimeMillis();
+ System.gc();
+ lastGC = System.currentTimeMillis();
+ final long after = free();
+ gcs[gcs_pos++] = after - before;
+ if (gcs_pos >= gcs.length) gcs_pos = 0;
+
+ if (log.isFine()) log.logInfo("[gc] before: " + Formatter.bytesToString(before) +
+ ", after: " + Formatter.bytesToString(after) +
+ ", freed: " + Formatter.bytesToString(after - before) +
+ ", rt: " + (lastGC - start) + " ms, call: " + info);
+ return true;
+ }
+
+ if (log.isFinest()) log.logFinest("[gc] no execute, last run: " + (elapsed / 1000) + " seconds ago, call: " + info);
+ return false;
+ }
+
+ /**
+ * This method calculates the average amount of bytes freed by the last GCs forced by this class
+ * @return the average amount of freed bytes of the last forced GCs or 0 if no
+ * GC has been run yet
+ */
+ protected final long getAverageGCFree() {
+ long x = 0;
+ int y = 0;
+ for (final long gc : gcs)
+ if (gc != 0) {
+ x += gc;
+ y++;
+ }
+ return (y == 0) ? 0 : x / y;
+ }
+
+ /**
+ * memory that is free without increasing of total memory taken from os
+ * @return bytes
+ */
+ protected final long free() {
+ return runtime.freeMemory();
+ }
+
+ /**
+ * memory that is available including increasing total memory up to maximum
+ * @return bytes
+ */
+ protected final long available() {
+ return maxMemory() - total() + free();
+ }
+
+ /**
+ * maximum memory the Java virtual will allocate machine; may vary over time in some cases
+ * @return bytes
+ */
+ protected final long maxMemory()
+ {
+ return runtime.maxMemory();
+ }
+
+ /**
+ * currently allocated memory in the Java virtual machine; may vary over time
+ * @return bytes
+ */
+ protected final long total()
+ {
+ return runtime.totalMemory();
+ }
+
+ /**
+ * Tries to free a specified amount of bytes.
+ *
+ * If the currently available memory is enough, the method returns true without
+ * performing additional steps. If not, the behaviour depends on the parameter force.
+ * If false, a Full GC is only performed if former GCs indicated that a GC should
+ * provide enough free memory. If former GCs didn't but force is set to true
+ * a Full GC is performed nevertheless.
+ *
+ *
+ * Setting the force parameter to false doesn't necessarily mean, that no GC may be
+ * performed by this method, if the currently available memory doesn't suffice!
+ *
+ * Be careful with this method as GCs should always be the last measure to take
+ *
+ * @param size the requested amount of free memory in bytes
+ * @param force specifies whether a GC should be run even in case former GCs didn't provide enough memory
+ * @return whether enough memory could be freed (or is free) or not
+ */
+ protected boolean request(final long size, final boolean force, boolean shortStatus) {
+ if (size <= 0) return true;
+ final boolean r = request0(size, force);
+ shortStatus = !r;
+ return r;
+ }
+ private boolean request0(final long size, final boolean force) {
+ final long avg = getAverageGCFree();
+ if (avg >= size) return true;
+ long avail = available();
+ if (avail >= size) return true;
+ if (log.isFine()) {
+ final String t = new Throwable("Stack trace").getStackTrace()[1].toString();
+ log.logFine(t + " requested " + (size >> 10) + " KB, got " + (avail >> 10) + " KB");
+ }
+ if (force || avg == 0 || avg + avail >= size) {
+ // this is only called if we expect that an allocation of bytes would cause the jvm to call the GC anyway
+
+ final long memBefore = avail;
+ final boolean performedGC = gc(10000, "serverMemory.runGC(...)");
+ avail = available();
+ if (performedGC) {
+ final long freed = avail - memBefore;
+ log.logInfo("performed " + ((force) ? "explicit" : "necessary") + " GC, freed " + (freed >> 10)
+ + " KB (requested/available/average: "
+ + (size >> 10) + " / " + (avail >> 10) + " / " + (avg >> 10) + " KB)");
+ }
+ checkProper(avail);
+ return avail >= size;
+ } else {
+ if (log.isFine()) log.logFine("former GCs indicate to not be able to free enough memory (requested/available/average: "
+ + (size >> 10) + " / " + (avail >> 10) + " / " + (avg >> 10) + " KB)");
+ return false;
+ }
+ }
+
+ /**
+ * memory that is currently bound in objects
+ * @return used bytes
+ */
+ protected final long used() {
+ return total() - free();
+ }
+
+ protected boolean properState() {
+ return proper;
+ }
+
+ protected void resetProperState() {
+ proper = true;
+ tresholdCount = 0;
+ }
+
+ /**
+ * set the memory to be available
+ */
+ protected void setProperMbyte(final long mbyte) {
+ properMbyte = mbyte;
+ tresholdCount = 0;
+ }
+
+ private void checkProper(final long available) {
+ // disable proper state if memory is less than treshold - 4 times, maximum 11 minutes between each detection
+ if ((available >> 20) < properMbyte) {
+ final long t = System.currentTimeMillis();
+ if(prevTreshold + 11L /* minutes */ * 60000L > t) {
+ tresholdCount++;
+ if(tresholdCount > 3 /* occurencies - 1 */) proper = false;
+ }
+ else tresholdCount = 1;
+
+ prevTreshold = t;
+
+ log.logInfo("checkProper: below treshold; tresholdCount: " + tresholdCount + "; proper: " + proper);
+ }
+ else if (!proper && (available >> 20) > (properMbyte * 2L)) // we were wrong!
+ resetProperState();
+ }
+
+}
diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java
index 1b3e3431f..082b6c398 100644
--- a/source/net/yacy/yacy.java
+++ b/source/net/yacy/yacy.java
@@ -209,6 +209,9 @@ public final class yacy {
}
sb = new Switchboard(dataHome, appHome, "defaults/yacy.init".replace("/", File.separator), newconf);
//sbSync.V(); // signal that the sb reference was set
+
+ // switch the memory strategy
+ MemoryControl.setStandardStrategy(sb.getConfigBool("memory.standardStrategy", true));
// save information about available memory at startup time
sb.setConfig("memoryFreeAfterStartup", startupMemFree);