diff --git a/htroot/Performance_p.html b/htroot/Performance_p.html
index ff4f43df8..cc8792e05 100644
--- a/htroot/Performance_p.html
+++ b/htroot/Performance_p.html
@@ -21,10 +21,12 @@
Total
Cycles |
Idle
Cycles |
Busy
Cycles |
+Short Mem
Cycles |
Sleep Time
per Cycle
(milliseconds) |
Exec Time
per Busy-Cycle
(milliseconds) |
Delay between
idle loops |
Delay between
busy loops |
+Minimum of
Required Memory |
Full Description |
#{table}#
@@ -40,10 +42,12 @@
#[totalcycles]# |
#[idlecycles]# |
#[busycycles]# |
+#[memscycles]# |
#[sleeppercycle]# |
#[execpercycle]# |
milliseconds |
milliseconds |
+ bytes |
#[longdescr]# |
#{/table}#
diff --git a/htroot/Performance_p.java b/htroot/Performance_p.java
index a5cad4b20..67d2c5725 100644
--- a/htroot/Performance_p.java
+++ b/htroot/Performance_p.java
@@ -81,11 +81,11 @@ public class Performance_p {
// set templates for latest news from the threads
long blocktime, sleeptime, exectime;
- long idlesleep, busysleep;
+ long idlesleep, busysleep, memprereq;
int queuesize;
threads = switchboard.threadNames();
int c = 0;
- long idleCycles, busyCycles;
+ long idleCycles, busyCycles, memshortageCycles;
while (threads.hasNext()) {
threadName = (String) threads.next();
thread = switchboard.getThread(threadName);
@@ -102,37 +102,43 @@ public class Performance_p {
exectime = thread.getExecTime();
idleCycles = thread.getIdleCycles();
busyCycles = thread.getBusyCycles();
+ memshortageCycles = thread.getOutOfMemoryCycles();
prop.put("table_" + c + "_blocktime", blocktime / 1000);
prop.put("table_" + c + "_blockpercent", "" + (100 * blocktime / blocktime_total));
prop.put("table_" + c + "_sleeptime", sleeptime / 1000);
prop.put("table_" + c + "_sleeppercent", "" + (100 * sleeptime / sleeptime_total));
prop.put("table_" + c + "_exectime", exectime / 1000);
prop.put("table_" + c + "_execpercent", "" + (100 * exectime / exectime_total));
- prop.put("table_" + c + "_totalcycles", "" + (idleCycles + busyCycles));
+ prop.put("table_" + c + "_totalcycles", "" + (idleCycles + busyCycles + memshortageCycles));
prop.put("table_" + c + "_idlecycles", "" + idleCycles);
prop.put("table_" + c + "_busycycles", "" + busyCycles);
+ prop.put("table_" + c + "_memscycles", "" + memshortageCycles);
prop.put("table_" + c + "_sleeppercycle", ((idleCycles + busyCycles) == 0) ? "-" : ("" + (sleeptime / (idleCycles + busyCycles))));
prop.put("table_" + c + "_execpercycle", (busyCycles == 0) ? "-" : ("" + (exectime / busyCycles)));
if ((post != null) && (post.containsKey("delaysubmit"))) {
// load with new values
- idlesleep = Long.parseLong((String) post.get(threadName + "_idlesleep", "1"));
- busysleep = Long.parseLong((String) post.get(threadName + "_busysleep", "1"));
-
+ idlesleep = Long.parseLong((String) post.get(threadName + "_idlesleep", "100"));
+ busysleep = Long.parseLong((String) post.get(threadName + "_busysleep", "1000"));
+ memprereq = Long.parseLong((String) post.get(threadName + "_memprereq", "0"));
+
// check values to prevent short-cut loops
if (idlesleep == 0) idlesleep = 1000;
// on-the-fly re-configuration
- switchboard.setThreadSleep(threadName, idlesleep, busysleep);
+ switchboard.setThreadPerformance(threadName, idlesleep, busysleep, memprereq);
switchboard.setConfig(threadName + "_idlesleep", idlesleep);
switchboard.setConfig(threadName + "_busysleep", busysleep);
+ switchboard.setConfig(threadName + "_memprereq", memprereq);
} else {
// load with old values
idlesleep = Long.parseLong(switchboard.getConfig(threadName + "_idlesleep" , "1000"));
busysleep = Long.parseLong(switchboard.getConfig(threadName + "_busysleep", "1000"));
+ memprereq = Long.parseLong(switchboard.getConfig(threadName + "_memprereq", "1000"));
}
prop.put("table_" + c + "_idlesleep", idlesleep);
prop.put("table_" + c + "_busysleep", busysleep);
+ prop.put("table_" + c + "_memprereq", memprereq);
c++;
}
diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java
index 524f2d774..e7eb8c184 100644
--- a/source/de/anomic/kelondro/kelondroRecords.java
+++ b/source/de/anomic/kelondro/kelondroRecords.java
@@ -77,8 +77,9 @@ import java.util.StringTokenizer;
public class kelondroRecords {
// constants
- private static int NUL = Integer.MIN_VALUE; // the meta value for the kelondroRecords' NUL abstraction
-
+ private static final int NUL = Integer.MIN_VALUE; // the meta value for the kelondroRecords' NUL abstraction
+ public static final long memBlock = 5000000; // do not fill cache further if the amount of available memory is less that this
+
// static seek pointers
private static long POS_MAGIC = 0; // 1 byte, byte: file type magic
private static long POS_BUSY = POS_MAGIC + 1; // 1 byte, byte: marker for synchronization
@@ -352,10 +353,12 @@ public class kelondroRecords {
// check for space in cache
// should be only called within a synchronized(XcacheHeaders) environment
if (XcacheSize == 0) return;
- while (XcacheHeaders.size() >= XcacheSize) {
+ Handle delkey;
+ while ((XcacheHeaders.size() >= XcacheSize) ||
+ ((XcacheHeaders.size() > 0) && (Runtime.getRuntime().freeMemory() < memBlock))) {
// delete one entry
try {
- Handle delkey = (Handle) XcacheScore.getMinObject(); // error (see below) here
+ delkey = (Handle) XcacheScore.getMinObject(); // error (see below) here
XcacheScore.deleteScore(delkey);
XcacheHeaders.remove(delkey);
} catch (NoSuchElementException e) {
@@ -365,6 +368,7 @@ public class kelondroRecords {
this.XcacheScore = new kelondroMScoreCluster();
this.XcacheHeaders = new HashMap();
}
+ delkey = null;
}
}
@@ -662,6 +666,7 @@ public class kelondroRecords {
synchronized (XcacheHeaders) {
// remember size to evaluate a cache size check need
int sizeBefore = XcacheHeaders.size();
+ //long memBefore = Runtime.getRuntime().freeMemory();
// generate cache entry
byte[][] cacheValue;
if (values == null) {
@@ -676,10 +681,11 @@ public class kelondroRecords {
cacheNode.ohBytes = this.ohBytes;
cacheNode.ohHandle = this.ohHandle;
// store the cache entry
- XcacheHeaders.put(cacheNode.handle, cacheNode);
- XcacheScore.setScore(handle, (int) ((System.currentTimeMillis() - XcacheStartup) / 1000));
+ boolean newentry = XcacheHeaders.put(cacheNode.handle, cacheNode) == null;
+ XcacheScore.setScore(cacheNode.handle, (int) ((System.currentTimeMillis() - XcacheStartup) / 1000));
// delete the cache entry
cacheNode = null;
+ //System.out.println("kelondroRecords cache4" + filename + ": cache record size = " + (memBefore - Runtime.getRuntime().freeMemory()) + " bytes" + ((newentry) ? " new" : ""));
// check cache size
if (XcacheHeaders.size() > sizeBefore) checkCacheSpace();
//System.out.println("kelondroRecords cache4" + filename + ": " + XcacheHeaders.size() + " entries, " + XcacheSize + " allowed.");
diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java
index af1d913e1..d12c3d781 100644
--- a/source/de/anomic/plasma/plasmaSwitchboard.java
+++ b/source/de/anomic/plasma/plasmaSwitchboard.java
@@ -148,7 +148,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// load slots
- public static int crawlSlots = 20;
+ public static int crawlSlots = 12;
// couloured list management
public static TreeSet blueList = null;
@@ -578,12 +578,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
//log.logDebug("CoreCrawl: queue is empty");
return false;
}
- if (Runtime.getRuntime().freeMemory() < 2000000) {
- log.logDebug("CoreCrawl: not enough memory available, dismissed (" +
- "free=" + Runtime.getRuntime().freeMemory() + ")");
- System.gc();
- return false;
- }
if (queueStack.size() >= crawlSlots) {
log.logDebug("CoreCrawl: too many processes in queue, dismissed (" +
"queueStack=" + queueStack.size() + ")");
@@ -639,12 +633,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
//log.logDebug("LimitCrawl: queue is empty");
return false;
}
- if (Runtime.getRuntime().freeMemory() < 2000000) {
- log.logDebug("limitCrawlTrigger: not enough memory available, dismissed (" +
- "free=" + Runtime.getRuntime().freeMemory() + ")");
- System.gc();
- return false;
- }
// if the server is busy, we do crawling more slowly
//if (!(cacheManager.idle())) try {Thread.currentThread().sleep(2000);} catch (InterruptedException e) {}
@@ -717,12 +705,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
//log.logDebug("GlobalCrawl: queue is empty");
return false;
}
- if (Runtime.getRuntime().freeMemory() < 2000000) {
- log.logDebug("remoteTriggeredCrawl: not enough memory available, dismissed (" +
- "free=" + Runtime.getRuntime().freeMemory() + ")");
- System.gc();
- return false;
- }
/*
if (queueStack.size() > 0) {
log.logDebug("GlobalCrawl: any processe is in queue, dismissed (" +
diff --git a/source/de/anomic/server/serverAbstractSwitch.java b/source/de/anomic/server/serverAbstractSwitch.java
index 274c70155..03e563076 100644
--- a/source/de/anomic/server/serverAbstractSwitch.java
+++ b/source/de/anomic/server/serverAbstractSwitch.java
@@ -254,31 +254,52 @@ public abstract class serverAbstractSwitch implements serverSwitch {
log.logInfo("Undeployed Action '" + action.getShortDescription() + "', (" + switchActions.size() + " actions registered)");
}
- public void deployThread(String threadName, String threadShortDescription, String threadLongDescription, serverThread newThread, long startupDelay) {
+ public void deployThread(
+ String threadName,
+ String threadShortDescription,
+ String threadLongDescription,
+ serverThread newThread,
+ long startupDelay) {
deployThread(threadName, threadShortDescription, threadLongDescription,
newThread, startupDelay,
- Long.parseLong(getConfig(threadName + "_idlesleep" , "novalue")),
- Long.parseLong(getConfig(threadName + "_busysleep" , "novalue")));
+ Long.parseLong(getConfig(threadName + "_idlesleep" , "100")),
+ Long.parseLong(getConfig(threadName + "_busysleep" , "1000")),
+ Long.parseLong(getConfig(threadName + "_memprereq" , "1000000")));
}
- public void deployThread(String threadName, String threadShortDescription, String threadLongDescription, serverThread newThread, long startupDelay, long initialIdleSleep, long initialBusySleep) {
+ public void deployThread(
+ String threadName,
+ String threadShortDescription,
+ String threadLongDescription,
+ serverThread newThread,
+ long startupDelay,
+ long initialIdleSleep,
+ long initialBusySleep,
+ long initialMemoryPreRequisite) {
if (newThread.isAlive()) throw new RuntimeException("undeployed threads must not live; they are started as part of the deployment");
newThread.setStartupSleep(startupDelay);
- long sleep;
+ long x;
try {
- sleep = Long.parseLong(getConfig(threadName + "_idlesleep" , "novalue"));
- newThread.setIdleSleep(sleep);
+ x = Long.parseLong(getConfig(threadName + "_idlesleep" , "novalue"));
+ newThread.setIdleSleep(x);
} catch (NumberFormatException e) {
newThread.setIdleSleep(initialIdleSleep);
setConfig(threadName + "_idlesleep", initialIdleSleep);
}
try {
- sleep = Long.parseLong(getConfig(threadName + "_busysleep" , "novalue"));
- newThread.setBusySleep(sleep);
+ x = Long.parseLong(getConfig(threadName + "_busysleep" , "novalue"));
+ newThread.setBusySleep(x);
} catch (NumberFormatException e) {
newThread.setBusySleep(initialBusySleep);
setConfig(threadName + "_busysleep", initialBusySleep);
}
+ try {
+ x = Long.parseLong(getConfig(threadName + "_memprereq" , "novalue"));
+ newThread.setMemPreReqisite(x);
+ } catch (NumberFormatException e) {
+ newThread.setMemPreReqisite(initialMemoryPreRequisite);
+ setConfig(threadName + "_memprereq", initialMemoryPreRequisite);
+ }
newThread.setLog(log);
newThread.setDescription(threadShortDescription, threadLongDescription);
workerThreads.put(threadName, newThread);
@@ -290,11 +311,12 @@ public abstract class serverAbstractSwitch implements serverSwitch {
return (serverThread) workerThreads.get(threadName);
}
- public void setThreadSleep(String threadName, long idleMillis, long busyMillis) {
+ public void setThreadPerformance(String threadName, long idleMillis, long busyMillis, long memprereqBytes) {
serverThread thread = (serverThread) workerThreads.get(threadName);
if (thread != null) {
thread.setIdleSleep(idleMillis);
thread.setBusySleep(busyMillis);
+ thread.setMemPreReqisite(memprereqBytes);
}
}
diff --git a/source/de/anomic/server/serverAbstractThread.java b/source/de/anomic/server/serverAbstractThread.java
index 7f9210443..abeb5847e 100644
--- a/source/de/anomic/server/serverAbstractThread.java
+++ b/source/de/anomic/server/serverAbstractThread.java
@@ -57,10 +57,10 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
private long startup = 0, idlePause = 0, busyPause = 0, blockPause = 0;
private boolean running = true;
private serverLog log = null;
- private long idletime = 0, busytime = 0;
+ private long idletime = 0, busytime = 0, memprereq = 0;
private String shortDescr = "", longDescr = "";
private long threadBlockTimestamp = System.currentTimeMillis();
- private long idleCycles = 0, busyCycles = 0;
+ private long idleCycles = 0, busyCycles = 0, outofmemoryCycles = 0;
protected final void announceThreadBlockApply() {
// shall only be used, if a thread blocks for an important reason
@@ -107,6 +107,11 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
busyPause = milliseconds;
}
+ public void setMemPreReqisite(long freeBytes) {
+ // sets minimum required amount of memory for the job execution
+ memprereq = freeBytes;
+ }
+
public final String getShortDescription() {
return this.shortDescr;
}
@@ -125,6 +130,12 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
return this.busyCycles;
}
+ public long getOutOfMemoryCycles() {
+ // returns the total number of cycles where
+ // a job execution was omitted because of memory shortage
+ return this.outofmemoryCycles;
+ }
+
public final long getBlockTime() {
// returns the total time that this thread has been blocked so far
return this.blockPause;
@@ -206,8 +217,10 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
long innerpause;
long timestamp;
boolean isBusy;
+ Runtime rt = Runtime.getRuntime();
+
while (running) {
- try {
+ if (rt.freeMemory() > memprereq) try {
// do job
timestamp = System.currentTimeMillis();
isBusy = this.job();
@@ -224,6 +237,14 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
// if the exception is too bad it should call terminate()
this.jobExceptionHandler(e);
busyCycles++;
+ } else {
+ // omit job, not enough memory
+ // process scheduled pause
+ timestamp = System.currentTimeMillis();
+ ratz(this.idlePause);
+ idletime += System.currentTimeMillis() - timestamp;
+ outofmemoryCycles++;
+ if (rt.freeMemory() <= memprereq) System.gc(); // give next loop a chance
}
}
this.close();
diff --git a/source/de/anomic/server/serverInstantThread.java b/source/de/anomic/server/serverInstantThread.java
index 57a6fd86f..3329063c6 100644
--- a/source/de/anomic/server/serverInstantThread.java
+++ b/source/de/anomic/server/serverInstantThread.java
@@ -117,6 +117,7 @@ public final class serverInstantThread extends serverAbstractThread implements s
thread.setStartupSleep(startupDelay);
thread.setIdleSleep(-1);
thread.setBusySleep(-1);
+ thread.setMemPreReqisite(0);
thread.setLog(log);
thread.start();
return thread;
diff --git a/source/de/anomic/server/serverSwitch.java b/source/de/anomic/server/serverSwitch.java
index c695ae5be..24eabff19 100644
--- a/source/de/anomic/server/serverSwitch.java
+++ b/source/de/anomic/server/serverSwitch.java
@@ -73,9 +73,11 @@ public interface serverSwitch {
String threadShortDescription,
String threadLongDescription,
serverThread newThread,
- long startupDelay, long initialIdleSleep, long initialBusySleep);
+ long startupDelay,
+ long initialIdleSleep, long initialBusySleep,
+ long initialMemoryPreRequisite);
public serverThread getThread(String threadName);
- public void setThreadSleep(String threadName, long idleMillis, long busyMillis);
+ public void setThreadPerformance(String threadName, long idleMillis, long busyMillis, long memprereq);
public void terminateThread(String threadName, boolean waitFor);
public void terminateAllThreads(boolean waitFor);
public Iterator /*of serverThread-Names (String)*/ threadNames();
diff --git a/source/de/anomic/server/serverThread.java b/source/de/anomic/server/serverThread.java
index 6d2d9f8d6..91c4126c9 100644
--- a/source/de/anomic/server/serverThread.java
+++ b/source/de/anomic/server/serverThread.java
@@ -65,6 +65,9 @@ public interface serverThread {
public void setBusySleep(long milliseconds);
// sets a sleep time for pauses between two jobs if the job returns true (busy)
+ public void setMemPreReqisite(long freeBytes);
+ // sets minimum required amount of memory for the job execution
+
public String getShortDescription();
// returns short description string for online display
@@ -77,6 +80,10 @@ public interface serverThread {
public long getBusyCycles();
// returns the total number of cycles of job execution with busy-result
+ public long getOutOfMemoryCycles();
+ // returns the total number of cycles where
+ // a job execution was omitted because of memory shortage
+
public long getBlockTime();
// returns the total time that this thread has been blocked so far
diff --git a/source/yacy.java b/source/yacy.java
index 6632e210d..ba5e2f289 100644
--- a/source/yacy.java
+++ b/source/yacy.java
@@ -291,7 +291,7 @@ public final class yacy {
serverLog.logFailure("STARTUP", "Failed to start server. Probably port " + port + " already in use.");
} else {
// first start the server
- sb.deployThread("10_httpd", "HTTPD Server/Proxy", "the HTTPD, used as web server and proxy", server, 0, 0, 0);
+ sb.deployThread("10_httpd", "HTTPD Server/Proxy", "the HTTPD, used as web server and proxy", server, 0, 0, 0, 0);
//server.start();
// open the browser window
diff --git a/yacy.init b/yacy.init
index 1013bd542..7b4d2b46d 100644
--- a/yacy.init
+++ b/yacy.init
@@ -392,24 +392,36 @@ xpstopw=true
# the idlesleep is the pause that an proces sleeps if the last call to the
# process job was without execution of anything;
# the busysleep is the pause after a full job execution
+# the prereq-value is a memory pre-requisite: that much bytes must
+# be available/free in the heap; othervise the loop is not executed
+# and another idlesleep is performed
20_dhtdistribution_idlesleep=20000
20_dhtdistribution_busysleep=5000
+20_dhtdistribution_memprereq=5000000
30_peerping_idlesleep=120000
30_peerping_busysleep=120000
+30_peerping_memprereq=20000
40_peerseedcycle_idlesleep=1800000
40_peerseedcycle_busysleep=1200000
-50_localcrawl_idlesleep=10000
+40_peerseedcycle_memprereq=2000000
+50_localcrawl_idlesleep=1000
50_localcrawl_busysleep=200
-61_globalcrawltrigger_idlesleep=10000
+50_localcrawl_memprereq=4000000
+61_globalcrawltrigger_idlesleep=2000
61_globalcrawltrigger_busysleep=200
+61_globalcrawltrigger_memprereq=4000000
62_remotetriggeredcrawl_idlesleep=10000
62_remotetriggeredcrawl_busysleep=200
+62_remotetriggeredcrawl_memprereq=5000000
70_cachemanager_idlesleep=1000
70_cachemanager_busysleep=0
+70_cachemanager_memprereq=10000
80_indexing_idlesleep=1000
80_indexing_busysleep=0
+80_indexing_memprereq=2000000
90_cleanup_idlesleep=300000
90_cleanup_busysleep=300000
+90_cleanup_memprereq=10000
# multiprocessor-settings
# you may want to run time-consuming processes on several processors
@@ -422,17 +434,17 @@ xpstopw=true
# ram cache for database files
-# ram cache for indexCache.db
-ramCacheRWI = 16777216
+# ram cache for assortment cache cluster (for all 50 files)
+ramCacheRWI = 8388608
# ram cache for responseHeader.db
ramCacheHTTP = 1048576
# ram cache for urlHash.db
-ramCacheLURL = 8388608
+ramCacheLURL = 4194304
# ram cache for urlNotice.db
-ramCacheNURL = 1048576
+ramCacheNURL = 524288
# ram cache for urlErr.db
ramCacheEURL = 131072