boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

extended fail-safe memory-managament. prevents too much allocation, too often GC and should help for the 100%CPU-bug

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@303 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 20 years ago
parent e3c92818db
commit 85075269a6
11 changed files with 117 additions and 54 deletions
Show Stats Download Patch File Download Diff File

4
htroot/Performance_p.html
Unescape View File

@ -21,10 +21,12 @@
<td class="small">Total<br>Cycles</td>
<td class="small">Idle<br>Cycles</td>
<td class="small">Busy<br>Cycles</td>
<td class="small">Short Mem<br>Cycles</td>
<td class="small">Sleep Time<br>per Cycle<br>(milliseconds)</td>
<td class="small">Exec Time<br>per Busy-Cycle<br>(milliseconds)</td>
<td class="small">Delay between<br>idle loops</td>
<td class="small">Delay between<br>busy loops</td>
<td class="small">Minimum of<br>Required Memory</td>
<td class="small">Full Description</td>
</tr>
#{table}#
@ -40,10 +42,12 @@
<td class="small" align="right">#[totalcycles]#</td>
<td class="small" align="right">#[idlecycles]#</td>
<td class="small" align="right">#[busycycles]#</td>
<td class="small" align="right">#[memscycles]#</td>
<td class="small" align="right">#[sleeppercycle]#</td>
<td class="small" align="right">#[execpercycle]#</td>
<td class="small" align="right"><input name="#[threadname]#_idlesleep" type="text" align="right" size="7" maxlength="7" value="#[idlesleep]#"> milliseconds</td>
<td class="small" align="right"><input name="#[threadname]#_busysleep" type="text" align="right" size="7" maxlength="7" value="#[busysleep]#"> milliseconds</td>
<td class="small" align="right"><input name="#[threadname]#_memprereq" type="text" align="right" size="8" maxlength="8" value="#[memprereq]#"> bytes</td>
<td class="small" align="left">#[longdescr]#</td>
</tr>
#{/table}#

20
htroot/Performance_p.java
Unescape View File

@ -81,11 +81,11 @@ public class Performance_p {
// set templates for latest news from the threads
long blocktime, sleeptime, exectime;
long idlesleep, busysleep;
long idlesleep, busysleep, memprereq;
int queuesize;
threads = switchboard.threadNames();
int c = 0;
long idleCycles, busyCycles;
long idleCycles, busyCycles, memshortageCycles;
while (threads.hasNext()) {
threadName = (String) threads.next();
thread = switchboard.getThread(threadName);
@ -102,37 +102,43 @@ public class Performance_p {
exectime = thread.getExecTime();
idleCycles = thread.getIdleCycles();
busyCycles = thread.getBusyCycles();
memshortageCycles = thread.getOutOfMemoryCycles();
prop.put("table_" + c + "_blocktime", blocktime / 1000);
prop.put("table_" + c + "_blockpercent", "" + (100 * blocktime / blocktime_total));
prop.put("table_" + c + "_sleeptime", sleeptime / 1000);
prop.put("table_" + c + "_sleeppercent", "" + (100 * sleeptime / sleeptime_total));
prop.put("table_" + c + "_exectime", exectime / 1000);
prop.put("table_" + c + "_execpercent", "" + (100 * exectime / exectime_total));
prop.put("table_" + c + "_totalcycles", "" + (idleCycles + busyCycles));
prop.put("table_" + c + "_totalcycles", "" + (idleCycles + busyCycles + memshortageCycles));
prop.put("table_" + c + "_idlecycles", "" + idleCycles);
prop.put("table_" + c + "_busycycles", "" + busyCycles);
prop.put("table_" + c + "_memscycles", "" + memshortageCycles);
prop.put("table_" + c + "_sleeppercycle", ((idleCycles + busyCycles) == 0) ? "-" : ("" + (sleeptime / (idleCycles + busyCycles))));
prop.put("table_" + c + "_execpercycle", (busyCycles == 0) ? "-" : ("" + (exectime / busyCycles)));
if ((post != null) && (post.containsKey("delaysubmit"))) {
// load with new values
idlesleep = Long.parseLong((String) post.get(threadName + "_idlesleep", "1"));
busysleep = Long.parseLong((String) post.get(threadName + "_busysleep", "1"));
idlesleep = Long.parseLong((String) post.get(threadName + "_idlesleep", "100"));
busysleep = Long.parseLong((String) post.get(threadName + "_busysleep", "1000"));
memprereq = Long.parseLong((String) post.get(threadName + "_memprereq", "0"));
// check values to prevent short-cut loops
if (idlesleep == 0) idlesleep = 1000;
// on-the-fly re-configuration
switchboard.setThreadSleep(threadName, idlesleep, busysleep);
switchboard.setThreadPerformance(threadName, idlesleep, busysleep, memprereq);
switchboard.setConfig(threadName + "_idlesleep", idlesleep);
switchboard.setConfig(threadName + "_busysleep", busysleep);
switchboard.setConfig(threadName + "_memprereq", memprereq);
} else {
// load with old values
idlesleep = Long.parseLong(switchboard.getConfig(threadName + "_idlesleep" , "1000"));
busysleep = Long.parseLong(switchboard.getConfig(threadName + "_busysleep", "1000"));
memprereq = Long.parseLong(switchboard.getConfig(threadName + "_memprereq", "1000"));
}
prop.put("table_" + c + "_idlesleep", idlesleep);
prop.put("table_" + c + "_busysleep", busysleep);
prop.put("table_" + c + "_memprereq", memprereq);
c++;
}

18
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -77,8 +77,9 @@ import java.util.StringTokenizer;
public class kelondroRecords {
// constants
private static int NUL = Integer.MIN_VALUE; // the meta value for the kelondroRecords' NUL abstraction
private static final int NUL = Integer.MIN_VALUE; // the meta value for the kelondroRecords' NUL abstraction
public static final long memBlock = 5000000; // do not fill cache further if the amount of available memory is less that this
// static seek pointers
private static long POS_MAGIC = 0; // 1 byte, byte: file type magic
private static long POS_BUSY = POS_MAGIC + 1; // 1 byte, byte: marker for synchronization
@ -352,10 +353,12 @@ public class kelondroRecords {
// check for space in cache
// should be only called within a synchronized(XcacheHeaders) environment
if (XcacheSize == 0) return;
while (XcacheHeaders.size() >= XcacheSize) {
Handle delkey;
while ((XcacheHeaders.size() >= XcacheSize) ||
((XcacheHeaders.size() > 0) && (Runtime.getRuntime().freeMemory() < memBlock))) {
// delete one entry
try {
Handle delkey = (Handle) XcacheScore.getMinObject(); // error (see below) here
delkey = (Handle) XcacheScore.getMinObject(); // error (see below) here
XcacheScore.deleteScore(delkey);
XcacheHeaders.remove(delkey);
} catch (NoSuchElementException e) {
@ -365,6 +368,7 @@ public class kelondroRecords {
this.XcacheScore = new kelondroMScoreCluster();
this.XcacheHeaders = new HashMap();
}
delkey = null;
}
}
@ -662,6 +666,7 @@ public class kelondroRecords {
synchronized (XcacheHeaders) {
// remember size to evaluate a cache size check need
int sizeBefore = XcacheHeaders.size();
//long memBefore = Runtime.getRuntime().freeMemory();
// generate cache entry
byte[][] cacheValue;
if (values == null) {
@ -676,10 +681,11 @@ public class kelondroRecords {
cacheNode.ohBytes = this.ohBytes;
cacheNode.ohHandle = this.ohHandle;
// store the cache entry
XcacheHeaders.put(cacheNode.handle, cacheNode);
XcacheScore.setScore(handle, (int) ((System.currentTimeMillis() - XcacheStartup) / 1000));
boolean newentry = XcacheHeaders.put(cacheNode.handle, cacheNode) == null;
XcacheScore.setScore(cacheNode.handle, (int) ((System.currentTimeMillis() - XcacheStartup) / 1000));
// delete the cache entry
cacheNode = null;
//System.out.println("kelondroRecords cache4" + filename + ": cache record size = " + (memBefore - Runtime.getRuntime().freeMemory()) + " bytes" + ((newentry) ? " new" : ""));
// check cache size
if (XcacheHeaders.size() > sizeBefore) checkCacheSpace();
//System.out.println("kelondroRecords cache4" + filename + ": " + XcacheHeaders.size() + " entries, " + XcacheSize + " allowed.");

20
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -148,7 +148,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// load slots
public static int crawlSlots = 20;
public static int crawlSlots = 12;
// couloured list management
public static TreeSet blueList = null;
@ -578,12 +578,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
//log.logDebug("CoreCrawl: queue is empty");
return false;
}
if (Runtime.getRuntime().freeMemory() < 2000000) {
log.logDebug("CoreCrawl: not enough memory available, dismissed (" +
"free=" + Runtime.getRuntime().freeMemory() + ")");
System.gc();
return false;
}
if (queueStack.size() >= crawlSlots) {
log.logDebug("CoreCrawl: too many processes in queue, dismissed (" +
"queueStack=" + queueStack.size() + ")");
@ -639,12 +633,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
//log.logDebug("LimitCrawl: queue is empty");
return false;
}
if (Runtime.getRuntime().freeMemory() < 2000000) {
log.logDebug("limitCrawlTrigger: not enough memory available, dismissed (" +
"free=" + Runtime.getRuntime().freeMemory() + ")");
System.gc();
return false;
}
// if the server is busy, we do crawling more slowly
//if (!(cacheManager.idle())) try {Thread.currentThread().sleep(2000);} catch (InterruptedException e) {}
@ -717,12 +705,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
//log.logDebug("GlobalCrawl: queue is empty");
return false;
}
if (Runtime.getRuntime().freeMemory() < 2000000) {
log.logDebug("remoteTriggeredCrawl: not enough memory available, dismissed (" +
"free=" + Runtime.getRuntime().freeMemory() + ")");
System.gc();
return false;
}
/*
if (queueStack.size() > 0) {
log.logDebug("GlobalCrawl: any processe is in queue, dismissed (" +

42
source/de/anomic/server/serverAbstractSwitch.java
Unescape View File

@ -254,31 +254,52 @@ public abstract class serverAbstractSwitch implements serverSwitch {
log.logInfo("Undeployed Action '" + action.getShortDescription() + "', (" + switchActions.size() + " actions registered)");
}
public void deployThread(String threadName, String threadShortDescription, String threadLongDescription, serverThread newThread, long startupDelay) {
public void deployThread(
String threadName,
String threadShortDescription,
String threadLongDescription,
serverThread newThread,
long startupDelay) {
deployThread(threadName, threadShortDescription, threadLongDescription,
newThread, startupDelay,
Long.parseLong(getConfig(threadName + "_idlesleep" , "novalue")),
Long.parseLong(getConfig(threadName + "_busysleep" , "novalue")));
Long.parseLong(getConfig(threadName + "_idlesleep" , "100")),
Long.parseLong(getConfig(threadName + "_busysleep" , "1000")),
Long.parseLong(getConfig(threadName + "_memprereq" , "1000000")));
}
public void deployThread(String threadName, String threadShortDescription, String threadLongDescription, serverThread newThread, long startupDelay, long initialIdleSleep, long initialBusySleep) {
public void deployThread(
String threadName,
String threadShortDescription,
String threadLongDescription,
serverThread newThread,
long startupDelay,
long initialIdleSleep,
long initialBusySleep,
long initialMemoryPreRequisite) {
if (newThread.isAlive()) throw new RuntimeException("undeployed threads must not live; they are started as part of the deployment");
newThread.setStartupSleep(startupDelay);
long sleep;
long x;
try {
sleep = Long.parseLong(getConfig(threadName + "_idlesleep" , "novalue"));
newThread.setIdleSleep(sleep);
x = Long.parseLong(getConfig(threadName + "_idlesleep" , "novalue"));
newThread.setIdleSleep(x);
} catch (NumberFormatException e) {
newThread.setIdleSleep(initialIdleSleep);
setConfig(threadName + "_idlesleep", initialIdleSleep);
}
try {
sleep = Long.parseLong(getConfig(threadName + "_busysleep" , "novalue"));
newThread.setBusySleep(sleep);
x = Long.parseLong(getConfig(threadName + "_busysleep" , "novalue"));
newThread.setBusySleep(x);
} catch (NumberFormatException e) {
newThread.setBusySleep(initialBusySleep);
setConfig(threadName + "_busysleep", initialBusySleep);
}
try {
x = Long.parseLong(getConfig(threadName + "_memprereq" , "novalue"));
newThread.setMemPreReqisite(x);
} catch (NumberFormatException e) {
newThread.setMemPreReqisite(initialMemoryPreRequisite);
setConfig(threadName + "_memprereq", initialMemoryPreRequisite);
}
newThread.setLog(log);
newThread.setDescription(threadShortDescription, threadLongDescription);
workerThreads.put(threadName, newThread);
@ -290,11 +311,12 @@ public abstract class serverAbstractSwitch implements serverSwitch {
return (serverThread) workerThreads.get(threadName);
}
public void setThreadSleep(String threadName, long idleMillis, long busyMillis) {
public void setThreadPerformance(String threadName, long idleMillis, long busyMillis, long memprereqBytes) {
serverThread thread = (serverThread) workerThreads.get(threadName);
if (thread != null) {
thread.setIdleSleep(idleMillis);
thread.setBusySleep(busyMillis);
thread.setMemPreReqisite(memprereqBytes);
}
}

27
source/de/anomic/server/serverAbstractThread.java
Unescape View File

@ -57,10 +57,10 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
private long startup = 0, idlePause = 0, busyPause = 0, blockPause = 0;
private boolean running = true;
private serverLog log = null;
private long idletime = 0, busytime = 0;
private long idletime = 0, busytime = 0, memprereq = 0;
private String shortDescr = "", longDescr = "";
private long threadBlockTimestamp = System.currentTimeMillis();
private long idleCycles = 0, busyCycles = 0;
private long idleCycles = 0, busyCycles = 0, outofmemoryCycles = 0;
protected final void announceThreadBlockApply() {
// shall only be used, if a thread blocks for an important reason
@ -107,6 +107,11 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
busyPause = milliseconds;
}
public void setMemPreReqisite(long freeBytes) {
// sets minimum required amount of memory for the job execution
memprereq = freeBytes;
}
public final String getShortDescription() {
return this.shortDescr;
}
@ -125,6 +130,12 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
return this.busyCycles;
}
public long getOutOfMemoryCycles() {
// returns the total number of cycles where
// a job execution was omitted because of memory shortage
return this.outofmemoryCycles;
}
public final long getBlockTime() {
// returns the total time that this thread has been blocked so far
return this.blockPause;
@ -206,8 +217,10 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
long innerpause;
long timestamp;
boolean isBusy;
Runtime rt = Runtime.getRuntime();
while (running) {
try {
if (rt.freeMemory() > memprereq) try {
// do job
timestamp = System.currentTimeMillis();
isBusy = this.job();
@ -224,6 +237,14 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
// if the exception is too bad it should call terminate()
this.jobExceptionHandler(e);
busyCycles++;
} else {
// omit job, not enough memory
// process scheduled pause
timestamp = System.currentTimeMillis();
ratz(this.idlePause);
idletime += System.currentTimeMillis() - timestamp;
outofmemoryCycles++;
if (rt.freeMemory() <= memprereq) System.gc(); // give next loop a chance
}
}
this.close();

1
source/de/anomic/server/serverInstantThread.java
Unescape View File

@ -117,6 +117,7 @@ public final class serverInstantThread extends serverAbstractThread implements s
thread.setStartupSleep(startupDelay);
thread.setIdleSleep(-1);
thread.setBusySleep(-1);
thread.setMemPreReqisite(0);
thread.setLog(log);
thread.start();
return thread;

6
source/de/anomic/server/serverSwitch.java
Unescape View File

@ -73,9 +73,11 @@ public interface serverSwitch {
String threadShortDescription,
String threadLongDescription,
serverThread newThread,
long startupDelay, long initialIdleSleep, long initialBusySleep);
long startupDelay,
long initialIdleSleep, long initialBusySleep,
long initialMemoryPreRequisite);
public serverThread getThread(String threadName);
public void setThreadSleep(String threadName, long idleMillis, long busyMillis);
public void setThreadPerformance(String threadName, long idleMillis, long busyMillis, long memprereq);
public void terminateThread(String threadName, boolean waitFor);
public void terminateAllThreads(boolean waitFor);
public Iterator /*of serverThread-Names (String)*/ threadNames();

7
source/de/anomic/server/serverThread.java
Unescape View File

@ -65,6 +65,9 @@ public interface serverThread {
public void setBusySleep(long milliseconds);
// sets a sleep time for pauses between two jobs if the job returns true (busy)
public void setMemPreReqisite(long freeBytes);
// sets minimum required amount of memory for the job execution
public String getShortDescription();
// returns short description string for online display
@ -77,6 +80,10 @@ public interface serverThread {
public long getBusyCycles();
// returns the total number of cycles of job execution with busy-result
public long getOutOfMemoryCycles();
// returns the total number of cycles where
// a job execution was omitted because of memory shortage
public long getBlockTime();
// returns the total time that this thread has been blocked so far

2
source/yacy.java
Unescape View File

@ -291,7 +291,7 @@ public final class yacy {
serverLog.logFailure("STARTUP", "Failed to start server. Probably port " + port + " already in use.");
} else {
// first start the server
sb.deployThread("10_httpd", "HTTPD Server/Proxy", "the HTTPD, used as web server and proxy", server, 0, 0, 0);
sb.deployThread("10_httpd", "HTTPD Server/Proxy", "the HTTPD, used as web server and proxy", server, 0, 0, 0, 0);
//server.start();
// open the browser window

24
yacy.init
Unescape View File

@ -392,24 +392,36 @@ xpstopw=true
# the idlesleep is the pause that an proces sleeps if the last call to the
# process job was without execution of anything;
# the busysleep is the pause after a full job execution
# the prereq-value is a memory pre-requisite: that much bytes must
# be available/free in the heap; othervise the loop is not executed
# and another idlesleep is performed
20_dhtdistribution_idlesleep=20000
20_dhtdistribution_busysleep=5000
20_dhtdistribution_memprereq=5000000
30_peerping_idlesleep=120000
30_peerping_busysleep=120000
30_peerping_memprereq=20000
40_peerseedcycle_idlesleep=1800000
40_peerseedcycle_busysleep=1200000
50_localcrawl_idlesleep=10000
40_peerseedcycle_memprereq=2000000
50_localcrawl_idlesleep=1000
50_localcrawl_busysleep=200
61_globalcrawltrigger_idlesleep=10000
50_localcrawl_memprereq=4000000
61_globalcrawltrigger_idlesleep=2000
61_globalcrawltrigger_busysleep=200
61_globalcrawltrigger_memprereq=4000000
62_remotetriggeredcrawl_idlesleep=10000
62_remotetriggeredcrawl_busysleep=200
62_remotetriggeredcrawl_memprereq=5000000
70_cachemanager_idlesleep=1000
70_cachemanager_busysleep=0
70_cachemanager_memprereq=10000
80_indexing_idlesleep=1000
80_indexing_busysleep=0
80_indexing_memprereq=2000000
90_cleanup_idlesleep=300000
90_cleanup_busysleep=300000
90_cleanup_memprereq=10000
# multiprocessor-settings
# you may want to run time-consuming processes on several processors
@ -422,17 +434,17 @@ xpstopw=true
# ram cache for database files
# ram cache for indexCache.db
ramCacheRWI = 16777216
# ram cache for assortment cache cluster (for all 50 files)
ramCacheRWI = 8388608
# ram cache for responseHeader.db
ramCacheHTTP = 1048576
# ram cache for urlHash.db
ramCacheLURL = 8388608
ramCacheLURL = 4194304
# ram cache for urlNotice.db
ramCacheNURL = 1048576
ramCacheNURL = 524288
# ram cache for urlErr.db
ramCacheEURL = 131072

Write Preview
Loading…
Cancel
Save