diff --git a/htroot/PerformanceQueues_p.java b/htroot/PerformanceQueues_p.java index 43622e60a..58edf2e9b 100644 --- a/htroot/PerformanceQueues_p.java +++ b/htroot/PerformanceQueues_p.java @@ -202,8 +202,11 @@ public class PerformanceQueues_p { if (idlesleep < 1000) idlesleep = 1000; if (threadName.equals("10_httpd")) { idlesleep = 0; busysleep = 0; memprereq = 0; } - onTheFlyReconfiguration(sb, threadName, idlesleep, busysleep, memprereq); - } if (setProfile) { + sb.setThreadPerformance(threadName, idlesleep, busysleep, memprereq); + idlesleep = sb.getConfigLong(threadName + "_idlesleep", idlesleep); + busysleep = sb.getConfigLong(threadName + "_busysleep", busysleep); + } + if (setProfile) { if (threadName.equals(SwitchboardConstants.PEER_PING) || threadName.equals(SwitchboardConstants.SEED_UPLOAD) || threadName.equals(SwitchboardConstants.CLEANUP) @@ -223,8 +226,7 @@ public class PerformanceQueues_p { if (idlesleep < 1000) idlesleep = 1000; if (threadName.equals("10_httpd")) { idlesleep = 0; busysleep = 0; memprereq = 0; } //if (threadName.equals(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL) && (busysleep < 50)) busysleep = 50; - - onTheFlyReconfiguration(sb, threadName, idlesleep, busysleep, memprereq); + sb.setThreadPerformance(threadName, idlesleep, busysleep, memprereq); } } prop.put("table_" + c + "_idlesleep", idlesleep); @@ -368,22 +370,6 @@ public class PerformanceQueues_p { // return rewrite values for templates return prop; } - - /** - * @param switchboard - * @param threadName - * @param idlesleep - * @param busysleep - * @param memprereq - */ - private static void onTheFlyReconfiguration(final Switchboard switchboard, final String threadName, final long idlesleep, - final long busysleep, final long memprereq) { - // on-the-fly re-configuration - switchboard.setThreadPerformance(threadName, idlesleep, busysleep, memprereq); - switchboard.setConfig(threadName + "_idlesleep", idlesleep); - switchboard.setConfig(threadName + "_busysleep", busysleep); - switchboard.setConfig(threadName + "_memprereq", memprereq); - } private static String d(final String a, final String b) { return (a == null) ? b : a; diff --git a/source/de/anomic/data/bookmarksDB.java b/source/de/anomic/data/bookmarksDB.java index 8e5f42696..a2ecd31b7 100644 --- a/source/de/anomic/data/bookmarksDB.java +++ b/source/de/anomic/data/bookmarksDB.java @@ -123,7 +123,7 @@ public class bookmarksDB { // autoReCrawl Switchboard sb = Switchboard.getSwitchboard(); - this.autoReCrawl = new InstantBusyThread(this, "autoReCrawl", null, null); + this.autoReCrawl = new InstantBusyThread(this, "autoReCrawl", null, null, Long.MIN_VALUE, Long.MAX_VALUE, Long.MIN_VALUE, Long.MAX_VALUE); long sleepTime = Long.parseLong(sb.getConfig("autoReCrawl_idlesleep" , SLEEP_TIME)); sb.deployThread("autoReCrawl", "autoReCrawl Scheduler", "simple scheduler for automatic re-crawls of bookmarked urls", null, autoReCrawl, 120000, sleepTime, sleepTime, Long.parseLong(sb.getConfig("autoReCrawl_memprereq" , "-1")) diff --git a/source/de/anomic/search/Switchboard.java b/source/de/anomic/search/Switchboard.java index f853b82dc..24ebd943b 100644 --- a/source/de/anomic/search/Switchboard.java +++ b/source/de/anomic/search/Switchboard.java @@ -585,21 +585,69 @@ public final class Switchboard extends serverSwitch { MemoryControl.gc(10000, "plasmaSwitchboard, help for profiler"); // help for profiler - thq deployThread(SwitchboardConstants.CLEANUP, "Cleanup", "simple cleaning process for monitoring information", null, - new InstantBusyThread(this, SwitchboardConstants.CLEANUP_METHOD_START, SwitchboardConstants.CLEANUP_METHOD_JOBCOUNT, SwitchboardConstants.CLEANUP_METHOD_FREEMEM), 600000); // all 5 Minutes, wait 10 minutes until first run + new InstantBusyThread( + this, + SwitchboardConstants.CLEANUP_METHOD_START, + SwitchboardConstants.CLEANUP_METHOD_JOBCOUNT, + SwitchboardConstants.CLEANUP_METHOD_FREEMEM, + 60000, Long.MAX_VALUE, 10000, Long.MAX_VALUE), + 600000); // all 5 Minutes, wait 10 minutes until first run deployThread(SwitchboardConstants.SURROGATES, "Surrogates", "A thread that polls the SURROGATES path and puts all Documents in one surroagte file into the indexing queue.", null, - new InstantBusyThread(this, SwitchboardConstants.SURROGATES_METHOD_START, SwitchboardConstants.SURROGATES_METHOD_JOBCOUNT, SwitchboardConstants.SURROGATES_METHOD_FREEMEM), 10000); + new InstantBusyThread( + this, + SwitchboardConstants.SURROGATES_METHOD_START, + SwitchboardConstants.SURROGATES_METHOD_JOBCOUNT, + SwitchboardConstants.SURROGATES_METHOD_FREEMEM, + 20000, Long.MAX_VALUE, 0, Long.MAX_VALUE), + 10000); deployThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL, "Remote Crawl Job", "thread that performes a single crawl/indexing step triggered by a remote peer", null, - new InstantBusyThread(crawlQueues, SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_START, SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_JOBCOUNT, SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_FREEMEM), 30000); + new InstantBusyThread( + crawlQueues, + SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_START, + SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_JOBCOUNT, + SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_FREEMEM, + 0, Long.MAX_VALUE, 0, Long.MAX_VALUE), + 30000); deployThread(SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER, "Remote Crawl URL Loader", "thread that loads remote crawl lists from other peers", "", - new InstantBusyThread(crawlQueues, SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_START, SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_JOBCOUNT, SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_FREEMEM), 30000); // error here? + new InstantBusyThread( + crawlQueues, + SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_START, + SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_JOBCOUNT, + SwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_FREEMEM, + 10000, Long.MAX_VALUE, 10000, Long.MAX_VALUE), + 30000); // error here? deployThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL, "Local Crawl", "thread that performes a single crawl step from the local crawl queue", "/IndexCreateWWWLocalQueue_p.html", - new InstantBusyThread(crawlQueues, SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_START, SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_JOBCOUNT, SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_FREEMEM), 10000); + new InstantBusyThread( + crawlQueues, + SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_START, + SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_JOBCOUNT, + SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_FREEMEM, + 0, Long.MAX_VALUE, 0, Long.MAX_VALUE), + 10000); deployThread(SwitchboardConstants.SEED_UPLOAD, "Seed-List Upload", "task that a principal peer performes to generate and upload a seed-list to a ftp account", null, - new InstantBusyThread(yc, SwitchboardConstants.SEED_UPLOAD_METHOD_START, SwitchboardConstants.SEED_UPLOAD_METHOD_JOBCOUNT, SwitchboardConstants.SEED_UPLOAD_METHOD_FREEMEM), 180000); + new InstantBusyThread( + yc, + SwitchboardConstants.SEED_UPLOAD_METHOD_START, + SwitchboardConstants.SEED_UPLOAD_METHOD_JOBCOUNT, + SwitchboardConstants.SEED_UPLOAD_METHOD_FREEMEM, + 600000, Long.MAX_VALUE, 300000, Long.MAX_VALUE), + 180000); deployThread(SwitchboardConstants.PEER_PING, "YaCy Core", "this is the p2p-control and peer-ping task", null, - new InstantBusyThread(yc, SwitchboardConstants.PEER_PING_METHOD_START, SwitchboardConstants.PEER_PING_METHOD_JOBCOUNT, SwitchboardConstants.PEER_PING_METHOD_FREEMEM), 2000); + new InstantBusyThread( + yc, + SwitchboardConstants.PEER_PING_METHOD_START, + SwitchboardConstants.PEER_PING_METHOD_JOBCOUNT, + SwitchboardConstants.PEER_PING_METHOD_FREEMEM, + 60000, Long.MAX_VALUE, 60000, Long.MAX_VALUE), + 2000); deployThread(SwitchboardConstants.INDEX_DIST, "DHT Distribution", "selection, transfer and deletion of index entries that are not searched on your peer, but on others", null, - new InstantBusyThread(this, SwitchboardConstants.INDEX_DIST_METHOD_START, SwitchboardConstants.INDEX_DIST_METHOD_JOBCOUNT, SwitchboardConstants.INDEX_DIST_METHOD_FREEMEM), 5000, + new InstantBusyThread( + this, + SwitchboardConstants.INDEX_DIST_METHOD_START, + SwitchboardConstants.INDEX_DIST_METHOD_JOBCOUNT, + SwitchboardConstants.INDEX_DIST_METHOD_FREEMEM, + 10000, Long.MAX_VALUE, 1000, Long.MAX_VALUE), + 5000, Long.parseLong(getConfig(SwitchboardConstants.INDEX_DIST_IDLESLEEP , "5000")), Long.parseLong(getConfig(SwitchboardConstants.INDEX_DIST_BUSYSLEEP , "0")), Long.parseLong(getConfig(SwitchboardConstants.INDEX_DIST_MEMPREREQ , "1000000"))); diff --git a/source/de/anomic/server/serverCore.java b/source/de/anomic/server/serverCore.java index 69aad202f..c3b910452 100644 --- a/source/de/anomic/server/serverCore.java +++ b/source/de/anomic/server/serverCore.java @@ -181,6 +181,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread { final serverSwitch switchboard, final int commandMaxLength ) { + super(Long.MIN_VALUE, Long.MAX_VALUE, Long.MIN_VALUE, Long.MAX_VALUE); this.timeout = timeout; this.commandMaxLength = commandMaxLength; diff --git a/source/de/anomic/server/serverSwitch.java b/source/de/anomic/server/serverSwitch.java index a5eb9209f..f1863d4bd 100644 --- a/source/de/anomic/server/serverSwitch.java +++ b/source/de/anomic/server/serverSwitch.java @@ -385,8 +385,9 @@ public class serverSwitch { public void setThreadPerformance(final String threadName, final long idleMillis, final long busyMillis, final long memprereqBytes) { final BusyThread thread = workerThreads.get(threadName); if (thread != null) { - thread.setIdleSleep(idleMillis); - thread.setBusySleep(busyMillis); + setConfig(threadName + "_idlesleep", thread.setIdleSleep(idleMillis)); + setConfig(threadName + "_busysleep", thread.setBusySleep(busyMillis)); + setConfig(threadName + "_memprereq", memprereqBytes); thread.setMemPreReqisite(memprereqBytes); } } diff --git a/source/net/yacy/kelondro/workflow/AbstractBusyThread.java b/source/net/yacy/kelondro/workflow/AbstractBusyThread.java index 8b2a813ff..9faec36fb 100644 --- a/source/net/yacy/kelondro/workflow/AbstractBusyThread.java +++ b/source/net/yacy/kelondro/workflow/AbstractBusyThread.java @@ -40,6 +40,20 @@ public abstract class AbstractBusyThread extends AbstractThread implements BusyT private boolean intermissionObedient = true; private final Object syncObject = new Object(); + private long maxIdleSleep = Long.MAX_VALUE, minIdleSleep = Long.MIN_VALUE; + private long maxBusySleep = Long.MAX_VALUE, minBusySleep = Long.MIN_VALUE; + + public AbstractBusyThread( + long minIdleSleep, + long maxIdleSleep, + long minBusySleep, + long maxBusySleep) { + this.minIdleSleep = minIdleSleep; + this.maxIdleSleep = maxIdleSleep; + this.minBusySleep = minBusySleep; + this.maxBusySleep = maxBusySleep; + } + protected final void announceMoreSleepTime(final long millis) { this.idletime += millis; } @@ -51,14 +65,14 @@ public abstract class AbstractBusyThread extends AbstractThread implements BusyT public final long setIdleSleep(final long milliseconds) { // sets a sleep time for pauses between two jobs - idlePause = milliseconds; - return milliseconds; + idlePause = Math.min(this.maxIdleSleep, Math.max(this.minIdleSleep, milliseconds)); + return idlePause; } public final long setBusySleep(final long milliseconds) { // sets a sleep time for pauses between two jobs - busyPause = milliseconds; - return milliseconds; + busyPause = Math.min(this.maxBusySleep, Math.max(this.minBusySleep, milliseconds)); + return busyPause; } public void setMemPreReqisite(final long freeBytes) { diff --git a/source/net/yacy/kelondro/workflow/InstantBusyThread.java b/source/net/yacy/kelondro/workflow/InstantBusyThread.java index f6532b1a9..979194b02 100644 --- a/source/net/yacy/kelondro/workflow/InstantBusyThread.java +++ b/source/net/yacy/kelondro/workflow/InstantBusyThread.java @@ -38,7 +38,17 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT public static int instantThreadCounter = 0; public static final TreeMap jobs = new TreeMap(); - public InstantBusyThread(final Object env, final String jobExec, final String jobCount, final String freemem) { + public InstantBusyThread( + final Object env, + final String jobExec, + final String jobCount, + final String freemem, + long minIdleSleep, + long maxIdleSleep, + long minBusySleep, + long maxBusySleep) { + super(minIdleSleep, maxIdleSleep, minBusySleep, maxBusySleep); + // jobExec is the name of a method of the object 'env' that executes the one-step-run // jobCount is the name of a method that returns the size of the job // freemem is the name of a method that tries to free memory and returns void @@ -146,7 +156,8 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT public static BusyThread oneTimeJob(final Object env, final String jobExec, final Log log, final long startupDelay) { // start the job and execute it once as background process - final BusyThread thread = new InstantBusyThread(env, jobExec, null, null); + final BusyThread thread = new InstantBusyThread( + env, jobExec, null, null, Long.MIN_VALUE, Long.MAX_VALUE, Long.MIN_VALUE, Long.MAX_VALUE); thread.setStartupSleep(startupDelay); thread.setIdleSleep(-1); thread.setBusySleep(-1);