From 859c6a88f5b1718030c49e403c11f47b20e3dcd1 Mon Sep 17 00:00:00 2001 From: theli Date: Fri, 27 Jan 2006 16:51:29 +0000 Subject: [PATCH] *) testing various thread pool eviction settings to avoid outOfMemory - Thread creation problem git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1467 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- .../de/anomic/plasma/plasmaCrawlLoader.java | 21 +++++++++-------- .../de/anomic/plasma/plasmaCrawlStacker.java | 1 + source/de/anomic/server/serverCore.java | 23 ++++++++++--------- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/source/de/anomic/plasma/plasmaCrawlLoader.java b/source/de/anomic/plasma/plasmaCrawlLoader.java index 4c77c1f62..4d199655c 100644 --- a/source/de/anomic/plasma/plasmaCrawlLoader.java +++ b/source/de/anomic/plasma/plasmaCrawlLoader.java @@ -63,7 +63,7 @@ public final class plasmaCrawlLoader extends Thread { private final CrawlerMessageQueue theQueue; private final CrawlerPool crawlwerPool; - private GenericObjectPool.Config cralwerPoolConfig = null; + private GenericObjectPool.Config crawlerPoolConfig = null; private final ThreadGroup theThreadGroup = new ThreadGroup("CrawlerThreads"); private boolean stopped = false; @@ -81,24 +81,25 @@ public final class plasmaCrawlLoader extends Thread { // configuring the crawler thread pool // implementation of session thread pool - this.cralwerPoolConfig = new GenericObjectPool.Config(); + this.crawlerPoolConfig = new GenericObjectPool.Config(); // The maximum number of active connections that can be allocated from pool at the same time, // 0 for no limit - this.cralwerPoolConfig.maxActive = Integer.parseInt(switchboard.getConfig("crawler.MaxActiveThreads","10")); + this.crawlerPoolConfig.maxActive = Integer.parseInt(switchboard.getConfig("crawler.MaxActiveThreads","10")); // The maximum number of idle connections connections in the pool // 0 = no limit. - this.cralwerPoolConfig.maxIdle = Integer.parseInt(switchboard.getConfig("crawler.MaxIdleThreads","7")); - this.cralwerPoolConfig.minIdle = Integer.parseInt(switchboard.getConfig("crawler.MinIdleThreads","5")); + this.crawlerPoolConfig.maxIdle = Integer.parseInt(switchboard.getConfig("crawler.MaxIdleThreads","7")); + this.crawlerPoolConfig.minIdle = Integer.parseInt(switchboard.getConfig("crawler.MinIdleThreads","5")); // block undefinitely - this.cralwerPoolConfig.maxWait = -1; + this.crawlerPoolConfig.maxWait = -1; // Action to take in case of an exhausted DBCP statement pool // 0 = fail, 1 = block, 2= grow - this.cralwerPoolConfig.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK; - this.cralwerPoolConfig.minEvictableIdleTimeMillis = 30000; + this.crawlerPoolConfig.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK; + this.crawlerPoolConfig.minEvictableIdleTimeMillis = 30000; + this.crawlerPoolConfig.timeBetweenEvictionRunsMillis = 30000; // config.testOnReturn = true; CrawlerFactory theFactory = new CrawlerFactory( @@ -107,14 +108,14 @@ public final class plasmaCrawlLoader extends Thread { cacheManager, log); - this.crawlwerPool = new CrawlerPool(theFactory,this.cralwerPoolConfig,this.theThreadGroup); + this.crawlwerPool = new CrawlerPool(theFactory,this.crawlerPoolConfig,this.theThreadGroup); // start the crawl loader this.start(); } public GenericObjectPool.Config getPoolConfig() { - return this.cralwerPoolConfig; + return this.crawlerPoolConfig; } public void setPoolConfig(GenericObjectPool.Config newConfig) { diff --git a/source/de/anomic/plasma/plasmaCrawlStacker.java b/source/de/anomic/plasma/plasmaCrawlStacker.java index a960d7a9c..061e14f52 100644 --- a/source/de/anomic/plasma/plasmaCrawlStacker.java +++ b/source/de/anomic/plasma/plasmaCrawlStacker.java @@ -105,6 +105,7 @@ public final class plasmaCrawlStacker { // 0 = fail, 1 = block, 2= grow this.theWorkerPoolConfig.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK; this.theWorkerPoolConfig.minEvictableIdleTimeMillis = 30000; + this.theWorkerPoolConfig.timeBetweenEvictionRunsMillis = 30000; // creating worker pool this.theWorkerPool = new WorkerPool(new WorkterFactory(this.theWorkerThreadGroup),this.theWorkerPoolConfig); diff --git a/source/de/anomic/server/serverCore.java b/source/de/anomic/server/serverCore.java index f509fd65c..367480848 100644 --- a/source/de/anomic/server/serverCore.java +++ b/source/de/anomic/server/serverCore.java @@ -112,7 +112,7 @@ public final class serverCore extends serverAbstractThread implements serverThre */ SessionPool theSessionPool; final ThreadGroup theSessionThreadGroup = new ThreadGroup("sessionThreadGroup"); - private Config cralwerPoolConfig = null; + private Config sessionPoolConfig = null; public ThreadGroup getSessionThreadGroup() { return this.theSessionThreadGroup; @@ -224,27 +224,28 @@ public final class serverCore extends serverAbstractThread implements serverThre this.log.logInfo("Initializing session pool ..."); // implementation of session thread pool - this.cralwerPoolConfig = new GenericObjectPool.Config(); + this.sessionPoolConfig = new GenericObjectPool.Config(); // The maximum number of active connections that can be allocated from pool at the same time, // 0 for no limit - this.cralwerPoolConfig.maxActive = Integer.valueOf(switchboard.getConfig("httpdMaxActiveSessions","150")).intValue(); + this.sessionPoolConfig.maxActive = Integer.valueOf(switchboard.getConfig("httpdMaxActiveSessions","150")).intValue(); // The maximum number of idle connections connections in the pool // 0 = no limit. - this.cralwerPoolConfig.maxIdle = Integer.valueOf(switchboard.getConfig("httpdMaxIdleSessions","75")).intValue(); - this.cralwerPoolConfig.minIdle = Integer.valueOf(switchboard.getConfig("httpdMinIdleSessions","5")).intValue(); + this.sessionPoolConfig.maxIdle = Integer.valueOf(switchboard.getConfig("httpdMaxIdleSessions","75")).intValue(); + this.sessionPoolConfig.minIdle = Integer.valueOf(switchboard.getConfig("httpdMinIdleSessions","5")).intValue(); // block undefinitely - this.cralwerPoolConfig.maxWait = timeout; + this.sessionPoolConfig.maxWait = timeout; // Action to take in case of an exhausted DBCP statement pool // 0 = fail, 1 = block, 2= grow - this.cralwerPoolConfig.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK; - this.cralwerPoolConfig.minEvictableIdleTimeMillis = this.thresholdSleep; - this.cralwerPoolConfig.testOnReturn = true; + this.sessionPoolConfig.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK; + this.sessionPoolConfig.minEvictableIdleTimeMillis = this.thresholdSleep; + this.sessionPoolConfig.timeBetweenEvictionRunsMillis = 30000; + this.sessionPoolConfig.testOnReturn = true; - this.theSessionPool = new SessionPool(new SessionFactory(this.theSessionThreadGroup),this.cralwerPoolConfig); + this.theSessionPool = new SessionPool(new SessionFactory(this.theSessionThreadGroup),this.sessionPoolConfig); } public void initPort(String thePort) throws IOException { @@ -348,7 +349,7 @@ public final class serverCore extends serverAbstractThread implements serverThre } public GenericObjectPool.Config getPoolConfig() { - return this.cralwerPoolConfig ; + return this.sessionPoolConfig ; } public void setPoolConfig(GenericObjectPool.Config newConfig) {