boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

*) testing various thread pool eviction settings to avoid outOfMemory - Thread creation problem

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1467 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
theli 19 years ago
parent 6d73c6b481
commit 859c6a88f5
3 changed files with 24 additions and 21 deletions
Show Stats Download Patch File Download Diff File

21
source/de/anomic/plasma/plasmaCrawlLoader.java
Unescape View File

@ -63,7 +63,7 @@ public final class plasmaCrawlLoader extends Thread {
private final CrawlerMessageQueue theQueue; private final CrawlerMessageQueue theQueue;
private final CrawlerPool crawlwerPool; private final CrawlerPool crawlwerPool;
private GenericObjectPool.Config cralwerPoolConfig = null; private GenericObjectPool.Config crawlerPoolConfig = null;
private final ThreadGroup theThreadGroup = new ThreadGroup("CrawlerThreads"); private final ThreadGroup theThreadGroup = new ThreadGroup("CrawlerThreads");
private boolean stopped = false; private boolean stopped = false;
@ -81,24 +81,25 @@ public final class plasmaCrawlLoader extends Thread {
// configuring the crawler thread pool // configuring the crawler thread pool
// implementation of session thread pool // implementation of session thread pool
this.cralwerPoolConfig = new GenericObjectPool.Config(); this.crawlerPoolConfig = new GenericObjectPool.Config();
// The maximum number of active connections that can be allocated from pool at the same time, // The maximum number of active connections that can be allocated from pool at the same time,
// 0 for no limit // 0 for no limit
this.cralwerPoolConfig.maxActive = Integer.parseInt(switchboard.getConfig("crawler.MaxActiveThreads","10")); this.crawlerPoolConfig.maxActive = Integer.parseInt(switchboard.getConfig("crawler.MaxActiveThreads","10"));
// The maximum number of idle connections connections in the pool // The maximum number of idle connections connections in the pool
// 0 = no limit. // 0 = no limit.
this.cralwerPoolConfig.maxIdle = Integer.parseInt(switchboard.getConfig("crawler.MaxIdleThreads","7")); this.crawlerPoolConfig.maxIdle = Integer.parseInt(switchboard.getConfig("crawler.MaxIdleThreads","7"));
this.cralwerPoolConfig.minIdle = Integer.parseInt(switchboard.getConfig("crawler.MinIdleThreads","5")); this.crawlerPoolConfig.minIdle = Integer.parseInt(switchboard.getConfig("crawler.MinIdleThreads","5"));
// block undefinitely // block undefinitely
this.cralwerPoolConfig.maxWait = -1; this.crawlerPoolConfig.maxWait = -1;
// Action to take in case of an exhausted DBCP statement pool // Action to take in case of an exhausted DBCP statement pool
// 0 = fail, 1 = block, 2= grow // 0 = fail, 1 = block, 2= grow
this.cralwerPoolConfig.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK; this.crawlerPoolConfig.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK;
this.cralwerPoolConfig.minEvictableIdleTimeMillis = 30000; this.crawlerPoolConfig.minEvictableIdleTimeMillis = 30000;
this.crawlerPoolConfig.timeBetweenEvictionRunsMillis = 30000;
// config.testOnReturn = true; // config.testOnReturn = true;
CrawlerFactory theFactory = new CrawlerFactory( CrawlerFactory theFactory = new CrawlerFactory(
@ -107,14 +108,14 @@ public final class plasmaCrawlLoader extends Thread {
cacheManager, cacheManager,
log); log);
this.crawlwerPool = new CrawlerPool(theFactory,this.cralwerPoolConfig,this.theThreadGroup); this.crawlwerPool = new CrawlerPool(theFactory,this.crawlerPoolConfig,this.theThreadGroup);
// start the crawl loader // start the crawl loader
this.start(); this.start();
} }
public GenericObjectPool.Config getPoolConfig() { public GenericObjectPool.Config getPoolConfig() {
return this.cralwerPoolConfig; return this.crawlerPoolConfig;
} }
public void setPoolConfig(GenericObjectPool.Config newConfig) { public void setPoolConfig(GenericObjectPool.Config newConfig) {

1
source/de/anomic/plasma/plasmaCrawlStacker.java
Unescape View File

@ -105,6 +105,7 @@ public final class plasmaCrawlStacker {
// 0 = fail, 1 = block, 2= grow // 0 = fail, 1 = block, 2= grow
this.theWorkerPoolConfig.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK; this.theWorkerPoolConfig.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK;
this.theWorkerPoolConfig.minEvictableIdleTimeMillis = 30000; this.theWorkerPoolConfig.minEvictableIdleTimeMillis = 30000;
this.theWorkerPoolConfig.timeBetweenEvictionRunsMillis = 30000;
// creating worker pool // creating worker pool
this.theWorkerPool = new WorkerPool(new WorkterFactory(this.theWorkerThreadGroup),this.theWorkerPoolConfig); this.theWorkerPool = new WorkerPool(new WorkterFactory(this.theWorkerThreadGroup),this.theWorkerPoolConfig);

23
source/de/anomic/server/serverCore.java
Unescape View File

@ -112,7 +112,7 @@ public final class serverCore extends serverAbstractThread implements serverThre
*/ */
SessionPool theSessionPool; SessionPool theSessionPool;
final ThreadGroup theSessionThreadGroup = new ThreadGroup("sessionThreadGroup"); final ThreadGroup theSessionThreadGroup = new ThreadGroup("sessionThreadGroup");
private Config cralwerPoolConfig = null; private Config sessionPoolConfig = null;
public ThreadGroup getSessionThreadGroup() { public ThreadGroup getSessionThreadGroup() {
return this.theSessionThreadGroup; return this.theSessionThreadGroup;
@ -224,27 +224,28 @@ public final class serverCore extends serverAbstractThread implements serverThre
this.log.logInfo("Initializing session pool ..."); this.log.logInfo("Initializing session pool ...");
// implementation of session thread pool // implementation of session thread pool
this.cralwerPoolConfig = new GenericObjectPool.Config(); this.sessionPoolConfig = new GenericObjectPool.Config();
// The maximum number of active connections that can be allocated from pool at the same time, // The maximum number of active connections that can be allocated from pool at the same time,
// 0 for no limit // 0 for no limit
this.cralwerPoolConfig.maxActive = Integer.valueOf(switchboard.getConfig("httpdMaxActiveSessions","150")).intValue(); this.sessionPoolConfig.maxActive = Integer.valueOf(switchboard.getConfig("httpdMaxActiveSessions","150")).intValue();
// The maximum number of idle connections connections in the pool // The maximum number of idle connections connections in the pool
// 0 = no limit. // 0 = no limit.
this.cralwerPoolConfig.maxIdle = Integer.valueOf(switchboard.getConfig("httpdMaxIdleSessions","75")).intValue(); this.sessionPoolConfig.maxIdle = Integer.valueOf(switchboard.getConfig("httpdMaxIdleSessions","75")).intValue();
this.cralwerPoolConfig.minIdle = Integer.valueOf(switchboard.getConfig("httpdMinIdleSessions","5")).intValue(); this.sessionPoolConfig.minIdle = Integer.valueOf(switchboard.getConfig("httpdMinIdleSessions","5")).intValue();
// block undefinitely // block undefinitely
this.cralwerPoolConfig.maxWait = timeout; this.sessionPoolConfig.maxWait = timeout;
// Action to take in case of an exhausted DBCP statement pool // Action to take in case of an exhausted DBCP statement pool
// 0 = fail, 1 = block, 2= grow // 0 = fail, 1 = block, 2= grow
this.cralwerPoolConfig.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK; this.sessionPoolConfig.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK;
this.cralwerPoolConfig.minEvictableIdleTimeMillis = this.thresholdSleep; this.sessionPoolConfig.minEvictableIdleTimeMillis = this.thresholdSleep;
this.cralwerPoolConfig.testOnReturn = true; this.sessionPoolConfig.timeBetweenEvictionRunsMillis = 30000;
this.sessionPoolConfig.testOnReturn = true;
this.theSessionPool = new SessionPool(new SessionFactory(this.theSessionThreadGroup),this.cralwerPoolConfig); this.theSessionPool = new SessionPool(new SessionFactory(this.theSessionThreadGroup),this.sessionPoolConfig);
} }
public void initPort(String thePort) throws IOException { public void initPort(String thePort) throws IOException {
@ -348,7 +349,7 @@ public final class serverCore extends serverAbstractThread implements serverThre
} }
public GenericObjectPool.Config getPoolConfig() { public GenericObjectPool.Config getPoolConfig() {
return this.cralwerPoolConfig ; return this.sessionPoolConfig ;
} }
public void setPoolConfig(GenericObjectPool.Config newConfig) { public void setPoolConfig(GenericObjectPool.Config newConfig) {

Write Preview
Loading…
Cancel
Save