*) Making CrawlStacker pool configurable via GUI and config file

See: http://www.yacy-forum.de/viewtopic.php?t=1448

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1087 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/PerformanceQueues_p.java

@@ -224,6 +224,25 @@ public class PerformanceQueues_p {
             switchboard.setConfig("httpdMaxActiveSessions",maxActive);
             switchboard.setConfig("httpdMaxIdleSessions",maxIdle);
             switchboard.setConfig("httpdMinIdleSessions",minIdle);
+            
+            /*
+             * Configuring the crawlStacker pool
+             */
+            GenericObjectPool.Config stackerPoolConfig = switchboard.sbStackCrawlThread.getPoolConfig();
+            maxActive = Integer.parseInt(post.get("CrawlStacker Session Pool_maxActive","10"));
+            maxIdle = Integer.parseInt(post.get("CrawlStacker Session Pool_maxIdle","10"));
+            minIdle = Integer.parseInt(post.get("CrawlStacker Session Pool_minIdle","5"));
+            
+            stackerPoolConfig.minIdle = (minIdle > maxIdle) ? maxIdle/2 : minIdle;
+            stackerPoolConfig.maxIdle = (maxIdle > maxActive) ? maxActive/2 : maxIdle;
+            stackerPoolConfig.maxActive = maxActive;   
+            
+            switchboard.sbStackCrawlThread.setPoolConfig(stackerPoolConfig);     
+            
+            // storing the new values into configfile
+            switchboard.setConfig("stacker.MaxActiveThreads",maxActive);
+            switchboard.setConfig("stacker.MaxIdleThreads",maxIdle);
+            switchboard.setConfig("stacker.MinIdleThreads",minIdle);
         }        
         
         if ((post != null) && (post.containsKey("proxyControlSubmit"))) {
@@ -266,9 +285,14 @@ public class PerformanceQueues_p {
         prop.put("pool_1_name","httpd Session Pool");
         prop.put("pool_1_maxActive",httpdPoolConfig.maxActive);
         prop.put("pool_1_maxIdle",httpdPoolConfig.maxIdle);
-        prop.put("pool_1_minIdle",httpdPoolConfig.minIdle);                
-        prop.put("pool",2);
+        prop.put("pool_1_minIdle",httpdPoolConfig.minIdle);     
         
+        GenericObjectPool.Config stackerPoolConfig = switchboard.sbStackCrawlThread.getPoolConfig();
+        prop.put("pool_2_name","CrawlStacker Session Pool");
+        prop.put("pool_2_maxActive",stackerPoolConfig.maxActive);
+        prop.put("pool_2_maxIdle",stackerPoolConfig.maxIdle);
+        prop.put("pool_2_minIdle",stackerPoolConfig.minIdle);                
+        prop.put("pool",3);        
         
         // return rewrite values for templates
         return prop;

source/de/anomic/plasma/plasmaCrawlStacker.java

@@ -72,6 +72,7 @@ import de.anomic.yacy.yacyCore;
 public final class plasmaCrawlStacker {
     
     final WorkerPool theWorkerPool;
+    private GenericObjectPool.Config theWorkerPoolConfig = null; 
     final ThreadGroup theWorkerThreadGroup = new ThreadGroup("stackCrawlThreadGroup");
     final serverLog log = new serverLog("STACKCRAWL");
     final plasmaSwitchboard sb;
@@ -85,10 +86,40 @@ public final class plasmaCrawlStacker {
         this.log.logInfo(this.queue.size() + " entries in the stackCrawl queue.");
         this.log.logInfo("STACKCRAWL thread initialized.");
         
-        this.theWorkerPool = new WorkerPool(new WorkterFactory(this.theWorkerThreadGroup));  
+        // configuring the thread pool
+        // implementation of session thread pool
+        this.theWorkerPoolConfig = new GenericObjectPool.Config();
+
+        // The maximum number of active connections that can be allocated from pool at the same time,
+        // 0 for no limit
+        this.theWorkerPoolConfig.maxActive = Integer.parseInt(sb.getConfig("stacker.MaxActiveThreads","50"));
+
+        // The maximum number of idle connections connections in the pool
+        // 0 = no limit.        
+        this.theWorkerPoolConfig.maxIdle = Integer.parseInt(sb.getConfig("stacker.MaxIdleThreads","10"));
+        this.theWorkerPoolConfig.minIdle = Integer.parseInt(sb.getConfig("stacker.MinIdleThreads","5"));    
+
+        // block undefinitely 
+        this.theWorkerPoolConfig.maxWait = -1; 
+
+        // Action to take in case of an exhausted DBCP statement pool
+        // 0 = fail, 1 = block, 2= grow        
+        this.theWorkerPoolConfig.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK; 
+        this.theWorkerPoolConfig.minEvictableIdleTimeMillis = 30000; 
+        
+        // creating worker pool
+        this.theWorkerPool = new WorkerPool(new WorkterFactory(this.theWorkerThreadGroup),this.theWorkerPoolConfig);  
         
     }
     
+    public GenericObjectPool.Config getPoolConfig() {
+        return this.theWorkerPoolConfig;
+    }    
+    
+    public void setPoolConfig(GenericObjectPool.Config newConfig) {
+        this.theWorkerPool.setConfig(newConfig);
+    }    
+    
     public void close() {
         try {
             this.log.logFine("Shutdown. Terminationg worker threads.");

yacy.init

@@ -559,6 +559,11 @@ crawler.MaxActiveThreads = 10
 crawler.MaxIdleThreads = 7
 crawler.MinIdleThreads = 5
 
+# maximum number of crawl-stacker threads
+stacker.MaxActiveThreads = 50
+stacker.MaxIdleThreads = 10
+stacker.MinIdleThreads = 5
+
 # maximum size of indexing queue
 indexer.slots = 100