*) further redesign of threadpools to solve too many thread problem

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1473 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/plasma/plasmaCrawlLoader.java

@@ -323,6 +323,7 @@ final class CrawlerPool extends GenericObjectPool {
         if (this.isClosed) return;
         if (obj instanceof plasmaCrawlWorker) {
             try {
+                ((plasmaCrawlWorker)obj).setName(plasmaCrawlWorker.threadBaseName + "_invalidated");
                 ((plasmaCrawlWorker)obj).setStopped(true);
                 super.invalidateObject(obj);
             } catch (Exception e) {
@@ -434,7 +435,10 @@ final class CrawlerFactory implements org.apache.commons.pool.PoolableObjectFact
         if (obj == null) return;
         if (obj instanceof plasmaCrawlWorker) {
             plasmaCrawlWorker theWorker = (plasmaCrawlWorker) obj;
+            theWorker.destroyed = true;
+            theWorker.setName(plasmaCrawlWorker.threadBaseName + "_destroyed");
             theWorker.setStopped(true);
+            theWorker.interrupt();
         }
     }
 

source/de/anomic/plasma/plasmaCrawlStacker.java

@@ -644,7 +644,8 @@ public final class plasmaCrawlStacker {
         public void destroyObject(Object obj) {
             if (obj instanceof Worker) {
                 Worker theWorker = (Worker) obj;
-                ((Worker)obj).setName("stackCrawlThread_destroyed");
+                theWorker.setName("stackCrawlThread_destroyed");
+                theWorker.destroyed = true;
                 theWorker.setStopped(true);
                 theWorker.interrupt();
             }
@@ -797,7 +798,8 @@ public final class plasmaCrawlStacker {
         
     }
     
-    public final class Worker extends Thread {        
+    public final class Worker extends Thread {  
+            boolean destroyed = false;
             private boolean running = false;
             private boolean stopped = false;
             private boolean done = false;
@@ -845,7 +847,6 @@ public final class plasmaCrawlStacker {
             }
             
             public void run()  {
-                boolean interrupted = false;
                 this.running = true;
                 
                 try {
@@ -868,10 +869,9 @@ public final class plasmaCrawlStacker {
                         }
                     }
                 } catch (InterruptedException ex) {
-                    interrupted = true;
                     serverLog.logInfo("STACKCRAWL-POOL","Interruption of thread '" + this.getName() + "' detected.");
                 } finally {
-                    if (plasmaCrawlStacker.this.theWorkerPool != null && !interrupted) 
+                    if (plasmaCrawlStacker.this.theWorkerPool != null && !this.destroyed) 
                         plasmaCrawlStacker.this.theWorkerPool.invalidateObject(this);
                 }
             }

source/de/anomic/plasma/plasmaCrawlWorker.java

@@ -83,6 +83,7 @@ public final class plasmaCrawlWorker extends Thread {
     private plasmaCrawlProfile.entry profile;
 //  private String error;
 
+    boolean destroyed = false;
     private boolean running = false;
     private boolean stopped = false;
     private boolean done = false;   
@@ -113,7 +114,7 @@ public final class plasmaCrawlWorker extends Thread {
             plasmaSwitchboard theSb,
             plasmaHTCache theCacheManager,
             serverLog theLog) {
-        super(theTG,threadBaseName + "_inPool");
+        super(theTG,threadBaseName + "_created");
 
         this.myPool = thePool;
         this.sb = theSb;
@@ -186,7 +187,7 @@ public final class plasmaCrawlWorker extends Thread {
         } catch (InterruptedException ex) {
             serverLog.logInfo("CRAWLER-POOL","Interruption of thread '" + this.getName() + "' detected."); 
         } finally {
-            if (this.myPool != null) 
+            if (this.myPool != null && !this.destroyed) 
                 this.myPool.invalidateObject(this);
         }
     }
@@ -486,7 +487,13 @@ public final class plasmaCrawlWorker extends Thread {
             boolean retryCrawling = false;
             String errorMsg = e.getMessage();
 
-            if (e instanceof MalformedURLException) {
+            if ((e instanceof IOException) && 
+                (errorMsg != null) && 
+                (errorMsg.indexOf("socket closed") >= 0) &&
+                (Thread.currentThread().isInterrupted())
+            ) {
+                log.logInfo("CRAWLER Interruption detected because of server shutdown.");
+            } else if (e instanceof MalformedURLException) {
                 log.logWarning("CRAWLER Malformed URL '" + url.toString() + "' detected. ");
             } else if (e instanceof NoRouteToHostException) {
                 log.logWarning("CRAWLER No route to host found while trying to crawl URL  '" + url.toString() + "'.");
@@ -523,7 +530,7 @@ public final class plasmaCrawlWorker extends Thread {
             } else if ((errorMsg != null) && (errorMsg.indexOf("Network is unreachable") >=0)) {
                 log.logSevere("CRAWLER Network is unreachable while trying to crawl URL '" + url.toString() + "'. ");
             } else if ((errorMsg != null) && (errorMsg.indexOf("No trusted certificate found")>= 0)) {
-                log.logSevere("CRAWLER No trusted certificate found for URL '" + url.toString() + "'. ");
+                log.logSevere("CRAWLER No trusted certificate found for URL '" + url.toString() + "'. ");            
             } else {
                 log.logSevere("CRAWLER Unexpected Error with URL '" + url.toString() + "': " + e.toString(),e);
             }

source/de/anomic/server/serverCore.java

@@ -616,6 +616,7 @@ public final class serverCore extends serverAbstractThread implements serverThre
             if (this.isClosed) return;
             if (obj instanceof Session) {
                 try {
+                    ((Session)obj).setName("Session_invalidated");
                     ((Session)obj).setStopped(true);
                     super.invalidateObject(obj);
                 } catch (Exception e) {
@@ -715,7 +716,10 @@ public final class serverCore extends serverAbstractThread implements serverThre
         public void destroyObject(Object obj) {
             if (obj instanceof Session) {
                 Session theSession = (Session) obj;
+                theSession.destroyed = true;
+                theSession.setName("Session_destroyed");
                 theSession.setStopped(true);
+                theSession.interrupt();
             }
         }
         
@@ -760,6 +764,7 @@ public final class serverCore extends serverAbstractThread implements serverThre
         // synchronization object needed for the threadpool implementation
         private Object syncObject;        
         
+        boolean destroyed = false;
         private boolean running = false;
         private boolean stopped = false;
         private boolean done = false;        
@@ -785,7 +790,7 @@ public final class serverCore extends serverAbstractThread implements serverThre
         
     	
     	public Session(ThreadGroup theThreadGroup) {
-            super(theThreadGroup,"Session");
+            super(theThreadGroup,"Session_created");
     	}
         
         public int getCommandCount() {
@@ -952,7 +957,7 @@ public final class serverCore extends serverAbstractThread implements serverThre
             } catch (InterruptedException ex) {
                 serverLog.logInfo("SESSION-POOL","Interruption of thread '" + this.getName() + "' detected."); 
             } finally {
-                if (serverCore.this.theSessionPool != null) 
+                if (serverCore.this.theSessionPool != null && !this.destroyed) 
                     serverCore.this.theSessionPool.invalidateObject(this);
             }
         }