From bea2b9edeeef51e726a0e243073279c3d3c88951 Mon Sep 17 00:00:00 2001 From: theli Date: Sat, 28 Jan 2006 16:18:07 +0000 Subject: [PATCH] *) further redesign of threadpools to solve too many thread problem git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1473 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/de/anomic/plasma/plasmaCrawlLoader.java | 4 ++++ source/de/anomic/plasma/plasmaCrawlStacker.java | 10 +++++----- source/de/anomic/plasma/plasmaCrawlWorker.java | 15 +++++++++++---- source/de/anomic/server/serverCore.java | 9 +++++++-- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/source/de/anomic/plasma/plasmaCrawlLoader.java b/source/de/anomic/plasma/plasmaCrawlLoader.java index 4cdfc3bb2..45b2c1c29 100644 --- a/source/de/anomic/plasma/plasmaCrawlLoader.java +++ b/source/de/anomic/plasma/plasmaCrawlLoader.java @@ -323,6 +323,7 @@ final class CrawlerPool extends GenericObjectPool { if (this.isClosed) return; if (obj instanceof plasmaCrawlWorker) { try { + ((plasmaCrawlWorker)obj).setName(plasmaCrawlWorker.threadBaseName + "_invalidated"); ((plasmaCrawlWorker)obj).setStopped(true); super.invalidateObject(obj); } catch (Exception e) { @@ -434,7 +435,10 @@ final class CrawlerFactory implements org.apache.commons.pool.PoolableObjectFact if (obj == null) return; if (obj instanceof plasmaCrawlWorker) { plasmaCrawlWorker theWorker = (plasmaCrawlWorker) obj; + theWorker.destroyed = true; + theWorker.setName(plasmaCrawlWorker.threadBaseName + "_destroyed"); theWorker.setStopped(true); + theWorker.interrupt(); } } diff --git a/source/de/anomic/plasma/plasmaCrawlStacker.java b/source/de/anomic/plasma/plasmaCrawlStacker.java index 92f360099..83704fe4c 100644 --- a/source/de/anomic/plasma/plasmaCrawlStacker.java +++ b/source/de/anomic/plasma/plasmaCrawlStacker.java @@ -644,7 +644,8 @@ public final class plasmaCrawlStacker { public void destroyObject(Object obj) { if (obj instanceof Worker) { Worker theWorker = (Worker) obj; - ((Worker)obj).setName("stackCrawlThread_destroyed"); + theWorker.setName("stackCrawlThread_destroyed"); + theWorker.destroyed = true; theWorker.setStopped(true); theWorker.interrupt(); } @@ -797,7 +798,8 @@ public final class plasmaCrawlStacker { } - public final class Worker extends Thread { + public final class Worker extends Thread { + boolean destroyed = false; private boolean running = false; private boolean stopped = false; private boolean done = false; @@ -845,7 +847,6 @@ public final class plasmaCrawlStacker { } public void run() { - boolean interrupted = false; this.running = true; try { @@ -868,10 +869,9 @@ public final class plasmaCrawlStacker { } } } catch (InterruptedException ex) { - interrupted = true; serverLog.logInfo("STACKCRAWL-POOL","Interruption of thread '" + this.getName() + "' detected."); } finally { - if (plasmaCrawlStacker.this.theWorkerPool != null && !interrupted) + if (plasmaCrawlStacker.this.theWorkerPool != null && !this.destroyed) plasmaCrawlStacker.this.theWorkerPool.invalidateObject(this); } } diff --git a/source/de/anomic/plasma/plasmaCrawlWorker.java b/source/de/anomic/plasma/plasmaCrawlWorker.java index b04c2f9b1..35f370844 100644 --- a/source/de/anomic/plasma/plasmaCrawlWorker.java +++ b/source/de/anomic/plasma/plasmaCrawlWorker.java @@ -83,6 +83,7 @@ public final class plasmaCrawlWorker extends Thread { private plasmaCrawlProfile.entry profile; // private String error; + boolean destroyed = false; private boolean running = false; private boolean stopped = false; private boolean done = false; @@ -113,7 +114,7 @@ public final class plasmaCrawlWorker extends Thread { plasmaSwitchboard theSb, plasmaHTCache theCacheManager, serverLog theLog) { - super(theTG,threadBaseName + "_inPool"); + super(theTG,threadBaseName + "_created"); this.myPool = thePool; this.sb = theSb; @@ -186,7 +187,7 @@ public final class plasmaCrawlWorker extends Thread { } catch (InterruptedException ex) { serverLog.logInfo("CRAWLER-POOL","Interruption of thread '" + this.getName() + "' detected."); } finally { - if (this.myPool != null) + if (this.myPool != null && !this.destroyed) this.myPool.invalidateObject(this); } } @@ -486,7 +487,13 @@ public final class plasmaCrawlWorker extends Thread { boolean retryCrawling = false; String errorMsg = e.getMessage(); - if (e instanceof MalformedURLException) { + if ((e instanceof IOException) && + (errorMsg != null) && + (errorMsg.indexOf("socket closed") >= 0) && + (Thread.currentThread().isInterrupted()) + ) { + log.logInfo("CRAWLER Interruption detected because of server shutdown."); + } else if (e instanceof MalformedURLException) { log.logWarning("CRAWLER Malformed URL '" + url.toString() + "' detected. "); } else if (e instanceof NoRouteToHostException) { log.logWarning("CRAWLER No route to host found while trying to crawl URL '" + url.toString() + "'."); @@ -523,7 +530,7 @@ public final class plasmaCrawlWorker extends Thread { } else if ((errorMsg != null) && (errorMsg.indexOf("Network is unreachable") >=0)) { log.logSevere("CRAWLER Network is unreachable while trying to crawl URL '" + url.toString() + "'. "); } else if ((errorMsg != null) && (errorMsg.indexOf("No trusted certificate found")>= 0)) { - log.logSevere("CRAWLER No trusted certificate found for URL '" + url.toString() + "'. "); + log.logSevere("CRAWLER No trusted certificate found for URL '" + url.toString() + "'. "); } else { log.logSevere("CRAWLER Unexpected Error with URL '" + url.toString() + "': " + e.toString(),e); } diff --git a/source/de/anomic/server/serverCore.java b/source/de/anomic/server/serverCore.java index 367480848..3780fbd39 100644 --- a/source/de/anomic/server/serverCore.java +++ b/source/de/anomic/server/serverCore.java @@ -616,6 +616,7 @@ public final class serverCore extends serverAbstractThread implements serverThre if (this.isClosed) return; if (obj instanceof Session) { try { + ((Session)obj).setName("Session_invalidated"); ((Session)obj).setStopped(true); super.invalidateObject(obj); } catch (Exception e) { @@ -715,7 +716,10 @@ public final class serverCore extends serverAbstractThread implements serverThre public void destroyObject(Object obj) { if (obj instanceof Session) { Session theSession = (Session) obj; + theSession.destroyed = true; + theSession.setName("Session_destroyed"); theSession.setStopped(true); + theSession.interrupt(); } } @@ -760,6 +764,7 @@ public final class serverCore extends serverAbstractThread implements serverThre // synchronization object needed for the threadpool implementation private Object syncObject; + boolean destroyed = false; private boolean running = false; private boolean stopped = false; private boolean done = false; @@ -785,7 +790,7 @@ public final class serverCore extends serverAbstractThread implements serverThre public Session(ThreadGroup theThreadGroup) { - super(theThreadGroup,"Session"); + super(theThreadGroup,"Session_created"); } public int getCommandCount() { @@ -952,7 +957,7 @@ public final class serverCore extends serverAbstractThread implements serverThre } catch (InterruptedException ex) { serverLog.logInfo("SESSION-POOL","Interruption of thread '" + this.getName() + "' detected."); } finally { - if (serverCore.this.theSessionPool != null) + if (serverCore.this.theSessionPool != null && !this.destroyed) serverCore.this.theSessionPool.invalidateObject(this); } }