From 1dc12d66593c79a4262e399d7b7509372b5af3cf Mon Sep 17 00:00:00 2001 From: theli Date: Wed, 20 Sep 2006 04:36:25 +0000 Subject: [PATCH] *) Bugfix for shutdown problem caused by cacheScan thread See: http://www.yacy-forum.de/viewtopic.php?p=25729 git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2636 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/de/anomic/plasma/plasmaHTCache.java | 91 +++++++++++++--------- 1 file changed, 55 insertions(+), 36 deletions(-) diff --git a/source/de/anomic/plasma/plasmaHTCache.java b/source/de/anomic/plasma/plasmaHTCache.java index 331410332..c1b3be482 100644 --- a/source/de/anomic/plasma/plasmaHTCache.java +++ b/source/de/anomic/plasma/plasmaHTCache.java @@ -63,6 +63,7 @@ import de.anomic.server.logging.serverLog; import de.anomic.server.serverFileUtils; import de.anomic.server.serverInstantThread; import de.anomic.server.serverSystem; +import de.anomic.server.serverThread; import de.anomic.tools.enumerateFiles; import java.io.File; @@ -98,6 +99,7 @@ public final class plasmaHTCache { public static final HashSet filesInUse = new HashSet(); // can we delete this file private ResourceInfoFactory objFactory; + private serverThread cacheScanThread; public plasmaHTCache(File htCachePath, long maxCacheSize, int bufferkb, long preloadTime) { // this.switchboard = switchboard; @@ -176,7 +178,7 @@ public final class plasmaHTCache { // start the cache startup thread // this will collect information about the current cache size and elements - serverInstantThread.oneTimeJob(this, "cacheScan", this.log, 120000); + this.cacheScanThread = serverInstantThread.oneTimeJob(this, "cacheScan", this.log, 120000); } private void deleteOldHTCache(File directory) { @@ -329,27 +331,29 @@ public final class plasmaHTCache { private void cleanupDoIt(long newCacheSize) { File obj; synchronized (cacheAge) { - Iterator iter = this.cacheAge.keySet().iterator(); - while (iter.hasNext() && this.curCacheSize >= newCacheSize) { - Object key = iter.next(); - obj = (File) this.cacheAge.get(key); - if (obj != null) { - if (filesInUse.contains(obj)) continue; - this.log.logFinest("Trying to delete old file: " + obj.toString()); - if (deleteFileandDirs (obj, "OLD")) { - try { - // As the file is gone, the entry in responseHeader.db is not needed anymore - this.log.logFinest("Trying to remove responseHeader for URL: " + - getURL(this.cachePath ,obj).toString()); - this.responseHeaderDB.remove(indexURL.urlHash(getURL(this.cachePath ,obj))); - } catch (IOException e) { - this.log.logInfo("IOExeption removing response header from DB: " + - e.getMessage(), e); + Iterator iter = this.cacheAge.keySet().iterator(); + while (iter.hasNext() && this.curCacheSize >= newCacheSize) { + if (Thread.currentThread().isInterrupted()) return; + + Object key = iter.next(); + obj = (File) this.cacheAge.get(key); + if (obj != null) { + if (filesInUse.contains(obj)) continue; + this.log.logFinest("Trying to delete old file: " + obj.toString()); + if (deleteFileandDirs (obj, "OLD")) { + try { + // As the file is gone, the entry in responseHeader.db is not needed anymore + this.log.logFinest("Trying to remove responseHeader for URL: " + + getURL(this.cachePath ,obj).toString()); + this.responseHeaderDB.remove(indexURL.urlHash(getURL(this.cachePath ,obj))); + } catch (IOException e) { + this.log.logInfo("IOExeption removing response header from DB: " + + e.getMessage(), e); + } } } + iter.remove(); } - iter.remove(); - } } } @@ -363,6 +367,12 @@ public final class plasmaHTCache { } public void close() { + // closing cache scan if still running + if ((this.cacheScanThread != null) && (this.cacheScanThread.isAlive())) { + this.cacheScanThread.terminate(true); + } + + // closing DB try {this.responseHeaderDB.close();} catch (IOException e) {} } @@ -380,18 +390,22 @@ public final class plasmaHTCache { public void cacheScan() { log.logConfig("STARTING HTCACHE SCANNING"); kelondroMScoreCluster doms = new kelondroMScoreCluster(); - int c = 0; - enumerateFiles ef = new enumerateFiles(this.cachePath, true, false, true, true); - File f; - while (ef.hasMoreElements()) { - c++; - f = (File) ef.nextElement(); - long d = f.lastModified(); + int fileCount = 0; + enumerateFiles fileEnum = new enumerateFiles(this.cachePath, true, false, true, true); + while (fileEnum.hasMoreElements()) { + if (Thread.currentThread().isInterrupted()) return; + fileCount++; + File nextFile = (File) fileEnum.nextElement(); + long nextFileModDate = nextFile.lastModified(); //System.out.println("Cache: " + dom(f)); - doms.incScore(dom(f)); - this.curCacheSize += f.length(); - this.cacheAge.put(ageString(d, f), f); - try {Thread.sleep(10);} catch (InterruptedException e) {} + doms.incScore(dom(nextFile)); + this.curCacheSize += nextFile.length(); + this.cacheAge.put(ageString(nextFileModDate, nextFile), nextFile); + try { + Thread.sleep(10); + } catch (InterruptedException e) { + return; + } } //System.out.println("%" + (String) cacheAge.firstKey() + "=" + cacheAge.get(cacheAge.firstKey())); long ageHours = 0; @@ -401,7 +415,7 @@ public final class plasmaHTCache { } catch (NumberFormatException e) { //e.printStackTrace(); } - this.log.logConfig("CACHE SCANNED, CONTAINS " + c + + this.log.logConfig("CACHE SCANNED, CONTAINS " + fileCount + " FILES = " + this.curCacheSize/1048576 + "MB, OLDEST IS " + ((ageHours < 24) ? (ageHours + " HOURS") : ((ageHours / 24) + " DAYS")) + " OLD"); cleanup(); @@ -411,19 +425,24 @@ public final class plasmaHTCache { String dom; long start = System.currentTimeMillis(); String result = ""; - c = 0; - while ((doms.size() > 0) && (c < 50) && ((System.currentTimeMillis() - start) < 60000)) { + fileCount = 0; + while ((doms.size() > 0) && (fileCount < 50) && ((System.currentTimeMillis() - start) < 60000)) { + if (Thread.currentThread().isInterrupted()) return; dom = (String) doms.getMaxObject(); InetAddress ip = httpc.dnsResolve(dom); if (ip == null) continue; result += ", " + dom + "=" + ip.getHostAddress(); this.log.logConfig("PRE-FILLED " + dom + "=" + ip.getHostAddress()); - c++; + fileCount++; doms.deleteScore(dom); // wait a short while to prevent that this looks like a DoS - try {Thread.sleep(100);} catch (InterruptedException e) {} + try { + Thread.sleep(100); + } catch (InterruptedException e) { + return; + } } - if (result.length() > 2) this.log.logConfig("PRE-FILLED DNS CACHE, FETCHED " + c + + if (result.length() > 2) this.log.logConfig("PRE-FILLED DNS CACHE, FETCHED " + fileCount + " ADDRESSES: " + result.substring(2)); }