From 6e0b57284d29aaaf631ded15d9f8dde573d195bd Mon Sep 17 00:00:00 2001 From: orbiter Date: Sun, 3 May 2009 22:54:47 +0000 Subject: [PATCH] better care for states of the IODispatcher git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5919 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/de/anomic/kelondro/blob/BLOBArray.java | 44 +++++++++++-------- .../de/anomic/kelondro/text/IODispatcher.java | 7 ++- source/de/anomic/kelondro/text/IndexCell.java | 2 +- .../text/ReferenceContainerArray.java | 8 ++-- source/de/anomic/plasma/plasmaWordIndex.java | 6 +++ 5 files changed, 41 insertions(+), 26 deletions(-) diff --git a/source/de/anomic/kelondro/blob/BLOBArray.java b/source/de/anomic/kelondro/blob/BLOBArray.java index 738b1954f..9683aa5c7 100755 --- a/source/de/anomic/kelondro/blob/BLOBArray.java +++ b/source/de/anomic/kelondro/blob/BLOBArray.java @@ -191,7 +191,7 @@ public class BLOBArray implements BLOB { blobItem b; for (int i = 0; i < this.blobs.size(); i++) { b = this.blobs.get(i); - if (b.location.equals(location)) { + if (b.location.getAbsolutePath().equals(location.getAbsolutePath())) { this.blobs.remove(i); b.blob.close(writeIDX); b.blob = null; @@ -199,6 +199,7 @@ public class BLOBArray implements BLOB { return; } } + Log.logSevere("BLOBArray", "file " + location + " cannot be unmounted. The file " + ((location.exists()) ? "exists." : "does not exist.")); } private File unmount(int idx) { @@ -211,32 +212,36 @@ public class BLOBArray implements BLOB { } public synchronized File[] unmountBestMatch(double maxq, long maxResultSize) { + if (this.blobs.size() < 2) return null; long l, r; + File lf, rf; double min = Double.MAX_VALUE; - int[] idx = new int[2]; + File[] bestMatch = new File[2]; maxResultSize = maxResultSize >> 1; for (int i = 0; i < this.blobs.size() - 1; i++) { for (int j = i + 1; j < this.blobs.size(); j++) { - l = 1 + (this.blobs.get(i).location.length() >> 1); - r = 1 + (this.blobs.get(j).location.length() >> 1); + lf = this.blobs.get(i).location; + rf = this.blobs.get(j).location; + l = 1 + (lf.length() >> 1); + r = 1 + (rf.length() >> 1); if (l + r > maxResultSize) continue; - double q = Math.max(((double) l)/((double) r), ((double) r)/((double) l)); + double q = Math.max((double) l, (double) r) / Math.min((double) l, (double) r); if (q < min) { min = q; - idx[0] = i; - idx[1] = j; + bestMatch[0] = lf; + bestMatch[1] = rf; } } } if (min > maxq) return null; - File[] bestmatch = new File[]{this.blobs.get(idx[0]).location, this.blobs.get(idx[1]).location}; - unmount(idx[1]); - unmount(idx[0]); - return bestmatch; + unmountBLOB(bestMatch[1], false); + unmountBLOB(bestMatch[0], false); + return bestMatch; } public synchronized File[] unmountSmallest(long maxResultSize) { - File f0 = smallestBLOB(null, maxResultSize); + if (this.blobs.size() < 2) return null; + File f0 = smallestBLOB(null, maxResultSize); if (f0 == null) return null; File f1 = smallestBLOB(f0, maxResultSize - f0.length()); if (f1 == null) return null; @@ -252,18 +257,19 @@ public class BLOBArray implements BLOB { public synchronized File smallestBLOB(File excluding, long maxsize) { if (this.blobs.size() == 0) return null; - int bestIndex = -1; + File bestFile = null; long smallest = Long.MAX_VALUE; + File f = null; for (int i = 0; i < this.blobs.size(); i++) { - if (this.blobs.get(i).location == excluding) continue; - if (this.blobs.get(i).location.length() < smallest) { - smallest = this.blobs.get(i).location.length(); - bestIndex = i; + f = this.blobs.get(i).location; + if (excluding != null && f.getAbsolutePath().equals(excluding.getAbsolutePath())) continue; + if (f.length() < smallest) { + smallest = f.length(); + bestFile = f; } } - if (bestIndex == -1) return null; if (smallest > maxsize) return null; - return this.blobs.get(bestIndex).location; + return bestFile; } public synchronized File unmountOldestBLOB(boolean smallestFromFirst2) { diff --git a/source/de/anomic/kelondro/text/IODispatcher.java b/source/de/anomic/kelondro/text/IODispatcher.java index a4ba373f1..d719b3971 100644 --- a/source/de/anomic/kelondro/text/IODispatcher.java +++ b/source/de/anomic/kelondro/text/IODispatcher.java @@ -94,7 +94,7 @@ public class IODispatcher extends Thread { } public synchronized int queueLength() { - return (controlQueue == null) ? 0 : controlQueue.availablePermits(); + return (controlQueue == null || !this.isAlive()) ? 0 : controlQueue.availablePermits(); } public synchronized void merge(File f1, File f2, BLOBArray array, Row payloadrow, File newFile) { @@ -160,12 +160,15 @@ public class IODispatcher extends Thread { } Log.logSevere("IODispatcher", "main loop in bad state, dumpQueue.size() = " + dumpQueue.size() + ", mergeQueue.size() = " + mergeQueue.size() + ", controlQueue.availablePermits() = " + controlQueue.availablePermits()); - assert false; // this should never happen + assert false : "this process statt should not be reached"; // this should never happen } Log.logInfo("IODispatcher", "loop terminated"); } catch (InterruptedException e) { e.printStackTrace(); Log.logSevere("IODispatcher", "main run job was interrupted (3)", e); + } catch (Exception e) { + e.printStackTrace(); + Log.logSevere("IODispatcher", "main run job failed (4)", e); } finally { Log.logInfo("IODispatcher", "terminating run job"); controlQueue = null; diff --git a/source/de/anomic/kelondro/text/IndexCell.java b/source/de/anomic/kelondro/text/IndexCell.java index 187d5b891..ecc628a60 100644 --- a/source/de/anomic/kelondro/text/IndexCell.java +++ b/source/de/anomic/kelondro/text/IndexCell.java @@ -276,7 +276,7 @@ public final class IndexCell extends AbstractBu private synchronized void cleanCache() { // dump the cache if necessary - if (this.ram.size() >= this.maxRamEntries || (this.ram.size() > 2000 && !MemoryControl.request(100L * 1024L * 1024L, false))) { + if (this.ram.size() >= this.maxRamEntries || (this.ram.size() > 3000 && !MemoryControl.request(80L * 1024L * 1024L, false))) { try { cacheDump(); } catch (IOException e) { diff --git a/source/de/anomic/kelondro/text/ReferenceContainerArray.java b/source/de/anomic/kelondro/text/ReferenceContainerArray.java index a8b1ce26b..e58049ac3 100644 --- a/source/de/anomic/kelondro/text/ReferenceContainerArray.java +++ b/source/de/anomic/kelondro/text/ReferenceContainerArray.java @@ -254,10 +254,10 @@ public final class ReferenceContainerArray { boolean donesomething = false; // first try to merge small files that match - while (this.merger.queueLength() < 3) { + while (this.merger.queueLength() < 3 || this.array.entries() >= 50) { File[] ff = this.array.unmountBestMatch(2.0, targetFileSize); if (ff == null) break; - Log.logInfo("RICELL-shrink", "unmountBestMatch(2.0, " + targetFileSize + ")"); + Log.logInfo("RICELL-shrink1", "unmountBestMatch(2.0, " + targetFileSize + ")"); merger.merge(ff[0], ff[1], this.array, this.payloadrow, newContainerBLOBFile()); donesomething = true; } @@ -266,7 +266,7 @@ public final class ReferenceContainerArray { while (this.merger.queueLength() < 2) { File[] ff = this.array.unmountSmallest(targetFileSize); if (ff == null) break; - Log.logInfo("RICELL-shrink", "unmountSmallest(" + targetFileSize + ")"); + Log.logInfo("RICELL-shrink2", "unmountSmallest(" + targetFileSize + ")"); merger.merge(ff[0], ff[1], this.array, this.payloadrow, newContainerBLOBFile()); donesomething = true; } @@ -275,7 +275,7 @@ public final class ReferenceContainerArray { while (this.merger.queueLength() < 1) { File[] ff = this.array.unmountBestMatch(2.0, maxFileSize); if (ff == null) break; - Log.logInfo("RICELL-shrink", "unmountBestMatch(2.0, " + maxFileSize + ")"); + Log.logInfo("RICELL-shrink3", "unmountBestMatch(2.0, " + maxFileSize + ")"); merger.merge(ff[0], ff[1], this.array, this.payloadrow, newContainerBLOBFile()); donesomething = true; } diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index 79f6b34a7..b82934427 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -426,6 +426,12 @@ public final class plasmaWordIndex { final int urlLength = url.toNormalform(true, true).length(); final int urlComps = htmlFilterContentScraper.urlComps(url.toString()).length; + // check if merger is running + if (this.merger != null && !this.merger.isAlive()) { + log.logSevere("re-starting IODispatcher"); + this.merger.start(); + } + // iterate over all words of context text final Iterator> i = condenser.words().entrySet().iterator(); Map.Entry wentry;