boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

better care for states of the IODispatcher

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5919 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent 1db9cdd4e4
commit 6e0b57284d
5 changed files with 41 additions and 26 deletions
Show Stats Download Patch File Download Diff File

42
source/de/anomic/kelondro/blob/BLOBArray.java
Unescape View File

@ -191,7 +191,7 @@ public class BLOBArray implements BLOB {
blobItem b;
for (int i = 0; i < this.blobs.size(); i++) {
b = this.blobs.get(i);
if (b.location.equals(location)) {
if (b.location.getAbsolutePath().equals(location.getAbsolutePath())) {
this.blobs.remove(i);
b.blob.close(writeIDX);
b.blob = null;
@ -199,6 +199,7 @@ public class BLOBArray implements BLOB {
return;
}
}
Log.logSevere("BLOBArray", "file " + location + " cannot be unmounted. The file " + ((location.exists()) ? "exists." : "does not exist."));
}
private File unmount(int idx) {
@ -211,31 +212,35 @@ public class BLOBArray implements BLOB {
}
public synchronized File[] unmountBestMatch(double maxq, long maxResultSize) {
if (this.blobs.size() < 2) return null;
long l, r;
File lf, rf;
double min = Double.MAX_VALUE;
int[] idx = new int[2];
File[] bestMatch = new File[2];
maxResultSize = maxResultSize >> 1;
for (int i = 0; i < this.blobs.size() - 1; i++) {
for (int j = i + 1; j < this.blobs.size(); j++) {
l = 1 + (this.blobs.get(i).location.length() >> 1);
r = 1 + (this.blobs.get(j).location.length() >> 1);
lf = this.blobs.get(i).location;
rf = this.blobs.get(j).location;
l = 1 + (lf.length() >> 1);
r = 1 + (rf.length() >> 1);
if (l + r > maxResultSize) continue;
double q = Math.max(((double) l)/((double) r), ((double) r)/((double) l));
double q = Math.max((double) l, (double) r) / Math.min((double) l, (double) r);
if (q < min) {
min = q;
idx[0] = i;
idx[1] = j;
bestMatch[0] = lf;
bestMatch[1] = rf;
}
}
}
if (min > maxq) return null;
File[] bestmatch = new File[]{this.blobs.get(idx[0]).location, this.blobs.get(idx[1]).location};
unmount(idx[1]);
unmount(idx[0]);
return bestmatch;
unmountBLOB(bestMatch[1], false);
unmountBLOB(bestMatch[0], false);
return bestMatch;
}
public synchronized File[] unmountSmallest(long maxResultSize) {
if (this.blobs.size() < 2) return null;
File f0 = smallestBLOB(null, maxResultSize);
if (f0 == null) return null;
File f1 = smallestBLOB(f0, maxResultSize - f0.length());
@ -252,18 +257,19 @@ public class BLOBArray implements BLOB {
public synchronized File smallestBLOB(File excluding, long maxsize) {
if (this.blobs.size() == 0) return null;
int bestIndex = -1;
File bestFile = null;
long smallest = Long.MAX_VALUE;
File f = null;
for (int i = 0; i < this.blobs.size(); i++) {
if (this.blobs.get(i).location == excluding) continue;
if (this.blobs.get(i).location.length() < smallest) {
smallest = this.blobs.get(i).location.length();
bestIndex = i;
f = this.blobs.get(i).location;
if (excluding != null && f.getAbsolutePath().equals(excluding.getAbsolutePath())) continue;
if (f.length() < smallest) {
smallest = f.length();
bestFile = f;
}
}
if (bestIndex == -1) return null;
if (smallest > maxsize) return null;
return this.blobs.get(bestIndex).location;
return bestFile;
}
public synchronized File unmountOldestBLOB(boolean smallestFromFirst2) {

7
source/de/anomic/kelondro/text/IODispatcher.java
Unescape View File

@ -94,7 +94,7 @@ public class IODispatcher <ReferenceType extends Reference> extends Thread {
}
public synchronized int queueLength() {
return (controlQueue == null) ? 0 : controlQueue.availablePermits();
return (controlQueue == null || !this.isAlive()) ? 0 : controlQueue.availablePermits();
}
public synchronized void merge(File f1, File f2, BLOBArray array, Row payloadrow, File newFile) {
@ -160,12 +160,15 @@ public class IODispatcher <ReferenceType extends Reference> extends Thread {
}
Log.logSevere("IODispatcher", "main loop in bad state, dumpQueue.size() = " + dumpQueue.size() + ", mergeQueue.size() = " + mergeQueue.size() + ", controlQueue.availablePermits() = " + controlQueue.availablePermits());
assert false; // this should never happen
assert false : "this process statt should not be reached"; // this should never happen
}
Log.logInfo("IODispatcher", "loop terminated");
} catch (InterruptedException e) {
e.printStackTrace();
Log.logSevere("IODispatcher", "main run job was interrupted (3)", e);
} catch (Exception e) {
e.printStackTrace();
Log.logSevere("IODispatcher", "main run job failed (4)", e);
} finally {
Log.logInfo("IODispatcher", "terminating run job");
controlQueue = null;

2
source/de/anomic/kelondro/text/IndexCell.java
Unescape View File

@ -276,7 +276,7 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
private synchronized void cleanCache() {
// dump the cache if necessary
if (this.ram.size() >= this.maxRamEntries || (this.ram.size() > 2000 && !MemoryControl.request(100L * 1024L * 1024L, false))) {
if (this.ram.size() >= this.maxRamEntries || (this.ram.size() > 3000 && !MemoryControl.request(80L * 1024L * 1024L, false))) {
try {
cacheDump();
} catch (IOException e) {

8
source/de/anomic/kelondro/text/ReferenceContainerArray.java
Unescape View File

@ -254,10 +254,10 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
boolean donesomething = false;
// first try to merge small files that match
while (this.merger.queueLength() < 3) {
while (this.merger.queueLength() < 3 || this.array.entries() >= 50) {
File[] ff = this.array.unmountBestMatch(2.0, targetFileSize);
if (ff == null) break;
Log.logInfo("RICELL-shrink", "unmountBestMatch(2.0, " + targetFileSize + ")");
Log.logInfo("RICELL-shrink1", "unmountBestMatch(2.0, " + targetFileSize + ")");
merger.merge(ff[0], ff[1], this.array, this.payloadrow, newContainerBLOBFile());
donesomething = true;
}
@ -266,7 +266,7 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
while (this.merger.queueLength() < 2) {
File[] ff = this.array.unmountSmallest(targetFileSize);
if (ff == null) break;
Log.logInfo("RICELL-shrink", "unmountSmallest(" + targetFileSize + ")");
Log.logInfo("RICELL-shrink2", "unmountSmallest(" + targetFileSize + ")");
merger.merge(ff[0], ff[1], this.array, this.payloadrow, newContainerBLOBFile());
donesomething = true;
}
@ -275,7 +275,7 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
while (this.merger.queueLength() < 1) {
File[] ff = this.array.unmountBestMatch(2.0, maxFileSize);
if (ff == null) break;
Log.logInfo("RICELL-shrink", "unmountBestMatch(2.0, " + maxFileSize + ")");
Log.logInfo("RICELL-shrink3", "unmountBestMatch(2.0, " + maxFileSize + ")");
merger.merge(ff[0], ff[1], this.array, this.payloadrow, newContainerBLOBFile());
donesomething = true;
}

6
source/de/anomic/plasma/plasmaWordIndex.java
Unescape View File

@ -426,6 +426,12 @@ public final class plasmaWordIndex {
final int urlLength = url.toNormalform(true, true).length();
final int urlComps = htmlFilterContentScraper.urlComps(url.toString()).length;
// check if merger is running
if (this.merger != null && !this.merger.isAlive()) {
log.logSevere("re-starting IODispatcher");
this.merger.start();
}
// iterate over all words of context text
final Iterator<Map.Entry<String, Word>> i = condenser.words().entrySet().iterator();
Map.Entry<String, Word> wentry;

Write Preview
Loading…
Cancel
Save