From 1eb813bd43c345c0c8de59e988875824ab646bcf Mon Sep 17 00:00:00 2001 From: orbiter Date: Fri, 12 Sep 2008 11:51:48 +0000 Subject: [PATCH] shifted index deletion-on-exit rule to the class where the errors are produced git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5141 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/de/anomic/index/indexCollectionRI.java | 4 -- .../anomic/kelondro/kelondroAbstractRA.java | 4 ++ .../kelondro/kelondroAbstractRecords.java | 4 ++ .../kelondro/kelondroBufferedEcoFS.java | 4 ++ .../kelondro/kelondroBufferedIOChunks.java | 3 ++ source/de/anomic/kelondro/kelondroCache.java | 4 ++ .../kelondro/kelondroCollectionIndex.java | 42 +++++++++++++------ source/de/anomic/kelondro/kelondroEcoFS.java | 4 ++ .../de/anomic/kelondro/kelondroEcoTable.java | 4 ++ .../kelondro/kelondroFlexWidthArray.java | 4 ++ .../de/anomic/kelondro/kelondroIOChunks.java | 2 + source/de/anomic/kelondro/kelondroIndex.java | 1 + source/de/anomic/kelondro/kelondroRA.java | 2 + .../anomic/kelondro/kelondroRAIOChunks.java | 3 ++ .../de/anomic/kelondro/kelondroRAMIndex.java | 4 ++ source/de/anomic/kelondro/kelondroRowSet.java | 3 ++ .../de/anomic/kelondro/kelondroSQLTable.java | 7 +++- .../anomic/kelondro/kelondroSplitTable.java | 4 ++ source/de/anomic/plasma/plasmaDHTChunk.java | 3 -- source/de/anomic/plasma/plasmaWordIndex.java | 4 -- source/de/anomic/yacy/yacySearch.java | 2 +- 21 files changed, 86 insertions(+), 26 deletions(-) diff --git a/source/de/anomic/index/indexCollectionRI.java b/source/de/anomic/index/indexCollectionRI.java index 3ed94a5b7..13468a686 100644 --- a/source/de/anomic/index/indexCollectionRI.java +++ b/source/de/anomic/index/indexCollectionRI.java @@ -67,10 +67,6 @@ public class indexCollectionRI implements indexRI { } } - public void deleteIndexOnExit() { - collectionIndex.deleteIndexOnExit(); - } - public long getUpdateTime(final String wordHash) { final indexContainer entries = getContainer(wordHash, null); if (entries == null) return 0; diff --git a/source/de/anomic/kelondro/kelondroAbstractRA.java b/source/de/anomic/kelondro/kelondroAbstractRA.java index f3289714b..4ddabc84b 100644 --- a/source/de/anomic/kelondro/kelondroAbstractRA.java +++ b/source/de/anomic/kelondro/kelondroAbstractRA.java @@ -237,5 +237,9 @@ abstract class kelondroAbstractRA implements kelondroRA { read(b, 0, l); return b; } + + public void deleteOnExit() { + if (this.file != null) this.file.deleteOnExit(); + } } \ No newline at end of file diff --git a/source/de/anomic/kelondro/kelondroAbstractRecords.java b/source/de/anomic/kelondro/kelondroAbstractRecords.java index afb1aee4e..e16c1e99e 100644 --- a/source/de/anomic/kelondro/kelondroAbstractRecords.java +++ b/source/de/anomic/kelondro/kelondroAbstractRecords.java @@ -1024,6 +1024,10 @@ public abstract class kelondroAbstractRecords implements kelondroRecords { return b; } + public void deleteOnExit() { + this.entryFile.deleteOnExit(); + } + public abstract kelondroNode newNode(kelondroHandle handle, byte[] bulk, int offset) throws IOException; } diff --git a/source/de/anomic/kelondro/kelondroBufferedEcoFS.java b/source/de/anomic/kelondro/kelondroBufferedEcoFS.java index a8e359498..0482cb670 100644 --- a/source/de/anomic/kelondro/kelondroBufferedEcoFS.java +++ b/source/de/anomic/kelondro/kelondroBufferedEcoFS.java @@ -128,4 +128,8 @@ public class kelondroBufferedEcoFS { efs.cleanLast(); } + public void deleteOnExit() { + efs.deleteOnExit(); + } + } diff --git a/source/de/anomic/kelondro/kelondroBufferedIOChunks.java b/source/de/anomic/kelondro/kelondroBufferedIOChunks.java index 4964d9115..f669b8343 100644 --- a/source/de/anomic/kelondro/kelondroBufferedIOChunks.java +++ b/source/de/anomic/kelondro/kelondroBufferedIOChunks.java @@ -169,4 +169,7 @@ public final class kelondroBufferedIOChunks extends kelondroAbstractIOChunks imp super.finalize(); } + public void deleteOnExit() { + this.ra.deleteOnExit(); + } } diff --git a/source/de/anomic/kelondro/kelondroCache.java b/source/de/anomic/kelondro/kelondroCache.java index b00e00354..d88d36a0e 100644 --- a/source/de/anomic/kelondro/kelondroCache.java +++ b/source/de/anomic/kelondro/kelondroCache.java @@ -460,4 +460,8 @@ public class kelondroCache implements kelondroIndex { init(); } + public void deleteOnExit() { + this.index.deleteOnExit(); + } + } diff --git a/source/de/anomic/kelondro/kelondroCollectionIndex.java b/source/de/anomic/kelondro/kelondroCollectionIndex.java index d20397514..33834c4ff 100644 --- a/source/de/anomic/kelondro/kelondroCollectionIndex.java +++ b/source/de/anomic/kelondro/kelondroCollectionIndex.java @@ -53,8 +53,9 @@ public class kelondroCollectionIndex { private static final int serialNumber = 0; private static final long minimumRAM4Eco = 20 * 1024 * 1024; private static final int EcoFSBufferSize = 1000; + private static final int errorLimit = 500; // if the index exceeds this number of errors, it is re-built next time the application starts - kelondroIndex index; + private kelondroIndex index; private final int keylength; private final File path; private final String filenameStub; @@ -63,6 +64,7 @@ public class kelondroCollectionIndex { private Map arrays; // Map of (partitionNumber"-"chunksize)/kelondroFixedWidthArray - Objects private final kelondroRow payloadrow; // definition of the payload (chunks inside the collections) private final int maxPartitions; // this is the maxmimum number of array files + private int indexErrors; // counter for exceptions when index returned wrong value private static final int idx_col_key = 0; // the index private static final int idx_col_chunksize = 1; // chunksize (number of bytes in a single chunk, needed for migration option) @@ -113,6 +115,7 @@ public class kelondroCollectionIndex { public kelondroCollectionIndex(final File path, final String filenameStub, final int keyLength, final kelondroByteOrder indexOrder, final int loadfactor, final int maxpartitions, final kelondroRow rowdef) throws IOException { // the buffersize is number of bytes that are only used if the kelondroFlexTable is backed up with a kelondroTree + indexErrors = 0; this.path = path; this.filenameStub = filenameStub; this.keylength = keyLength; @@ -179,7 +182,7 @@ public class kelondroCollectionIndex { public void deleteIndexOnExit() { // will be rebuilt on next start - new File(this.path, this.filenameStub + ".index").deleteOnExit(); + this.index.deleteOnExit(); } private void openAllArrayFiles(final boolean indexGeneration, final kelondroByteOrder indexOrder) throws IOException { @@ -501,9 +504,11 @@ public class kelondroCollectionIndex { newPartitionNumber, serialNumber, this.payloadrow.objectsize); // modifies indexrow } - if ((int) indexrow.getColLong(idx_col_chunkcount) != collection.size()) - serverLog.logSevere("kelondroCollectionIndex", "UPDATE (put) ERROR: array has different chunkcount than index after merge: index = " + (int) indexrow.getColLong(idx_col_chunkcount) + ", collection.size() = " + collection.size()); - + if ((int) indexrow.getColLong(idx_col_chunkcount) != collection.size()) { + this.indexErrors++; + if (this.indexErrors == errorLimit) deleteIndexOnExit(); // delete index on exit for rebuild + serverLog.logSevere("kelondroCollectionIndex", "UPDATE (put) ERROR: array has different chunkcount than index after merge: index = " + (int) indexrow.getColLong(idx_col_chunkcount) + ", collection.size() = " + collection.size() + " (error #" + indexErrors + ")"); + } index.put(indexrow); // write modified indexrow } @@ -562,9 +567,11 @@ public class kelondroCollectionIndex { final int collectionsize = collection.size(); // extra variable for easier debugging final int indexrowcount = (int) indexrow.getColLong(idx_col_chunkcount); - if (indexrowcount != collectionsize) - serverLog.logSevere("kelondroCollectionIndex", "UPDATE (merge) ERROR: array has different chunkcount than index after merge: index = " + indexrowcount + ", collection.size() = " + collectionsize); - + if (indexrowcount != collectionsize) { + this.indexErrors++; + if (this.indexErrors == errorLimit) deleteIndexOnExit(); // delete index on exit for rebuild + serverLog.logSevere("kelondroCollectionIndex", "UPDATE (merge) ERROR: array has different chunkcount than index after merge: index = " + indexrowcount + ", collection.size() = " + collectionsize + " (error #" + indexErrors + ")"); + } index.put(indexrow); // write modified indexrow } } @@ -750,7 +757,12 @@ public class kelondroCollectionIndex { // open array entry final kelondroFixedWidthArray array = getArray(clusteridx, serialnumber, index.row().objectOrder, chunksize); final kelondroRow.Entry arrayrow = array.get(rownumber); - if (arrayrow == null) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString(), "array does not contain expected row"); + if (arrayrow == null) { + // the index appears to be corrupted + this.indexErrors++; + if (this.indexErrors == errorLimit) deleteIndexOnExit(); // delete index on exit for rebuild + throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString(), "array does not contain expected row (error #" + indexErrors + ")"); + } // read the row and define a collection final byte[] indexkey = indexrow.getColBytes(idx_col_key); @@ -759,7 +771,9 @@ public class kelondroCollectionIndex { // cleanup for a bad bug that corrupted the database index.remove(indexkey); // the RowCollection must be considered lost array.remove(rownumber); // loose the RowCollection (we don't know how much is lost) - serverLog.logSevere("kelondroCollectionIndex." + array.filename, "lost a RowCollection because of a bad arraykey"); + this.indexErrors++; + if (this.indexErrors == errorLimit) deleteIndexOnExit(); // delete index on exit for rebuild + serverLog.logSevere("kelondroCollectionIndex." + array.filename, "lost a RowCollection because of a bad arraykey (error #" + indexErrors + ")"); return new kelondroRowSet(this.payloadrow, 0); } final kelondroRowSet collection = new kelondroRowSet(this.payloadrow, arrayrow, 1); // FIXME: this does not yet work with different rowdef in case of several rowdef.objectsize() @@ -777,14 +791,18 @@ public class kelondroCollectionIndex { indexEntry.setCol(idx_col_lastread, kelondroRowCollection.daysSince2000(System.currentTimeMillis())); indexEntry.setCol(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis())); index.put(indexEntry); - serverLog.logSevere("kelondroCollectionIndex." + array.filename, "array contains wrong row '" + new String(arrayrow.getColBytes(0)) + "', expected is '" + new String(indexrow.getColBytes(idx_col_key)) + "', the row has been fixed"); + this.indexErrors++; + if (this.indexErrors == errorLimit) deleteIndexOnExit(); // delete index on exit for rebuild + serverLog.logSevere("kelondroCollectionIndex." + array.filename, "array contains wrong row '" + new String(arrayrow.getColBytes(0)) + "', expected is '" + new String(indexrow.getColBytes(idx_col_key)) + "', the row has been fixed (error #" + indexErrors + ")"); } final int chunkcountInArray = collection.size(); if (chunkcountInArray != chunkcount) { // fix the entry in index indexrow.setCol(idx_col_chunkcount, chunkcountInArray); index.put(indexrow); - array.logFailure("INCONSISTENCY (get) in " + arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString() + ": array has different chunkcount than index: index = " + chunkcount + ", array = " + chunkcountInArray + "; the index has been auto-fixed"); + this.indexErrors++; + if (this.indexErrors == errorLimit) deleteIndexOnExit(); // delete index on exit for rebuild + array.logFailure("INCONSISTENCY (get) in " + arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString() + ": array has different chunkcount than index: index = " + chunkcount + ", array = " + chunkcountInArray + "; the index has been auto-fixed (error #" + indexErrors + ")"); } if (remove) array.remove(rownumber); // index is removed in calling method return collection; diff --git a/source/de/anomic/kelondro/kelondroEcoFS.java b/source/de/anomic/kelondro/kelondroEcoFS.java index 02c738779..3247593d9 100644 --- a/source/de/anomic/kelondro/kelondroEcoFS.java +++ b/source/de/anomic/kelondro/kelondroEcoFS.java @@ -665,5 +665,9 @@ public class kelondroEcoFS { e.printStackTrace(); } } + + public void deleteOnExit() { + this.tablefile.deleteOnExit(); + } } diff --git a/source/de/anomic/kelondro/kelondroEcoTable.java b/source/de/anomic/kelondro/kelondroEcoTable.java index 8e386a751..86e2522c0 100644 --- a/source/de/anomic/kelondro/kelondroEcoTable.java +++ b/source/de/anomic/kelondro/kelondroEcoTable.java @@ -715,4 +715,8 @@ public class kelondroEcoTable implements kelondroIndex { */ } + public void deleteOnExit() { + this.file.deleteOnExit(); + } + } diff --git a/source/de/anomic/kelondro/kelondroFlexWidthArray.java b/source/de/anomic/kelondro/kelondroFlexWidthArray.java index 461302285..ac4176d96 100644 --- a/source/de/anomic/kelondro/kelondroFlexWidthArray.java +++ b/source/de/anomic/kelondro/kelondroFlexWidthArray.java @@ -417,4 +417,8 @@ public class kelondroFlexWidthArray implements kelondroArray { e.printStackTrace(); } } + + public void deleteOnExit() { + for (int i = 0; i < this.col.length; i++) this.col[i].deleteOnExit(); + } } diff --git a/source/de/anomic/kelondro/kelondroIOChunks.java b/source/de/anomic/kelondro/kelondroIOChunks.java index 5ed81f77b..7e72e5b73 100644 --- a/source/de/anomic/kelondro/kelondroIOChunks.java +++ b/source/de/anomic/kelondro/kelondroIOChunks.java @@ -56,4 +56,6 @@ public interface kelondroIOChunks { public void write(long pos, byte[] b) throws IOException; public kelondroProfile profile(); + + public void deleteOnExit(); } diff --git a/source/de/anomic/kelondro/kelondroIndex.java b/source/de/anomic/kelondro/kelondroIndex.java index 40f5b8629..229d60efc 100644 --- a/source/de/anomic/kelondro/kelondroIndex.java +++ b/source/de/anomic/kelondro/kelondroIndex.java @@ -54,6 +54,7 @@ public interface kelondroIndex { public kelondroRow.Entry removeOne() throws IOException; public kelondroCloneableIterator keys(boolean up, byte[] firstKey) throws IOException; // iterates only the key public kelondroCloneableIterator rows(boolean up, byte[] firstKey) throws IOException; // iterates the whole row + public void deleteOnExit(); public void clear() throws IOException; public void close(); } diff --git a/source/de/anomic/kelondro/kelondroRA.java b/source/de/anomic/kelondro/kelondroRA.java index 418f32fb1..b6143ca4e 100644 --- a/source/de/anomic/kelondro/kelondroRA.java +++ b/source/de/anomic/kelondro/kelondroRA.java @@ -79,4 +79,6 @@ public interface kelondroRA { public void writeArray(byte[] b) throws IOException; public byte[] readArray() throws IOException; + + public void deleteOnExit(); } diff --git a/source/de/anomic/kelondro/kelondroRAIOChunks.java b/source/de/anomic/kelondro/kelondroRAIOChunks.java index 798b545ce..7e256f065 100644 --- a/source/de/anomic/kelondro/kelondroRAIOChunks.java +++ b/source/de/anomic/kelondro/kelondroRAIOChunks.java @@ -74,4 +74,7 @@ public final class kelondroRAIOChunks extends kelondroAbstractIOChunks implement super.finalize(); } + public void deleteOnExit() { + this.ra.deleteOnExit(); + } } diff --git a/source/de/anomic/kelondro/kelondroRAMIndex.java b/source/de/anomic/kelondro/kelondroRAMIndex.java index c905d01cb..a2116a610 100644 --- a/source/de/anomic/kelondro/kelondroRAMIndex.java +++ b/source/de/anomic/kelondro/kelondroRAMIndex.java @@ -231,5 +231,9 @@ public class kelondroRAMIndex implements kelondroIndex { public String filename() { return null; // this does not have a file name } + + public void deleteOnExit() { + // do nothing, there is no file + } } diff --git a/source/de/anomic/kelondro/kelondroRowSet.java b/source/de/anomic/kelondro/kelondroRowSet.java index aa3a1fbcb..5fa16e79c 100644 --- a/source/de/anomic/kelondro/kelondroRowSet.java +++ b/source/de/anomic/kelondro/kelondroRowSet.java @@ -551,4 +551,7 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd return null; } + public void deleteOnExit() { + // do nothing, there is no file + } } diff --git a/source/de/anomic/kelondro/kelondroSQLTable.java b/source/de/anomic/kelondro/kelondroSQLTable.java index f922ff78f..529a00114 100644 --- a/source/de/anomic/kelondro/kelondroSQLTable.java +++ b/source/de/anomic/kelondro/kelondroSQLTable.java @@ -297,7 +297,10 @@ public class kelondroSQLTable implements kelondroIndex { } public void clear() { - // TODO Auto-generated method stub - + // do nothing + } + + public void deleteOnExit() { + // do nothing } } diff --git a/source/de/anomic/kelondro/kelondroSplitTable.java b/source/de/anomic/kelondro/kelondroSplitTable.java index f8ec95c72..0bb4f5561 100644 --- a/source/de/anomic/kelondro/kelondroSplitTable.java +++ b/source/de/anomic/kelondro/kelondroSplitTable.java @@ -434,5 +434,9 @@ public class kelondroSplitTable implements kelondroIndex { public static void main(final String[] args) { System.out.println(dateSuffix(new Date())); } + + public void deleteOnExit() { + for (kelondroIndex i: this.tables.values()) i.deleteOnExit(); + } } diff --git a/source/de/anomic/plasma/plasmaDHTChunk.java b/source/de/anomic/plasma/plasmaDHTChunk.java index 1cbd020be..89411d6ff 100644 --- a/source/de/anomic/plasma/plasmaDHTChunk.java +++ b/source/de/anomic/plasma/plasmaDHTChunk.java @@ -66,7 +66,6 @@ public class plasmaDHTChunk { private long selectionEndTime = 0; private int transferFailedCounter = 0; - private static int kelondroExceptionCounter = 0; public indexContainer firstContainer() { return indexContainers[0]; @@ -271,8 +270,6 @@ public class plasmaDHTChunk { indexContainers = new indexContainer[0]; urlCache = new HashMap(); this.status = chunkStatus_FAILED; - kelondroExceptionCounter++; - if (kelondroExceptionCounter > 500) wordIndex.deleteIndexOnExit(); // delete index on exit for rebuild return 0; } } diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index a5614c488..22df773bf 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -219,10 +219,6 @@ public final class plasmaWordIndex implements indexRI { queuePreStack.clear(); } - public void deleteIndexOnExit() { - collections.deleteIndexOnExit(); - } - private void initActiveCrawlProfiles() { this.defaultProxyProfile = null; this.defaultRemoteProfile = null; diff --git a/source/de/anomic/yacy/yacySearch.java b/source/de/anomic/yacy/yacySearch.java index 4fde7ae5b..4b28e78b5 100644 --- a/source/de/anomic/yacy/yacySearch.java +++ b/source/de/anomic/yacy/yacySearch.java @@ -191,7 +191,7 @@ public class yacySearch extends Thread { c = seedcount; while (dhtEnum.hasNext() && c > 0) { seed = dhtEnum.next(); - if (seed == null) continue; + if (seed == null || seed.hash == null) continue; distance = yacyDHTAction.dhtDistance(seed.hash, wordhash); if (distance > 0.2) continue; // catch bug in peer selection if (!seed.getFlagAcceptRemoteIndex()) continue; // probably a robinson peer