boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

better self-healing of corrupted databases

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5406 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent 963da8c3f9
commit fc8189f3fb
2 changed files with 39 additions and 21 deletions
Show Stats Download Patch File Download Diff File

26
source/de/anomic/kelondro/kelondroAbstractRecords.java
Unescape View File

@ -219,7 +219,7 @@ public abstract class kelondroAbstractRecords implements kelondroRecords {
if (USAGE.FREEC == 0) {
// generate new entry
final int index = USAGE.allCount();
entryFile.write(seekpos(index) + overhead, chunk, 0, ROW.objectsize); // occupy space, othervise the USAGE computaton does not work
entryFile.write(seekpos(index) + overhead, chunk, 0, ROW.objectsize); // occupy space, otherwise the USAGE computation does not work
USAGE.USEDC++;
writeused(false);
return index;
@ -230,13 +230,14 @@ public abstract class kelondroAbstractRecords implements kelondroRecords {
// take link
int index = USAGE.FREEH.index;
if (index == kelondroHandle.NUL) {
serverLog.logSevere("kelondroTray/" + filename, "INTERNAL ERROR (DATA INCONSISTENCY): re-use of records failed, lost " + (USAGE.FREEC + 1) + " records.");
serverLog.logSevere("kelondroAbstractRecords/" + filename, "INTERNAL ERROR (DATA INCONSISTENCY): re-use of records failed, lost " + (USAGE.FREEC + 1) + " records.");
// try to heal..
USAGE.USEDC = USAGE.allCount() + 1;
USAGE.USEDC = (int) ((entryFile.length() - POS_NODES) / recordsize);
index = USAGE.USEDC;
USAGE.USEDC++;
USAGE.FREEC = 0;
index = USAGE.USEDC - 1;
//entryFile.write(seekpos(index) + overhead, spaceChunk, 0, ROW.objectsize); // overwrite space
} else {
//System.out.println("*DEBUG* ALLOCATED DELETED INDEX " + index);
// check for valid seek position
final long seekp = seekpos(USAGE.FREEH);
if (seekp >= entryFile.length()) {
@ -249,12 +250,23 @@ public abstract class kelondroAbstractRecords implements kelondroRecords {
} else {
// read link to next element of FREEH chain
USAGE.FREEH.index = entryFile.readInt(seekp);
assert ((USAGE.FREEH.index == kelondroHandle.NUL) && (USAGE.FREEC == 0)) || seekpos(USAGE.FREEH) < entryFile.length() : "allocatePayload: USAGE.FREEH.index = " + USAGE.FREEH.index + ", entryFile.length() = " + entryFile.length();
// check consistency
if (((USAGE.FREEH.index != kelondroHandle.NUL) || (USAGE.FREEC != 0)) && seekpos(USAGE.FREEH) >= entryFile.length()) {
// the FREEH pointer cannot be correct, because it points to a place outside of the file.
// to correct this, we reset the FREH pointer and return a index that has been calculated as if USAGE.FREE == 0
serverLog.logSevere("kelondroAbstractRecords/" + filename, "INTERNAL ERROR (DATA INCONSISTENCY): USAGE.FREEH.index = " + USAGE.FREEH.index + ", entryFile.length() = " + entryFile.length() + "; wrong FREEH has been patched, lost " + (USAGE.FREEC + 1) + " records.");
// try to heal..
USAGE.USEDC = (int) ((entryFile.length() - POS_NODES) / recordsize);
index = USAGE.USEDC;
USAGE.USEDC++;
USAGE.FREEC = 0;
//entryFile.write(seekpos(index) + overhead, spaceChunk, 0, ROW.objectsize); // overwrite space
}
}
}
entryFile.write(seekpos(index) + overhead, chunk, 0, ROW.objectsize); // overwrite space
USAGE.writeused(false);
USAGE.writefree();
entryFile.write(seekpos(index) + overhead, chunk, 0, ROW.objectsize); // overwrite space
return index;
}
//}

34
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -373,20 +373,26 @@ public class kelondroCollectionIndex {
arrayEntry.setCol(1, collection.exportCollection());
// write a new entry in this array
final int newRowNumber = array.add(arrayEntry);
try {
final int newRowNumber = array.add(arrayEntry);
// store the new row number in the index
indexrow.setCol(idx_col_key, key);
indexrow.setCol(idx_col_chunksize, this.payloadrow.objectsize);
indexrow.setCol(idx_col_chunkcount, collection.size());
indexrow.setCol(idx_col_clusteridx, (byte) partitionNumber);
indexrow.setCol(idx_col_flags, (byte) 0);
indexrow.setCol(idx_col_indexpos, newRowNumber);
indexrow.setCol(idx_col_lastread, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
indexrow.setCol(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
// store the new row number in the index
indexrow.setCol(idx_col_key, key);
indexrow.setCol(idx_col_chunksize, this.payloadrow.objectsize);
indexrow.setCol(idx_col_chunkcount, collection.size());
indexrow.setCol(idx_col_clusteridx, (byte) partitionNumber);
indexrow.setCol(idx_col_flags, (byte) 0);
indexrow.setCol(idx_col_indexpos, newRowNumber);
indexrow.setCol(idx_col_lastread, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
indexrow.setCol(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
// after calling this method there must be an index.addUnique(indexrow);
return indexrow;
// after calling this method there must be an index.addUnique(indexrow);
return indexrow;
} catch (Exception e) {
// the index appears to be corrupted at a particular point
serverLog.logWarning("kelondroCollectionIndex", "array " + arrayFile(this.path, this.filenameStub, this.loadfactor, this.payloadrow.objectsize, partitionNumber, serialNumber).toString() + " has errors \"" + e.getMessage() + "\" (error #" + indexErrors + ")");
return null;
}
}
private void array_add(
@ -451,7 +457,7 @@ public class kelondroCollectionIndex {
// create new row and index entry
if ((collection != null) && (collection.size() > 0)) {
indexrow = array_new(key, collection); // modifies indexrow
index.addUnique(indexrow);
if (indexrow != null) index.addUnique(indexrow);
}
return;
}
@ -497,7 +503,7 @@ public class kelondroCollectionIndex {
kelondroRow.Entry indexrow = index.get(key);
if (indexrow == null) {
indexrow = array_new(key, container); // modifies indexrow
index.addUnique(indexrow); // write modified indexrow
if (indexrow != null) index.addUnique(indexrow); // write modified indexrow
} else {
// merge with the old collection
// attention! this modifies the indexrow entry which must be written with index.put(indexrow) afterwards!

Write Preview
Loading…
Cancel
Save