boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

better self-healing of corrupted databases

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5406 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent 963da8c3f9
commit fc8189f3fb
2 changed files with 39 additions and 21 deletions
Show Stats Download Patch File Download Diff File

26
source/de/anomic/kelondro/kelondroAbstractRecords.java
Unescape View File

@ -219,7 +219,7 @@ public abstract class kelondroAbstractRecords implements kelondroRecords {
if (USAGE.FREEC == 0) { if (USAGE.FREEC == 0) {
// generate new entry // generate new entry
final int index = USAGE.allCount(); final int index = USAGE.allCount();
entryFile.write(seekpos(index) + overhead, chunk, 0, ROW.objectsize); // occupy space, othervise the USAGE computaton does not work entryFile.write(seekpos(index) + overhead, chunk, 0, ROW.objectsize); // occupy space, otherwise the USAGE computation does not work
USAGE.USEDC++; USAGE.USEDC++;
writeused(false); writeused(false);
return index; return index;
@ -230,13 +230,14 @@ public abstract class kelondroAbstractRecords implements kelondroRecords {
// take link // take link
int index = USAGE.FREEH.index; int index = USAGE.FREEH.index;
if (index == kelondroHandle.NUL) { if (index == kelondroHandle.NUL) {
serverLog.logSevere("kelondroTray/" + filename, "INTERNAL ERROR (DATA INCONSISTENCY): re-use of records failed, lost " + (USAGE.FREEC + 1) + " records."); serverLog.logSevere("kelondroAbstractRecords/" + filename, "INTERNAL ERROR (DATA INCONSISTENCY): re-use of records failed, lost " + (USAGE.FREEC + 1) + " records.");
// try to heal.. // try to heal..
USAGE.USEDC = USAGE.allCount() + 1; USAGE.USEDC = (int) ((entryFile.length() - POS_NODES) / recordsize);
index = USAGE.USEDC;
USAGE.USEDC++;
USAGE.FREEC = 0; USAGE.FREEC = 0;
index = USAGE.USEDC - 1; //entryFile.write(seekpos(index) + overhead, spaceChunk, 0, ROW.objectsize); // overwrite space
} else { } else {
//System.out.println("*DEBUG* ALLOCATED DELETED INDEX " + index);
// check for valid seek position // check for valid seek position
final long seekp = seekpos(USAGE.FREEH); final long seekp = seekpos(USAGE.FREEH);
if (seekp >= entryFile.length()) { if (seekp >= entryFile.length()) {
@ -249,12 +250,23 @@ public abstract class kelondroAbstractRecords implements kelondroRecords {
} else { } else {
// read link to next element of FREEH chain // read link to next element of FREEH chain
USAGE.FREEH.index = entryFile.readInt(seekp); USAGE.FREEH.index = entryFile.readInt(seekp);
assert ((USAGE.FREEH.index == kelondroHandle.NUL) && (USAGE.FREEC == 0)) || seekpos(USAGE.FREEH) < entryFile.length() : "allocatePayload: USAGE.FREEH.index = " + USAGE.FREEH.index + ", entryFile.length() = " + entryFile.length(); // check consistency
if (((USAGE.FREEH.index != kelondroHandle.NUL) || (USAGE.FREEC != 0)) && seekpos(USAGE.FREEH) >= entryFile.length()) {
// the FREEH pointer cannot be correct, because it points to a place outside of the file.
// to correct this, we reset the FREH pointer and return a index that has been calculated as if USAGE.FREE == 0
serverLog.logSevere("kelondroAbstractRecords/" + filename, "INTERNAL ERROR (DATA INCONSISTENCY): USAGE.FREEH.index = " + USAGE.FREEH.index + ", entryFile.length() = " + entryFile.length() + "; wrong FREEH has been patched, lost " + (USAGE.FREEC + 1) + " records.");
// try to heal..
USAGE.USEDC = (int) ((entryFile.length() - POS_NODES) / recordsize);
index = USAGE.USEDC;
USAGE.USEDC++;
USAGE.FREEC = 0;
//entryFile.write(seekpos(index) + overhead, spaceChunk, 0, ROW.objectsize); // overwrite space
}
} }
} }
entryFile.write(seekpos(index) + overhead, chunk, 0, ROW.objectsize); // overwrite space
USAGE.writeused(false); USAGE.writeused(false);
USAGE.writefree(); USAGE.writefree();
entryFile.write(seekpos(index) + overhead, chunk, 0, ROW.objectsize); // overwrite space
return index; return index;
} }
//} //}

34
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -373,20 +373,26 @@ public class kelondroCollectionIndex {
arrayEntry.setCol(1, collection.exportCollection()); arrayEntry.setCol(1, collection.exportCollection());
// write a new entry in this array // write a new entry in this array
final int newRowNumber = array.add(arrayEntry); try {
final int newRowNumber = array.add(arrayEntry);
// store the new row number in the index // store the new row number in the index
indexrow.setCol(idx_col_key, key); indexrow.setCol(idx_col_key, key);
indexrow.setCol(idx_col_chunksize, this.payloadrow.objectsize); indexrow.setCol(idx_col_chunksize, this.payloadrow.objectsize);
indexrow.setCol(idx_col_chunkcount, collection.size()); indexrow.setCol(idx_col_chunkcount, collection.size());
indexrow.setCol(idx_col_clusteridx, (byte) partitionNumber); indexrow.setCol(idx_col_clusteridx, (byte) partitionNumber);
indexrow.setCol(idx_col_flags, (byte) 0); indexrow.setCol(idx_col_flags, (byte) 0);
indexrow.setCol(idx_col_indexpos, newRowNumber); indexrow.setCol(idx_col_indexpos, newRowNumber);
indexrow.setCol(idx_col_lastread, kelondroRowCollection.daysSince2000(System.currentTimeMillis())); indexrow.setCol(idx_col_lastread, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
indexrow.setCol(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis())); indexrow.setCol(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
// after calling this method there must be an index.addUnique(indexrow); // after calling this method there must be an index.addUnique(indexrow);
return indexrow; return indexrow;
} catch (Exception e) {
// the index appears to be corrupted at a particular point
serverLog.logWarning("kelondroCollectionIndex", "array " + arrayFile(this.path, this.filenameStub, this.loadfactor, this.payloadrow.objectsize, partitionNumber, serialNumber).toString() + " has errors \"" + e.getMessage() + "\" (error #" + indexErrors + ")");
return null;
}
} }
private void array_add( private void array_add(
@ -451,7 +457,7 @@ public class kelondroCollectionIndex {
// create new row and index entry // create new row and index entry
if ((collection != null) && (collection.size() > 0)) { if ((collection != null) && (collection.size() > 0)) {
indexrow = array_new(key, collection); // modifies indexrow indexrow = array_new(key, collection); // modifies indexrow
index.addUnique(indexrow); if (indexrow != null) index.addUnique(indexrow);
} }
return; return;
} }
@ -497,7 +503,7 @@ public class kelondroCollectionIndex {
kelondroRow.Entry indexrow = index.get(key); kelondroRow.Entry indexrow = index.get(key);
if (indexrow == null) { if (indexrow == null) {
indexrow = array_new(key, container); // modifies indexrow indexrow = array_new(key, container); // modifies indexrow
index.addUnique(indexrow); // write modified indexrow if (indexrow != null) index.addUnique(indexrow); // write modified indexrow
} else { } else {
// merge with the old collection // merge with the old collection
// attention! this modifies the indexrow entry which must be written with index.put(indexrow) afterwards! // attention! this modifies the indexrow entry which must be written with index.put(indexrow) afterwards!

Write Preview
Loading…
Cancel
Save