boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

self-healing for broken table files (may cause other problems, but better than nothing)

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6826 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent 13f5b8e7ba
commit 8c40f1cb8e
4 changed files with 57 additions and 30 deletions
Show Stats Download Patch File Download Diff File

1
source/de/anomic/crawler/Balancer.java
Unescape View File

@ -465,6 +465,7 @@ public class Balancer {
}
byte[] n = entry.getValue().getFirst();
if (n == null) continue;
if (delay) {
final long w = Latency.waitingRemainingGuessed(n, minimumLocalDelta, minimumGlobalDelta);
if (w > maximumwaiting) {

11
source/net/yacy/kelondro/io/Records.java
Unescape View File

@ -138,6 +138,17 @@ public final class Records {
return size / recordsize;
}
public final static void fixTableSize(final File tablefile, final long recordsize) throws IOException {
if (!tablefile.exists()) return;
final long size = tablefile.length();
long cut = size % recordsize;
if (cut > 0) {
RandomAccessFile raf = new RandomAccessFile(tablefile, "rw");
raf.setLength(size - cut);
raf.close();
}
}
/**
* @return the number of records in file plus number of records in buffer
* @throws IOException

14
source/net/yacy/kelondro/table/SplitTable.java
Unescape View File

@ -153,6 +153,7 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
f.renameTo(new File(path, newname));
}
}
// read new list again
tablefile = path.list();
// first pass: find tables
@ -173,14 +174,11 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
time = d.getTime();
if (time > maxtime) {
current = tablefile[i];
assert this.current != null;
maxtime = time;
}
try {
t.put(tablefile[i], Table.staticRAMIndexNeed(f, rowdef));
} catch (IOException e) {
Log.logWarning("SplitTable", "file " + f.toString() + " appears to be corrupted: " + e.getMessage());
}
t.put(tablefile[i], Table.staticRAMIndexNeed(f, rowdef));
}
}
@ -215,6 +213,7 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
}
tables.put(maxf, table);
}
assert this.current == null || this.tables.get(this.current) != null : "this.current = " + this.current;
// init the thread pool for the keeperOf executor service
this.executor = new ThreadPoolExecutor(
@ -223,8 +222,6 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
TimeUnit.SECONDS,
new LinkedBlockingQueue<Runnable>(),
new NamePrefixThreadFactory(prefix));
}
public void clear() throws IOException {
@ -317,6 +314,7 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
}
}
tables.put(this.current, table);
assert this.current == null || this.tables.get(this.current) != null : "this.current = " + this.current;
return table;
}
@ -343,6 +341,7 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
ObjectIndex keeper = keeperOf(row.getColBytes(0, true));
if (keeper != null) return keeper.replace(row);
synchronized (this.tables) {
assert this.current == null || this.tables.get(this.current) != null : "this.current = " + this.current;
keeper = (this.current == null) ? newTable() : checkTable(this.tables.get(this.current));
}
keeper.put(row);
@ -373,6 +372,7 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
assert row.objectsize() <= this.rowdef.objectsize;
ObjectIndex table = (this.current == null) ? null : tables.get(this.current);
synchronized (this.tables) {
assert this.current == null || this.tables.get(this.current) != null : "this.current = " + this.current;
if (table == null) table = newTable(); else table = checkTable(table);
}
table.addUnique(row);

61
source/net/yacy/kelondro/table/Table.java
Unescape View File

@ -70,16 +70,16 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
// static tracker objects
private final static TreeMap<String, Table> tableTracker = new TreeMap<String, Table>();
public final static long maxarraylength = 134217727L; // that may be the maxmimum size of array length in some JVMs
private final static long maxarraylength = 134217727L; // that may be the maximum size of array length in some JVMs
private final long minmemremaining; // if less than this memory is remaininig, the memory copy of a table is abandoned
private final int buffersize;
protected final Row rowdef;
protected final File tablefile;
protected final Row taildef;
protected HandleMap index;
protected BufferedRecords file;
protected RowSet table;
private final long minmemremaining; // if less than this memory is remaininig, the memory copy of a table is abandoned
private final int buffersize;
private final Row rowdef;
private final File tablefile;
private final Row taildef;
private HandleMap index;
private BufferedRecords file;
private RowSet table;
public Table(
final File tablefile,
@ -119,7 +119,7 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
try {
// open an existing table file
final int fileSize = (int) tableSize(tablefile, rowdef.objectsize);
int fileSize = (int) tableSize(tablefile, rowdef.objectsize, true);
// initialize index and copy table
final int records = Math.max(fileSize, initialSpace);
@ -260,9 +260,24 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
return this.index.largestKey();
}
public static long tableSize(final File tablefile, final int recordsize) throws IOException {
// returns number of records in table
return Records.tableSize(tablefile, recordsize);
public static long tableSize(final File tablefile, final int recordsize, boolean fixIfCorrupted) throws kelondroException {
try {
return Records.tableSize(tablefile, recordsize);
} catch (final IOException e) {
if (!fixIfCorrupted) {
Log.logSevere("Table", "table size broken for file " + tablefile.toString(), e);
throw new kelondroException(e.getMessage());
}
Log.logSevere("Table", "table size broken, try to fix " + tablefile.toString());
try {
Records.fixTableSize(tablefile, recordsize);
Log.logInfo("Table", "successfully fixed table file " + tablefile.toString());
return Records.tableSize(tablefile, recordsize);
} catch (final IOException ee) {
Log.logSevere("Table", "table size fix did not work", ee);
throw new kelondroException(e.getMessage());
}
}
}
public static final Iterator<String> filenames() {
@ -296,8 +311,8 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
return this.table != null;
}
public static long staticRAMIndexNeed(final File f, final Row rowdef) throws IOException {
return (((long)(rowdef.primaryKeyLength + 4)) * tableSize(f, rowdef.objectsize) * RowCollection.growfactorLarge100 / 100L);
public static long staticRAMIndexNeed(final File f, final Row rowdef) {
return (((long)(rowdef.primaryKeyLength + 4)) * tableSize(f, rowdef.objectsize, true) * RowCollection.growfactorLarge100 / 100L);
}
public synchronized void addUnique(final Entry row) throws IOException, RowSpaceExceededException {
@ -757,8 +772,8 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
return new rowIteratorNoOrder();
}
public class rowIteratorNoOrder implements CloneableIterator<Entry> {
final Iterator<byte[]> ri;
private class rowIteratorNoOrder implements CloneableIterator<Entry> {
private final Iterator<byte[]> ri;
public rowIteratorNoOrder() throws IOException {
ri = new ChunkIterator(tablefile, rowdef.objectsize, rowdef.objectsize);
@ -792,13 +807,13 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
return new rowIterator(up, firstKey);
}
public class rowIterator implements CloneableIterator<Entry> {
Iterator<byte[]> i;
boolean up;
byte[] fk;
int c;
private class rowIterator implements CloneableIterator<Entry> {
private Iterator<byte[]> i;
private boolean up;
private byte[] fk;
private int c;
public rowIterator(final boolean up, final byte[] firstKey) {
private rowIterator(final boolean up, final byte[] firstKey) {
this.up = up;
this.fk = firstKey;
this.i = index.keys(up, firstKey);

Write Preview
Loading…
Cancel
Save