From 8c40f1cb8eaebb120bf59e9088c40927ddb96262 Mon Sep 17 00:00:00 2001 From: orbiter Date: Wed, 21 Apr 2010 11:29:27 +0000 Subject: [PATCH] self-healing for broken table files (may cause other problems, but better than nothing) git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6826 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/de/anomic/crawler/Balancer.java | 1 + source/net/yacy/kelondro/io/Records.java | 11 ++++ .../net/yacy/kelondro/table/SplitTable.java | 14 ++--- source/net/yacy/kelondro/table/Table.java | 61 ++++++++++++------- 4 files changed, 57 insertions(+), 30 deletions(-) diff --git a/source/de/anomic/crawler/Balancer.java b/source/de/anomic/crawler/Balancer.java index 939b02e0e..1ae48eef4 100644 --- a/source/de/anomic/crawler/Balancer.java +++ b/source/de/anomic/crawler/Balancer.java @@ -465,6 +465,7 @@ public class Balancer { } byte[] n = entry.getValue().getFirst(); + if (n == null) continue; if (delay) { final long w = Latency.waitingRemainingGuessed(n, minimumLocalDelta, minimumGlobalDelta); if (w > maximumwaiting) { diff --git a/source/net/yacy/kelondro/io/Records.java b/source/net/yacy/kelondro/io/Records.java index bfb91fff1..699a13772 100644 --- a/source/net/yacy/kelondro/io/Records.java +++ b/source/net/yacy/kelondro/io/Records.java @@ -138,6 +138,17 @@ public final class Records { return size / recordsize; } + public final static void fixTableSize(final File tablefile, final long recordsize) throws IOException { + if (!tablefile.exists()) return; + final long size = tablefile.length(); + long cut = size % recordsize; + if (cut > 0) { + RandomAccessFile raf = new RandomAccessFile(tablefile, "rw"); + raf.setLength(size - cut); + raf.close(); + } + } + /** * @return the number of records in file plus number of records in buffer * @throws IOException diff --git a/source/net/yacy/kelondro/table/SplitTable.java b/source/net/yacy/kelondro/table/SplitTable.java index 4fdbc4967..9318c3e32 100644 --- a/source/net/yacy/kelondro/table/SplitTable.java +++ b/source/net/yacy/kelondro/table/SplitTable.java @@ -153,6 +153,7 @@ public class SplitTable implements ObjectIndex, Iterable { f.renameTo(new File(path, newname)); } } + // read new list again tablefile = path.list(); // first pass: find tables @@ -173,14 +174,11 @@ public class SplitTable implements ObjectIndex, Iterable { time = d.getTime(); if (time > maxtime) { current = tablefile[i]; + assert this.current != null; maxtime = time; } - try { - t.put(tablefile[i], Table.staticRAMIndexNeed(f, rowdef)); - } catch (IOException e) { - Log.logWarning("SplitTable", "file " + f.toString() + " appears to be corrupted: " + e.getMessage()); - } + t.put(tablefile[i], Table.staticRAMIndexNeed(f, rowdef)); } } @@ -215,6 +213,7 @@ public class SplitTable implements ObjectIndex, Iterable { } tables.put(maxf, table); } + assert this.current == null || this.tables.get(this.current) != null : "this.current = " + this.current; // init the thread pool for the keeperOf executor service this.executor = new ThreadPoolExecutor( @@ -223,8 +222,6 @@ public class SplitTable implements ObjectIndex, Iterable { TimeUnit.SECONDS, new LinkedBlockingQueue(), new NamePrefixThreadFactory(prefix)); - - } public void clear() throws IOException { @@ -317,6 +314,7 @@ public class SplitTable implements ObjectIndex, Iterable { } } tables.put(this.current, table); + assert this.current == null || this.tables.get(this.current) != null : "this.current = " + this.current; return table; } @@ -343,6 +341,7 @@ public class SplitTable implements ObjectIndex, Iterable { ObjectIndex keeper = keeperOf(row.getColBytes(0, true)); if (keeper != null) return keeper.replace(row); synchronized (this.tables) { + assert this.current == null || this.tables.get(this.current) != null : "this.current = " + this.current; keeper = (this.current == null) ? newTable() : checkTable(this.tables.get(this.current)); } keeper.put(row); @@ -373,6 +372,7 @@ public class SplitTable implements ObjectIndex, Iterable { assert row.objectsize() <= this.rowdef.objectsize; ObjectIndex table = (this.current == null) ? null : tables.get(this.current); synchronized (this.tables) { + assert this.current == null || this.tables.get(this.current) != null : "this.current = " + this.current; if (table == null) table = newTable(); else table = checkTable(table); } table.addUnique(row); diff --git a/source/net/yacy/kelondro/table/Table.java b/source/net/yacy/kelondro/table/Table.java index 3b91cc789..b02d37d06 100644 --- a/source/net/yacy/kelondro/table/Table.java +++ b/source/net/yacy/kelondro/table/Table.java @@ -70,16 +70,16 @@ public class Table implements ObjectIndex, Iterable { // static tracker objects private final static TreeMap tableTracker = new TreeMap(); - public final static long maxarraylength = 134217727L; // that may be the maxmimum size of array length in some JVMs + private final static long maxarraylength = 134217727L; // that may be the maximum size of array length in some JVMs - private final long minmemremaining; // if less than this memory is remaininig, the memory copy of a table is abandoned - private final int buffersize; - protected final Row rowdef; - protected final File tablefile; - protected final Row taildef; - protected HandleMap index; - protected BufferedRecords file; - protected RowSet table; + private final long minmemremaining; // if less than this memory is remaininig, the memory copy of a table is abandoned + private final int buffersize; + private final Row rowdef; + private final File tablefile; + private final Row taildef; + private HandleMap index; + private BufferedRecords file; + private RowSet table; public Table( final File tablefile, @@ -119,7 +119,7 @@ public class Table implements ObjectIndex, Iterable { try { // open an existing table file - final int fileSize = (int) tableSize(tablefile, rowdef.objectsize); + int fileSize = (int) tableSize(tablefile, rowdef.objectsize, true); // initialize index and copy table final int records = Math.max(fileSize, initialSpace); @@ -260,9 +260,24 @@ public class Table implements ObjectIndex, Iterable { return this.index.largestKey(); } - public static long tableSize(final File tablefile, final int recordsize) throws IOException { - // returns number of records in table - return Records.tableSize(tablefile, recordsize); + public static long tableSize(final File tablefile, final int recordsize, boolean fixIfCorrupted) throws kelondroException { + try { + return Records.tableSize(tablefile, recordsize); + } catch (final IOException e) { + if (!fixIfCorrupted) { + Log.logSevere("Table", "table size broken for file " + tablefile.toString(), e); + throw new kelondroException(e.getMessage()); + } + Log.logSevere("Table", "table size broken, try to fix " + tablefile.toString()); + try { + Records.fixTableSize(tablefile, recordsize); + Log.logInfo("Table", "successfully fixed table file " + tablefile.toString()); + return Records.tableSize(tablefile, recordsize); + } catch (final IOException ee) { + Log.logSevere("Table", "table size fix did not work", ee); + throw new kelondroException(e.getMessage()); + } + } } public static final Iterator filenames() { @@ -296,8 +311,8 @@ public class Table implements ObjectIndex, Iterable { return this.table != null; } - public static long staticRAMIndexNeed(final File f, final Row rowdef) throws IOException { - return (((long)(rowdef.primaryKeyLength + 4)) * tableSize(f, rowdef.objectsize) * RowCollection.growfactorLarge100 / 100L); + public static long staticRAMIndexNeed(final File f, final Row rowdef) { + return (((long)(rowdef.primaryKeyLength + 4)) * tableSize(f, rowdef.objectsize, true) * RowCollection.growfactorLarge100 / 100L); } public synchronized void addUnique(final Entry row) throws IOException, RowSpaceExceededException { @@ -757,8 +772,8 @@ public class Table implements ObjectIndex, Iterable { return new rowIteratorNoOrder(); } - public class rowIteratorNoOrder implements CloneableIterator { - final Iterator ri; + private class rowIteratorNoOrder implements CloneableIterator { + private final Iterator ri; public rowIteratorNoOrder() throws IOException { ri = new ChunkIterator(tablefile, rowdef.objectsize, rowdef.objectsize); @@ -792,13 +807,13 @@ public class Table implements ObjectIndex, Iterable { return new rowIterator(up, firstKey); } - public class rowIterator implements CloneableIterator { - Iterator i; - boolean up; - byte[] fk; - int c; + private class rowIterator implements CloneableIterator { + private Iterator i; + private boolean up; + private byte[] fk; + private int c; - public rowIterator(final boolean up, final byte[] firstKey) { + private rowIterator(final boolean up, final byte[] firstKey) { this.up = up; this.fk = firstKey; this.i = index.keys(up, firstKey);