From 7cc4cec9c9c30f9d989a2a64d5b89272f6227d24 Mon Sep 17 00:00:00 2001 From: orbiter Date: Fri, 1 Dec 2006 01:30:05 +0000 Subject: [PATCH] bugfix for assertion bugs documented in http://www.yacy-forum.de/viewtopic.php?p=28261#28261 git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3030 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/de/anomic/index/indexContainerOrder.java | 8 ++++++++ .../de/anomic/kelondro/kelondroBase64Order.java | 14 ++++++++++++++ .../anomic/kelondro/kelondroCollectionIndex.java | 16 ++++++++++++---- .../de/anomic/kelondro/kelondroNaturalOrder.java | 7 +++++++ source/de/anomic/kelondro/kelondroOrder.java | 3 +++ .../anomic/kelondro/kelondroRowCollection.java | 2 ++ source/de/anomic/kelondro/kelondroRowSet.java | 12 ++++++++++-- source/de/anomic/plasma/plasmaSwitchboard.java | 5 +++-- 8 files changed, 59 insertions(+), 8 deletions(-) diff --git a/source/de/anomic/index/indexContainerOrder.java b/source/de/anomic/index/indexContainerOrder.java index e39872a60..c68a61ecf 100644 --- a/source/de/anomic/index/indexContainerOrder.java +++ b/source/de/anomic/index/indexContainerOrder.java @@ -10,6 +10,14 @@ public class indexContainerOrder implements kelondroOrder { this.embeddedOrder = embedOrder; } + public boolean wellformed(byte[] a) { + return embeddedOrder.wellformed(a); + } + + public boolean wellformed(byte[] a, int astart, int alength) { + return embeddedOrder.wellformed(a, astart, alength); + } + public void direction(boolean ascending) { this.embeddedOrder.direction(ascending); } diff --git a/source/de/anomic/kelondro/kelondroBase64Order.java b/source/de/anomic/kelondro/kelondroBase64Order.java index 15ccd7b9e..2ab6c9b76 100644 --- a/source/de/anomic/kelondro/kelondroBase64Order.java +++ b/source/de/anomic/kelondro/kelondroBase64Order.java @@ -90,6 +90,20 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond return o; } + public final boolean wellformed(byte[] a) { + return wellformed(a, 0, a.length); + } + + public final boolean wellformed(byte[] a, int astart, int alength) { + assert (astart + alength <= a.length) : "astart = " + astart + ", alength = " + alength + ", a.length = " + a.length; + int b; + for (int i = astart + alength - 1; i >= astart; i--) { + b = a[i]; + if ((b < 0) || (b >= 128) || (ahpla[b] == -1)) return false; + } + return true; + } + public final static kelondroOrder bySignature(String signature) { if (signature.equals("Bd")) return new kelondroBase64Order(false, false); if (signature.equals("bd")) return new kelondroBase64Order(false, true); diff --git a/source/de/anomic/kelondro/kelondroCollectionIndex.java b/source/de/anomic/kelondro/kelondroCollectionIndex.java index de4584c18..9d90484bf 100644 --- a/source/de/anomic/kelondro/kelondroCollectionIndex.java +++ b/source/de/anomic/kelondro/kelondroCollectionIndex.java @@ -442,11 +442,19 @@ public class kelondroCollectionIndex { if (arrayrow == null) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString(), "array does not contain expected row"); // read the row and define a collection + byte[] indexkey = indexrow.getColBytes(idx_col_key); + byte[] arraykey = arrayrow.getColBytes(0); + if (!(index.order().wellformed(arraykey))) { + // cleanup for a bad bug that corrupted the database + index.remove(indexkey); // the RowCollection must be considered lost + array.remove(rownumber); // loose the RowCollection (we don't know how much is lost) + serverLog.logSevere("kelondroCollectionIndex." + array.filename, "lost a RowCollection because of a bad arraykey"); + return new kelondroRowSet(this.payloadrow, 0); + } kelondroRowSet collection = new kelondroRowSet(this.payloadrow, arrayrow.getColBytes(1)); // FIXME: this does not yet work with different rowdef in case of several rowdef.objectsize() - byte[] key = indexrow.getColBytes(idx_col_key); - if (index.order().compare(arrayrow.getColBytes(0), key) != 0) { + if ((!(index.order().wellformed(indexkey))) || (index.order().compare(arraykey, indexkey) != 0)) { // check if we got the right row; this row is wrong. Fix it: - index.remove(key); // the wrong row cannot be fixed + index.remove(indexkey); // the wrong row cannot be fixed // store the row number in the index; this may be a double-entry, but better than nothing kelondroRow.Entry indexEntry = index.row().newEntry(); indexEntry.setCol(idx_col_key, arrayrow.getColBytes(0)); @@ -458,7 +466,7 @@ public class kelondroCollectionIndex { indexEntry.setCol(idx_col_lastread, kelondroRowCollection.daysSince2000(System.currentTimeMillis())); indexEntry.setCol(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis())); index.put(indexEntry); - throw new kelondroException(array.filename, "array contains wrong row '" + new String(arrayrow.getColBytes(0)) + "', expected is '" + new String(indexrow.getColBytes(idx_col_key)) + "', the row has been fixed"); + serverLog.logSevere("kelondroCollectionIndex." + array.filename, "array contains wrong row '" + new String(arrayrow.getColBytes(0)) + "', expected is '" + new String(indexrow.getColBytes(idx_col_key)) + "', the row has been fixed"); } int chunkcountInArray = collection.size(); if (chunkcountInArray != chunkcount) { diff --git a/source/de/anomic/kelondro/kelondroNaturalOrder.java b/source/de/anomic/kelondro/kelondroNaturalOrder.java index b5f67ca67..cd4f188f7 100644 --- a/source/de/anomic/kelondro/kelondroNaturalOrder.java +++ b/source/de/anomic/kelondro/kelondroNaturalOrder.java @@ -55,6 +55,13 @@ public class kelondroNaturalOrder extends kelondroAbstractOrder implements kelon this.asc = ascending; this.zero = null; } + public boolean wellformed(byte[] a) { + return true; + } + + public boolean wellformed(byte[] a, int astart, int alength) { + return true; + } public final Object clone() { kelondroNaturalOrder o = new kelondroNaturalOrder(this.asc); diff --git a/source/de/anomic/kelondro/kelondroOrder.java b/source/de/anomic/kelondro/kelondroOrder.java index 4fa0c1f10..b4b2aa345 100644 --- a/source/de/anomic/kelondro/kelondroOrder.java +++ b/source/de/anomic/kelondro/kelondroOrder.java @@ -49,6 +49,9 @@ import java.util.Comparator; public interface kelondroOrder extends Comparator { + public boolean wellformed(byte[] a); // returns true if and only if a has only characters that belong to the implemented order + public boolean wellformed(byte[] a, int astart, int alength); + public Object clone(); public void direction(boolean ascending); // the ordering direction can be changed at any time diff --git a/source/de/anomic/kelondro/kelondroRowCollection.java b/source/de/anomic/kelondro/kelondroRowCollection.java index e35249105..ee0bee24a 100644 --- a/source/de/anomic/kelondro/kelondroRowCollection.java +++ b/source/de/anomic/kelondro/kelondroRowCollection.java @@ -496,6 +496,8 @@ public class kelondroRowCollection { assert (this.sortColumn == 0) : "this.sortColumn = " + this.sortColumn; int keylength = this.rowdef.width(this.sortColumn); int colstart = this.rowdef.colstart[this.sortColumn]; + if (bugappearance(chunkcache, i * this.rowdef.objectsize() + colstart, keylength)) throw new kelondroException("bugappearance i"); + if (bugappearance(chunkcache, j * this.rowdef.objectsize() + colstart, keylength)) throw new kelondroException("bugappearance j"); int c = this.sortOrder.compare( chunkcache, i * this.rowdef.objectsize() + colstart, diff --git a/source/de/anomic/kelondro/kelondroRowSet.java b/source/de/anomic/kelondro/kelondroRowSet.java index 594d01e08..3c2fca99c 100644 --- a/source/de/anomic/kelondro/kelondroRowSet.java +++ b/source/de/anomic/kelondro/kelondroRowSet.java @@ -29,6 +29,8 @@ import java.util.Iterator; import java.util.Random; import java.util.TreeMap; +import de.anomic.server.logging.serverLog; + public class kelondroRowSet extends kelondroRowCollection implements kelondroIndex { private static final int collectionReSortLimit = 90; @@ -171,8 +173,14 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd public void shape() { assert (this.sortOrder != null); // we cannot shape without an object order synchronized (chunkcache) { - resolveMarkedRemoved(); - super.sort(); + try { + resolveMarkedRemoved(); + super.sort(); + } catch (kelondroException e) { + // bad bug, cannot be fixed. We abandon all data + serverLog.logSevere("kelondroRowSet", "abandoned row"); + this.clear(); + } //if (super.rowdef.column(0).cellwidth() == 4) System.out.println("TABLE OF " + super.rowdef.toString() + "\n" + serverLog.table(super.chunkcache, super.rowdef.objectsize, 0)); // DEBUG } } diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 65866e279..b4291f3f1 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -2389,8 +2389,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser if (wordIndex.size() < 100) { return "no DHT distribution: not enough words - wordIndex.size() = " + wordIndex.size(); } - if ((getConfig("allowDistributeIndexWhileCrawling","false").equalsIgnoreCase("false")) && (urlPool.noticeURL.stackSize() > 0)) { - return "no DHT distribution: crawl in progress - noticeURL.stackSize() = " + urlPool.noticeURL.stackSize(); + if ((getConfig("allowDistributeIndexWhileCrawling","false").equalsIgnoreCase("false")) && + ((urlPool.noticeURL.stackSize() > 0) || (sbQueue.size() > 0))) { + return "no DHT distribution: crawl in progress: noticeURL.stackSize() = " + urlPool.noticeURL.stackSize() + ", sbQueue.size() = " + sbQueue.size(); } return null; }