bugfix for assertion bugs documented in

http://www.yacy-forum.de/viewtopic.php?p=28261#28261

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3030 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/index/indexContainerOrder.java

@@ -10,6 +10,14 @@ public class indexContainerOrder implements kelondroOrder {
         this.embeddedOrder = embedOrder;
     }
 
+    public boolean wellformed(byte[] a) {
+        return embeddedOrder.wellformed(a);
+    }
+    
+    public boolean wellformed(byte[] a, int astart, int alength) {
+        return embeddedOrder.wellformed(a, astart, alength);
+    }
+    
     public void direction(boolean ascending) {
         this.embeddedOrder.direction(ascending);
     }

source/de/anomic/kelondro/kelondroBase64Order.java

@@ -90,6 +90,20 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
         return o;
     }
     
+    public final boolean wellformed(byte[] a) {
+        return wellformed(a, 0, a.length);
+    }
+    
+    public final boolean wellformed(byte[] a, int astart, int alength) {
+        assert (astart + alength <= a.length) : "astart = " + astart + ", alength = " + alength + ", a.length = " + a.length;
+        int b;
+        for (int i = astart + alength - 1; i >= astart; i--) {
+            b = a[i];
+            if ((b < 0) || (b >= 128) || (ahpla[b] == -1)) return false;
+        }
+        return true;
+    }
+    
     public final static kelondroOrder bySignature(String signature) {
         if (signature.equals("Bd")) return new kelondroBase64Order(false, false);
         if (signature.equals("bd")) return new kelondroBase64Order(false, true);

source/de/anomic/kelondro/kelondroCollectionIndex.java

@@ -442,11 +442,19 @@ public class kelondroCollectionIndex {
         if (arrayrow == null) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString(), "array does not contain expected row");
 
         // read the row and define a collection
+        byte[] indexkey = indexrow.getColBytes(idx_col_key);
+        byte[] arraykey = arrayrow.getColBytes(0);
+        if (!(index.order().wellformed(arraykey))) {
+            // cleanup for a bad bug that corrupted the database
+            index.remove(indexkey);  // the RowCollection must be considered lost
+            array.remove(rownumber); // loose the RowCollection (we don't know how much is lost)
+            serverLog.logSevere("kelondroCollectionIndex." + array.filename, "lost a RowCollection because of a bad arraykey");
+            return new kelondroRowSet(this.payloadrow, 0);
+        }
         kelondroRowSet collection = new kelondroRowSet(this.payloadrow, arrayrow.getColBytes(1)); // FIXME: this does not yet work with different rowdef in case of several rowdef.objectsize()
-        byte[] key = indexrow.getColBytes(idx_col_key);
-        if (index.order().compare(arrayrow.getColBytes(0), key) != 0) {
+        if ((!(index.order().wellformed(indexkey))) || (index.order().compare(arraykey, indexkey) != 0)) {
             // check if we got the right row; this row is wrong. Fix it:
-            index.remove(key); // the wrong row cannot be fixed
+            index.remove(indexkey); // the wrong row cannot be fixed
             // store the row number in the index; this may be a double-entry, but better than nothing
             kelondroRow.Entry indexEntry = index.row().newEntry();
             indexEntry.setCol(idx_col_key, arrayrow.getColBytes(0));
@@ -458,7 +466,7 @@ public class kelondroCollectionIndex {
             indexEntry.setCol(idx_col_lastread, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
             indexEntry.setCol(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
             index.put(indexEntry);
-            throw new kelondroException(array.filename, "array contains wrong row '" + new String(arrayrow.getColBytes(0)) + "', expected is '" + new String(indexrow.getColBytes(idx_col_key)) + "', the row has been fixed");
+            serverLog.logSevere("kelondroCollectionIndex." + array.filename, "array contains wrong row '" + new String(arrayrow.getColBytes(0)) + "', expected is '" + new String(indexrow.getColBytes(idx_col_key)) + "', the row has been fixed");
         }
         int chunkcountInArray = collection.size();
         if (chunkcountInArray != chunkcount) {

source/de/anomic/kelondro/kelondroNaturalOrder.java

@@ -55,6 +55,13 @@ public class kelondroNaturalOrder extends kelondroAbstractOrder implements kelon
         this.asc = ascending;
         this.zero = null;
     }
+    public boolean wellformed(byte[] a) {
+        return true;
+    }
+    
+    public boolean wellformed(byte[] a, int astart, int alength) {
+        return true;
+    }
     
     public final Object clone() {
         kelondroNaturalOrder o = new kelondroNaturalOrder(this.asc);

source/de/anomic/kelondro/kelondroOrder.java

@@ -49,6 +49,9 @@ import java.util.Comparator;
 
 public interface kelondroOrder extends Comparator {
 
+    public boolean wellformed(byte[] a); // returns true if and only if a has only characters that belong to the implemented order
+    public boolean wellformed(byte[] a, int astart, int alength);
+    
     public Object clone();
     
     public void direction(boolean ascending); // the ordering direction can be changed at any time

source/de/anomic/kelondro/kelondroRowCollection.java

@@ -496,6 +496,8 @@ public class kelondroRowCollection {
         assert (this.sortColumn == 0) : "this.sortColumn = " + this.sortColumn;
         int keylength = this.rowdef.width(this.sortColumn);
         int colstart  = this.rowdef.colstart[this.sortColumn];
+        if (bugappearance(chunkcache, i * this.rowdef.objectsize() + colstart, keylength)) throw new kelondroException("bugappearance i");
+        if (bugappearance(chunkcache, j * this.rowdef.objectsize() + colstart, keylength)) throw new kelondroException("bugappearance j");
         int c = this.sortOrder.compare(
                 chunkcache,
                 i * this.rowdef.objectsize() + colstart,

source/de/anomic/kelondro/kelondroRowSet.java

@@ -29,6 +29,8 @@ import java.util.Iterator;
 import java.util.Random;
 import java.util.TreeMap;
 
+import de.anomic.server.logging.serverLog;
+
 public class kelondroRowSet extends kelondroRowCollection implements kelondroIndex {
 
     private static final int collectionReSortLimit = 90;
@@ -171,8 +173,14 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
     public void shape() {
         assert (this.sortOrder != null); // we cannot shape without an object order
         synchronized (chunkcache) {
-            resolveMarkedRemoved();
-            super.sort();
+            try {
+                resolveMarkedRemoved();
+                super.sort();
+            } catch (kelondroException e) {
+                // bad bug, cannot be fixed. We abandon all data
+                serverLog.logSevere("kelondroRowSet", "abandoned row");
+                this.clear();
+            }
             //if (super.rowdef.column(0).cellwidth() == 4) System.out.println("TABLE OF " + super.rowdef.toString() + "\n" + serverLog.table(super.chunkcache, super.rowdef.objectsize, 0)); // DEBUG
         }
     }

source/de/anomic/plasma/plasmaSwitchboard.java

@@ -2389,8 +2389,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
         if (wordIndex.size() < 100) {
             return "no DHT distribution: not enough words - wordIndex.size() = " + wordIndex.size();
         }
-        if ((getConfig("allowDistributeIndexWhileCrawling","false").equalsIgnoreCase("false")) && (urlPool.noticeURL.stackSize() > 0)) {
-            return "no DHT distribution: crawl in progress - noticeURL.stackSize() = " + urlPool.noticeURL.stackSize();
+        if ((getConfig("allowDistributeIndexWhileCrawling","false").equalsIgnoreCase("false")) &&
+            ((urlPool.noticeURL.stackSize() > 0) || (sbQueue.size() > 0))) {
+            return "no DHT distribution: crawl in progress: noticeURL.stackSize() = " + urlPool.noticeURL.stackSize() + ", sbQueue.size() = " + sbQueue.size();
         }
         return null;
     }