- automatic migration of old RWI entries to new format during remote search

if new collections are activated
- one more assert in RowSet, control of removeMarker

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2993 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/yacysearch.java

@@ -140,7 +140,7 @@ public class yacysearch {
         boolean global = (post == null) ? true : post.get("resource", "global").equals("global");
         final boolean indexDistributeGranted = sb.getConfig("allowDistributeIndex", "true").equals("true");
         final boolean indexReceiveGranted = sb.getConfig("allowReceiveIndex", "true").equals("true");
-        if (!indexDistributeGranted || !indexReceiveGranted) { global = false; }
+        //if (!indexDistributeGranted || !indexReceiveGranted) { global = false; }
         final long searchtime = 1000 * Long.parseLong(post.get("time", "10"));
         String urlmask = "";
         if (post.containsKey("urlmask") && post.get("urlmask").equals("no")) {

source/de/anomic/index/indexContainer.java

@@ -85,11 +85,13 @@ public class indexContainer extends kelondroRowSet {
     }
 
     public int add(indexRWIEntry entry) {
+        assert entry.toKelondroEntry().objectsize() == super.rowdef.objectsize();
         this.addUnique(entry.toKelondroEntry());
         return 1;
     }
 
     public int add(indexRWIEntry entry, long updateTime) {
+        assert entry.toKelondroEntry().objectsize() == super.rowdef.objectsize();
         this.add(entry);
         this.lastTimeWrote = updateTime;
         return 1;
@@ -121,6 +123,7 @@ public class indexContainer extends kelondroRowSet {
     }
     
     private boolean addi(indexRWIEntry entry) {
+        assert entry.toKelondroEntry().objectsize() == super.rowdef.objectsize();
         // returns true if the new entry was added, false if it already existed
         kelondroRow.Entry oldEntryRow = this.put(entry.toKelondroEntry());
         if (oldEntryRow == null) {

source/de/anomic/kelondro/kelondroRowSet.java

@@ -27,7 +27,7 @@ package de.anomic.kelondro;
 import java.util.Date;
 import java.util.Iterator;
 import java.util.Random;
-import java.util.TreeSet;
+import java.util.TreeMap;
 
 public class kelondroRowSet extends kelondroRowCollection implements kelondroIndex {
 
@@ -35,11 +35,11 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
     private static final int removeMaxSize = 100;
 
     private kelondroProfile profile;
-    private TreeSet removeMarker;
+    private TreeMap removeMarker;
 
     public kelondroRowSet(kelondroRow rowdef, int objectCount, byte[] cache, kelondroOrder sortOrder, int sortColumn, int sortBound) {
         super(rowdef, objectCount, cache, sortOrder, sortColumn, sortBound);
-        this.removeMarker = new TreeSet();
+        this.removeMarker = new TreeMap();
         this.profile = new kelondroProfile();
     }
         
@@ -51,13 +51,13 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
 
     public kelondroRowSet(kelondroRow rowdef, int objectCount) {
         super(rowdef, objectCount);
-        this.removeMarker = new TreeSet();
+        this.removeMarker = new TreeMap();
         this.profile = new kelondroProfile();
     }
     
     public kelondroRowSet(kelondroRow rowdef, byte[] exportedCollectionRowinstance) {
         super(rowdef, exportedCollectionRowinstance);
-        this.removeMarker = new TreeSet();
+        this.removeMarker = new TreeMap();
         this.profile = new kelondroProfile();
     }
     
@@ -150,7 +150,7 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
                 entry = get(p);
                 if (p < sortBound) {
                     // mark entry as to-be-deleted
-                    removeMarker.add(new Integer(p));
+                    removeMarker.put(new Integer(p), entry.getColBytes(super.sortColumn));
                     if (removeMarker.size() > removeMaxSize) resolveMarkedRemoved();
                 } else {
                     // remove directly by swap
@@ -165,7 +165,7 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
     }
     
     private boolean isMarkedRemoved(int index) {
-        return removeMarker.contains(new Integer(index));
+        return removeMarker.containsKey(new Integer(index));
     }
     
     public void shape() {
@@ -187,15 +187,17 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
             return;
         }
         
-        Integer nxt = (Integer) removeMarker.first();
+        Integer nxt = (Integer) removeMarker.firstKey();
         removeMarker.remove(nxt);
         int idx = nxt.intValue();
         assert (idx < sortBound);
         int d = 1;
+        byte[] a;
         while (removeMarker.size() > 0) {
-            nxt = (Integer) removeMarker.first();
+            nxt = (Integer) removeMarker.firstKey();
+            a = (byte[]) removeMarker.remove(nxt);
+            assert kelondroNaturalOrder.compares(a, 0, a.length, get(nxt.intValue()).getColBytes(this.sortColumn), 0, a.length) == 0;
             assert (nxt.intValue() < sortBound);
-            removeMarker.remove(nxt);
             super.removeShift(idx, d, nxt.intValue());
             idx = nxt.intValue() - d;
             d++;

source/de/anomic/yacy/yacyClient.java

@@ -56,6 +56,8 @@ import de.anomic.htmlFilter.htmlFilterContentScraper;
 import de.anomic.http.httpc;
 import de.anomic.index.indexContainer;
 import de.anomic.index.indexRWIEntry;
+import de.anomic.index.indexRWIEntryNew;
+import de.anomic.index.indexRWIEntryOld;
 import de.anomic.plasma.plasmaURL;
 import de.anomic.index.indexURLEntry;
 import de.anomic.kelondro.kelondroBase64Order;
@@ -512,7 +514,7 @@ public final class yacyClient {
                 urlManager.stack(urlEntry, yacyCore.seedDB.mySeed.hash, targetPeer.hash, 2);
 
                 // save the url entry
-                final indexRWIEntry entry;
+                indexRWIEntry entry;
                 if (urlEntry.word() == null) {
                     // the old way to define words
                     int urlLength = comp.url().toNormalform().length();
@@ -546,6 +548,9 @@ public final class yacyClient {
                 }
                 // add the url entry to the word indexes
                 for (int m = 0; m < words; m++) {
+                    if ((wordIndex.useCollectionIndex) && (entry instanceof indexRWIEntryOld)) {
+                        entry = new indexRWIEntryNew((indexRWIEntryOld) entry);
+                    }
                     container[m].add(new indexRWIEntry[]{entry}, System.currentTimeMillis());
                 }
                 // store url hash for statistics