- more asserts to solve the ooB-problem

- better caching (?), lets see how it behaves

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4937 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/kelondro/kelondroBytesIntMap.java

@@ -85,13 +85,16 @@ public class kelondroBytesIntMap {
         ArrayList<Integer[]> report = new ArrayList<Integer[]>();
         Integer[] is;
         Iterator<kelondroRow.Entry> ei;
-        int c;
+        int c, i;
+        int initialSize = this.size();
         for (kelondroRowCollection delset: index.removeDoubles()) {
             is = new Integer[delset.size()];
             ei = delset.rows();
             c = 0;
             while (ei.hasNext()) {
-                is[c++] = new Integer((int) ei.next().getColLong(1));
+                i = (int) ei.next().getColLong(1);
+                assert i < initialSize : "i = " + i + ", initialSize = " + initialSize;
+                is[c++] = new Integer(i);
             }
             report.add(is);
         }

source/de/anomic/kelondro/kelondroCollectionIndex.java

@@ -268,10 +268,10 @@ public class kelondroCollectionIndex {
         // open/create index table
         File f = new File(path, filenameStub + ".index");
         kelondroRow indexRowdef = indexRow(keylength, indexOrder);
-        
+        kelondroIndex theindex;
         if (f.isDirectory()) {
             // use a flextable
-            kelondroIndex theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", indexRowdef, initialSpace, true));
+            theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", indexRowdef, initialSpace, true));
         
             // save/check property file for this array
             File propfile = propertyFile(path, filenameStub, loadfactor, rowdef.objectsize);
@@ -287,14 +287,19 @@ public class kelondroCollectionIndex {
             }
             props.put("rowdef", rowdef.toString());
             serverFileUtils.saveMap(propfile, props, "CollectionIndex properties");
-        
-            return theindex;
         } else {
             // open a ecotable
             long records = f.length() / indexRowdef.objectsize;
             long necessaryRAM4fullTable = minimumRAM4Eco + (indexRowdef.objectsize + 4) * records * 3 / 2;
-            return new kelondroEcoTable(f, indexRowdef, (serverMemory.request(necessaryRAM4fullTable, false)) ? kelondroEcoTable.tailCacheUsageAuto : kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, initialSpace);
+            boolean fullCache = serverMemory.request(necessaryRAM4fullTable, false);
+            if (fullCache) {
+                theindex = new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize, initialSpace);
+                if (!((kelondroEcoTable) theindex).usesFullCopy()) theindex = new kelondroCache(theindex);
+            } else {
+                theindex = new kelondroCache(new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, initialSpace));
+            }
         }
+        return theindex;
     }
     
     private kelondroFixedWidthArray openArrayFile(int partitionNumber, int serialNumber, kelondroByteOrder indexOrder, boolean create) throws IOException {

source/de/anomic/kelondro/kelondroEcoTable.java

@@ -176,7 +176,7 @@ public class kelondroEcoTable implements kelondroIndex {
                 byte[] record = new byte[rowdef.objectsize];
                 key = new byte[rowdef.primaryKeyLength];
                 for (Integer[] ds: doubles) {
-                    file.get(ds[0].longValue(), record, 0);
+                    file.get(ds[0].intValue(), record, 0);
                     System.arraycopy(record, 0, key, 0, rowdef.primaryKeyLength);
                     if (!index.addi(key, ds[0].intValue())) fail++;
                 }
@@ -254,6 +254,10 @@ public class kelondroEcoTable implements kelondroIndex {
         return map;
     }
     
+    public boolean usesFullCopy() {
+        return this.table != null;
+    }
+    
     public static int staticRAMIndexNeed(File f, kelondroRow rowdef) {
         return (int) ((rowdef.primaryKeyLength + 4) * tableSize(f, rowdef.objectsize) * kelondroRowCollection.growfactor);
     }
@@ -284,17 +288,31 @@ public class kelondroEcoTable implements kelondroIndex {
         return c;
     }
 
+    /**
+     * remove double-entries from the table
+     * this process calls the underlying removeDoubles() method from the table index
+     * and 
+     */
     public synchronized ArrayList<kelondroRowCollection> removeDoubles() throws IOException {
+        assert file.size() == index.size() + fail : "file.size() = " + file.size() + ", index.size() = " + index.size();
         ArrayList<kelondroRowCollection> report = new ArrayList<kelondroRowCollection>();
         kelondroRowSet rows;
         TreeSet<Integer> d = new TreeSet<Integer>();
         byte[] b = new byte[rowdef.objectsize];
+        Integer L;
+        kelondroRow.Entry inconsistentEntry;
+        // iterate over all entries that have inconsistent index references
         for (Integer[] is: index.removeDoubles()) {
+            // 'is' is the set of all indexes, that have the same reference
+            // we collect that entries now here
             rows = new kelondroRowSet(this.rowdef, is.length);
             for (int j = 0; j < is.length; j++) {
-                d.add(is[j]);
-                file.get(is[j].intValue(), b, 0); // TODO: fix IndexOutOfBoundsException here
-                rows.addUnique(rowdef.newEntry(b));
+                L = is[j];
+                assert L.intValue() < file.size() : "L.intValue() = " + L.intValue() + ", file.size = " + file.size(); // prevent ooBounds Exception
+                d.add(L);
+                file.get(L.intValue(), b, 0); // TODO: fix IndexOutOfBoundsException here
+                inconsistentEntry = rowdef.newEntry(b);
+                rows.addUnique(inconsistentEntry);
             }
             report.add(rows);
         }
@@ -305,6 +323,7 @@ public class kelondroEcoTable implements kelondroIndex {
             d.remove(s);
             this.removeInFile(s.intValue());
         }
+        assert file.size() == index.size() + fail : "file.size() = " + file.size() + ", index.size() = " + index.size();
         return report;
     }