diff --git a/source/de/anomic/kelondro/kelondroBytesIntMap.java b/source/de/anomic/kelondro/kelondroBytesIntMap.java index 0c9e28f3d..4ecaf260d 100644 --- a/source/de/anomic/kelondro/kelondroBytesIntMap.java +++ b/source/de/anomic/kelondro/kelondroBytesIntMap.java @@ -85,13 +85,16 @@ public class kelondroBytesIntMap { ArrayList report = new ArrayList(); Integer[] is; Iterator ei; - int c; + int c, i; + int initialSize = this.size(); for (kelondroRowCollection delset: index.removeDoubles()) { is = new Integer[delset.size()]; ei = delset.rows(); c = 0; while (ei.hasNext()) { - is[c++] = new Integer((int) ei.next().getColLong(1)); + i = (int) ei.next().getColLong(1); + assert i < initialSize : "i = " + i + ", initialSize = " + initialSize; + is[c++] = new Integer(i); } report.add(is); } diff --git a/source/de/anomic/kelondro/kelondroCollectionIndex.java b/source/de/anomic/kelondro/kelondroCollectionIndex.java index 8a2e4cbc2..3566b3383 100644 --- a/source/de/anomic/kelondro/kelondroCollectionIndex.java +++ b/source/de/anomic/kelondro/kelondroCollectionIndex.java @@ -268,10 +268,10 @@ public class kelondroCollectionIndex { // open/create index table File f = new File(path, filenameStub + ".index"); kelondroRow indexRowdef = indexRow(keylength, indexOrder); - + kelondroIndex theindex; if (f.isDirectory()) { // use a flextable - kelondroIndex theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", indexRowdef, initialSpace, true)); + theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", indexRowdef, initialSpace, true)); // save/check property file for this array File propfile = propertyFile(path, filenameStub, loadfactor, rowdef.objectsize); @@ -287,14 +287,19 @@ public class kelondroCollectionIndex { } props.put("rowdef", rowdef.toString()); serverFileUtils.saveMap(propfile, props, "CollectionIndex properties"); - - return theindex; } else { // open a ecotable long records = f.length() / indexRowdef.objectsize; long necessaryRAM4fullTable = minimumRAM4Eco + (indexRowdef.objectsize + 4) * records * 3 / 2; - return new kelondroEcoTable(f, indexRowdef, (serverMemory.request(necessaryRAM4fullTable, false)) ? kelondroEcoTable.tailCacheUsageAuto : kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, initialSpace); + boolean fullCache = serverMemory.request(necessaryRAM4fullTable, false); + if (fullCache) { + theindex = new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize, initialSpace); + if (!((kelondroEcoTable) theindex).usesFullCopy()) theindex = new kelondroCache(theindex); + } else { + theindex = new kelondroCache(new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, initialSpace)); + } } + return theindex; } private kelondroFixedWidthArray openArrayFile(int partitionNumber, int serialNumber, kelondroByteOrder indexOrder, boolean create) throws IOException { diff --git a/source/de/anomic/kelondro/kelondroEcoTable.java b/source/de/anomic/kelondro/kelondroEcoTable.java index cd7374e05..bea6779ff 100644 --- a/source/de/anomic/kelondro/kelondroEcoTable.java +++ b/source/de/anomic/kelondro/kelondroEcoTable.java @@ -176,7 +176,7 @@ public class kelondroEcoTable implements kelondroIndex { byte[] record = new byte[rowdef.objectsize]; key = new byte[rowdef.primaryKeyLength]; for (Integer[] ds: doubles) { - file.get(ds[0].longValue(), record, 0); + file.get(ds[0].intValue(), record, 0); System.arraycopy(record, 0, key, 0, rowdef.primaryKeyLength); if (!index.addi(key, ds[0].intValue())) fail++; } @@ -254,6 +254,10 @@ public class kelondroEcoTable implements kelondroIndex { return map; } + public boolean usesFullCopy() { + return this.table != null; + } + public static int staticRAMIndexNeed(File f, kelondroRow rowdef) { return (int) ((rowdef.primaryKeyLength + 4) * tableSize(f, rowdef.objectsize) * kelondroRowCollection.growfactor); } @@ -284,17 +288,31 @@ public class kelondroEcoTable implements kelondroIndex { return c; } + /** + * remove double-entries from the table + * this process calls the underlying removeDoubles() method from the table index + * and + */ public synchronized ArrayList removeDoubles() throws IOException { + assert file.size() == index.size() + fail : "file.size() = " + file.size() + ", index.size() = " + index.size(); ArrayList report = new ArrayList(); kelondroRowSet rows; TreeSet d = new TreeSet(); byte[] b = new byte[rowdef.objectsize]; + Integer L; + kelondroRow.Entry inconsistentEntry; + // iterate over all entries that have inconsistent index references for (Integer[] is: index.removeDoubles()) { + // 'is' is the set of all indexes, that have the same reference + // we collect that entries now here rows = new kelondroRowSet(this.rowdef, is.length); for (int j = 0; j < is.length; j++) { - d.add(is[j]); - file.get(is[j].intValue(), b, 0); // TODO: fix IndexOutOfBoundsException here - rows.addUnique(rowdef.newEntry(b)); + L = is[j]; + assert L.intValue() < file.size() : "L.intValue() = " + L.intValue() + ", file.size = " + file.size(); // prevent ooBounds Exception + d.add(L); + file.get(L.intValue(), b, 0); // TODO: fix IndexOutOfBoundsException here + inconsistentEntry = rowdef.newEntry(b); + rows.addUnique(inconsistentEntry); } report.add(rows); } @@ -305,6 +323,7 @@ public class kelondroEcoTable implements kelondroIndex { d.remove(s); this.removeInFile(s.intValue()); } + assert file.size() == index.size() + fail : "file.size() = " + file.size() + ", index.size() = " + index.size(); return report; }