diff --git a/source/dbtest.java b/source/dbtest.java index 45b6f5049..d75a6723d 100644 --- a/source/dbtest.java +++ b/source/dbtest.java @@ -13,10 +13,8 @@ import java.util.Random; import javax.imageio.ImageIO; import de.anomic.kelondro.kelondroBase64Order; -import de.anomic.kelondro.kelondroCache; import de.anomic.kelondro.kelondroCloneableIterator; import de.anomic.kelondro.kelondroEcoTable; -import de.anomic.kelondro.kelondroFlexTable; import de.anomic.kelondro.kelondroIndex; import de.anomic.kelondro.kelondroIntBytesMap; import de.anomic.kelondro.kelondroProfile; @@ -198,14 +196,6 @@ public class dbtest { if (dbe.equals("kelondroRowSet")) { return new kelondroRowSet(testRow, 0); } - if (dbe.equals("kelondroTree")) { - final File tablefile = new File(tablename + ".kelondro.db"); - return new kelondroCache(new kelondroTree(tablefile, true, 0, testRow)); - } - if (dbe.equals("kelondroFlexTable")) { - final File tablepath = new File(tablename).getParentFile(); - return new kelondroFlexTable(tablepath, new File(tablename).getName(), testRow, 0, true); - } if (dbe.equals("kelondroSplitTable")) { final File tablepath = new File(tablename).getParentFile(); return new kelondroSplitTable(tablepath, new File(tablename).getName(), testRow, true); diff --git a/source/de/anomic/kelondro/kelondroBLOBTree.java b/source/de/anomic/kelondro/kelondroBLOBTree.java index ce473c63e..a63df0165 100644 --- a/source/de/anomic/kelondro/kelondroBLOBTree.java +++ b/source/de/anomic/kelondro/kelondroBLOBTree.java @@ -55,6 +55,10 @@ public class kelondroBLOBTree implements kelondroBLOB { private final kelondroRow rowdef; private File file; + /** + * Deprecated Class. Please use kelondroBLOBHeap instead + */ + @Deprecated public kelondroBLOBTree(final File file, final boolean useNodeCache, final boolean useObjectCache, final int key, final int nodesize, final char fillChar, final kelondroByteOrder objectOrder, final boolean usetree, final boolean writebuffer, final boolean resetOnFail) { // creates or opens a dynamic tree diff --git a/source/de/anomic/kelondro/kelondroCollectionIndex.java b/source/de/anomic/kelondro/kelondroCollectionIndex.java index f73033458..6103e22e6 100644 --- a/source/de/anomic/kelondro/kelondroCollectionIndex.java +++ b/source/de/anomic/kelondro/kelondroCollectionIndex.java @@ -105,12 +105,6 @@ public class kelondroCollectionIndex { final String sn = fillZ(Integer.toHexString(serialNumber).toUpperCase(), 2); return new File(path, filenameStub + "." + lf + "." + cs + "." + pn + "." + sn + ".kca"); // kelondro collection array } - - private static File propertyFile(final File path, final String filenameStub, final int loadfactor, final int chunksize) { - final String lf = fillZ(Integer.toHexString(loadfactor).toUpperCase(), 2); - final String cs = fillZ(Integer.toHexString(chunksize).toUpperCase(), 4); - return new File(path, filenameStub + "." + lf + "." + cs + ".properties"); - } public kelondroCollectionIndex(final File path, final String filenameStub, final int keyLength, final kelondroByteOrder indexOrder, final int loadfactor, final int maxpartitions, final kelondroRow rowdef, boolean useCommons) throws IOException { @@ -130,7 +124,9 @@ public class kelondroCollectionIndex { this.commonsPath.mkdirs(); } final File f = new File(path, filenameStub + ".index"); - + if (f.isDirectory()) { + kelondroFlexTable.delete(path, filenameStub + ".index"); + } if (f.exists()) { serverLog.logFine("STARTUP", "OPENING COLLECTION INDEX"); @@ -161,16 +157,9 @@ public class kelondroCollectionIndex { serverLog.logFine("STARTUP", "STARTED INITIALIZATION OF NEW COLLECTION INDEX WITH " + initialSpace + " ENTRIES. THIS WILL TAKE SOME TIME. " + (serverMemory.available() / 1024 / 1024) + "MB AVAILABLE."); final kelondroRow indexRowdef = indexRow(keyLength, indexOrder); final long necessaryRAM4fullTable = minimumRAM4Eco + (indexRowdef.objectsize + 4) * initialSpace * 3 / 2; - final long necessaryRAM4fullIndex = minimumRAM4Eco + (indexRowdef.primaryKeyLength + 4) * initialSpace * 3 / 2; // initialize (new generation) index table from file - if (serverMemory.request(necessaryRAM4fullTable, false)) { - index = new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize, initialSpace); - } else if (serverMemory.request(necessaryRAM4fullIndex, false)) { - index = new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, initialSpace); - } else { - index = new kelondroFlexTable(path, filenameStub + ".index", indexRowdef, initialSpace, true); - } + index = new kelondroEcoTable(f, indexRowdef, (serverMemory.request(necessaryRAM4fullTable, false)) ? kelondroEcoTable.tailCacheUsageAuto : kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, initialSpace); // open array files this.arrays = new HashMap(); // all entries will be dynamically created with getArray() @@ -283,35 +272,18 @@ public class kelondroCollectionIndex { final kelondroRow indexRowdef = indexRow(keylength, indexOrder); kelondroIndex theindex; if (f.isDirectory()) { - // use a flextable - theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", indexRowdef, initialSpace, true)); - - // save/check property file for this array - final File propfile = propertyFile(path, filenameStub, loadfactor, rowdef.objectsize); - Map props = new HashMap(); - if (propfile.exists()) { - props = serverFileUtils.loadHashMap(propfile); - final String stored_rowdef = props.get("rowdef"); - if ((rowdef != null) && (!(rowdef.subsumes(new kelondroRow(stored_rowdef, rowdef.objectOrder, 0))))) { - System.out.println("FATAL ERROR: stored rowdef '" + stored_rowdef + "' does not match with new rowdef '" + - rowdef + "' for array cluster '" + path + "/" + filenameStub + "'"); - System.exit(-1); - } - } - props.put("rowdef", rowdef.toString()); - serverFileUtils.saveMap(propfile, props, "CollectionIndex properties"); + kelondroFlexTable.delete(path, filenameStub + ".index"); + } + // open a ecotable + final long records = f.length() / indexRowdef.objectsize; + final long necessaryRAM4fullTable = minimumRAM4Eco + (indexRowdef.objectsize + 4) * records * 3 / 2; + final boolean fullCache = serverMemory.request(necessaryRAM4fullTable, false); + if (fullCache) { + theindex = new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize, initialSpace); + //if (!((kelondroEcoTable) theindex).usesFullCopy()) theindex = new kelondroCache(theindex); } else { - // open a ecotable - final long records = f.length() / indexRowdef.objectsize; - final long necessaryRAM4fullTable = minimumRAM4Eco + (indexRowdef.objectsize + 4) * records * 3 / 2; - final boolean fullCache = serverMemory.request(necessaryRAM4fullTable, false); - if (fullCache) { - theindex = new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize, initialSpace); - //if (!((kelondroEcoTable) theindex).usesFullCopy()) theindex = new kelondroCache(theindex); - } else { - //theindex = new kelondroCache(new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, initialSpace)); - theindex = new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, initialSpace); - } + //theindex = new kelondroCache(new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, initialSpace)); + theindex = new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, initialSpace); } return theindex; } @@ -918,7 +890,7 @@ public class kelondroCollectionIndex { // printout of index collectionIndex.close(); - final kelondroFlexTable index = new kelondroFlexTable(path, filenameStub + ".index", kelondroCollectionIndex.indexRow(9, kelondroNaturalOrder.naturalOrder), 0, true); + final kelondroEcoTable index = new kelondroEcoTable(new File(path, filenameStub + ".index"), kelondroCollectionIndex.indexRow(9, kelondroNaturalOrder.naturalOrder), kelondroEcoTable.tailCacheUsageAuto, 0, 0); index.print(); index.close(); } catch (final IOException e) { diff --git a/source/de/anomic/kelondro/kelondroEcoTable.java b/source/de/anomic/kelondro/kelondroEcoTable.java index 41585f43c..921c84cc3 100644 --- a/source/de/anomic/kelondro/kelondroEcoTable.java +++ b/source/de/anomic/kelondro/kelondroEcoTable.java @@ -676,6 +676,20 @@ public class kelondroEcoTable implements kelondroIndex { } } + public void print() throws IOException { + System.out.println("PRINTOUT of table, length=" + size()); + Entry row; + byte[] key; + kelondroCloneableIterator i = keys(true, null); + while (i.hasNext()) { + System.out.print("row " + i + ": "); + key = i.next(); + row = get(key); + System.out.println(row.toString()); + } + System.out.println("EndOfTable"); + } + public static void main(final String[] args) { // open a file, add one entry and exit final File f = new File(args[0]); diff --git a/source/de/anomic/kelondro/kelondroFlexTable.java b/source/de/anomic/kelondro/kelondroFlexTable.java index 39584b15b..a0f3f2cb7 100644 --- a/source/de/anomic/kelondro/kelondroFlexTable.java +++ b/source/de/anomic/kelondro/kelondroFlexTable.java @@ -47,6 +47,10 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr protected kelondroBytesIntMap index; private boolean RAMIndex; + /** + * Deprecated Class. Please use kelondroEcoTable instead + */ + @Deprecated public kelondroFlexTable(final File path, final String tablename, final kelondroRow rowdef, int minimumSpace, final boolean resetOnFail) { // the buffersize applies to a possible load of the ram-index // the minimumSpace is a initial allocation space for the index; names the number of index slots @@ -72,43 +76,21 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr System.out.println("*** Last Startup time: " + stt + " milliseconds"); final long start = System.currentTimeMillis(); - if (serverMemory.request(neededRAM, false)) { - // we can use a RAM index - if (indexfile.exists()) { - // delete existing index file - System.out.println("*** Delete File index " + indexfile); - indexfile.delete(); - } - - // fill the index - System.out.print("*** Loading RAM index for " + size() + " entries from " + newpath + "; available RAM = " + (serverMemory.available() >> 20) + " MB, allocating " + (neededRAM >> 20) + " MB for index."); - index = initializeRamIndex(minimumSpace); + // we use a RAM index + if (indexfile.exists()) { + // delete existing index file + System.out.println("*** Delete File index " + indexfile); + indexfile.delete(); + } - System.out.println(" -done-"); - System.out.println(index.size() + " index entries initialized and sorted from " + super.col[0].size() + " keys."); - RAMIndex = true; - tableTracker.put(this.filename(), this); - } else { - // too less ram for a ram index - kelondroIndex ki; - if (indexfile.exists()) { - // use existing index file - System.out.println("*** Using File index " + indexfile); - ki = new kelondroCache(kelondroTree.open(indexfile, true, 0, treeIndexRow(rowdef.width(0), rowdef.objectOrder), 2, 80)); - RAMIndex = false; - } else { - // generate new index file - System.out.println("*** Generating File index for " + size() + " entries from " + indexfile); - System.out.println("*** Cause: too less RAM (" + serverMemory.available() + " Bytes) configured. Assign at least " + (neededRAM / 1024 / 1024) + " MB more RAM to enable a RAM index."); - ki = initializeTreeIndex(indexfile, 0, rowdef.objectOrder); + // fill the index + System.out.print("*** Loading RAM index for " + size() + " entries from " + newpath + "; available RAM = " + (serverMemory.available() >> 20) + " MB, allocating " + (neededRAM >> 20) + " MB for index."); + index = initializeRamIndex(minimumSpace); - System.out.println(" -done-"); - System.out.println(ki.size() + " entries indexed from " + super.col[0].size() + " keys."); - RAMIndex = false; - } - index = new kelondroBytesIntMap(ki); - assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); - } + System.out.println(" -done-"); + System.out.println(index.size() + " index entries initialized and sorted from " + super.col[0].size() + " keys."); + RAMIndex = true; + tableTracker.put(this.filename(), this); // check consistency final ArrayList doubles = index.removeDoubles(); @@ -185,36 +167,6 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr return ri; } - private kelondroIndex initializeTreeIndex(final File indexfile, final long preloadTime, final kelondroByteOrder objectOrder) throws IOException { - final kelondroIndex treeindex = new kelondroCache(new kelondroTree(indexfile, true, preloadTime, treeIndexRow(rowdef.primaryKeyLength, objectOrder), 2, 80)); - final Iterator content = super.col[0].contentNodes(-1); - kelondroNode node; - kelondroRow.Entry indexentry; - int i, c = 0; - final int all = super.col[0].size(); - final long start = System.currentTimeMillis(); - long last = start; - while (content.hasNext()) { - node = content.next(); - i = node.handle().hashCode(); - indexentry = treeindex.row().newEntry(); - indexentry.setCol(0, node.getValueRow()); - indexentry.setCol(1, i); - treeindex.addUnique(indexentry); - c++; - if (System.currentTimeMillis() - last > 30000) { - System.out.println(".. generated " + c + "/" + all + " entries, " + ((System.currentTimeMillis() - start) / c * (all - c) / 60000) + " minutes remaining"); - System.out.flush(); - last = System.currentTimeMillis(); - } - } - return treeindex; - } - - private static final kelondroRow treeIndexRow(final int keywidth, final kelondroByteOrder objectOrder) { - return new kelondroRow("byte[] key-" + keywidth + ", int reference-4 {b256}", objectOrder, 0); - } - public synchronized kelondroRow.Entry get(final byte[] key) throws IOException { if (index == null) return null; // case may happen during shutdown final int pos = index.geti(key); diff --git a/source/de/anomic/kelondro/kelondroSplitTable.java b/source/de/anomic/kelondro/kelondroSplitTable.java index 845300022..ace5c9dd4 100644 --- a/source/de/anomic/kelondro/kelondroSplitTable.java +++ b/source/de/anomic/kelondro/kelondroSplitTable.java @@ -47,7 +47,6 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import de.anomic.server.NamePrefixThreadFactory; -import de.anomic.server.serverMemory; import de.anomic.server.serverProcessor; import de.anomic.server.logging.serverLog; @@ -57,7 +56,6 @@ public class kelondroSplitTable implements kelondroIndex { // the set is divided into tables with different entry date // the table type can be either kelondroFlex or kelondroEco - private static final long minimumRAM4Eco = 80 * 1024 * 1024; private static final int EcoFSBufferSize = 20; static final kelondroIndex dummyIndex = new kelondroRAMIndex(new kelondroRow(new kelondroColumn[]{new kelondroColumn("key", kelondroColumn.celltype_binary, kelondroColumn.encoder_bytes, 2, "key")}, kelondroNaturalOrder.naturalOrder, 0), 0); @@ -137,12 +135,11 @@ public class kelondroSplitTable implements kelondroIndex { f = new File(path, maxf); if (f.isDirectory()) { // this is a kelonodroFlex table - serverLog.logInfo("kelondroSplitTable", "opening partial flex table " + f); - table = new kelondroFlexTable(path, maxf, rowdef, 0, resetOnFail); - } else { - serverLog.logInfo("kelondroSplitTable", "opening partial eco table " + f); - table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize, 0); + kelondroFlexTable.delete(path, maxf); + serverLog.logInfo("kelondroSplitTable", "replaced partial flex table " + f + " by new eco table"); } + serverLog.logInfo("kelondroSplitTable", "opening partial eco table " + f); + table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize, 0); tables.put(date, table); } } @@ -239,21 +236,13 @@ public class kelondroSplitTable implements kelondroIndex { final File f = new File(path, tablename + "." + suffix); if (f.exists()) { if (f.isDirectory()) { - // open a flex table - table = new kelondroFlexTable(path, tablename + "." + suffix, rowdef, 0, true); - } else { - // open a eco table - table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0); + kelondroFlexTable.delete(path, tablename + "." + suffix); } + // open a eco table + table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0); } else { // make new table - if (serverMemory.request(minimumRAM4Eco, true)) { - // enough memory for a ecoTable - table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0); - } else { - // use the flex table - table = new kelondroFlexTable(path, tablename + "." + suffix, rowdef, 0, true); - } + table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0); } tables.put(suffix, table); } @@ -320,13 +309,7 @@ public class kelondroSplitTable implements kelondroIndex { kelondroIndex table = tables.get(suffix); if (table == null) { // make new table - if (serverMemory.request(minimumRAM4Eco, true)) { - // enough memory for a ecoTable - table = new kelondroEcoTable(new File(path, tablename + "." + suffix), rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0); - } else { - // use the flex table - table = new kelondroFlexTable(path, tablename + "." + suffix, rowdef, 0, true); - } + table = new kelondroEcoTable(new File(path, tablename + "." + suffix), rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0); tables.put(suffix, table); } table.addUnique(row); diff --git a/source/de/anomic/kelondro/kelondroTree.java b/source/de/anomic/kelondro/kelondroTree.java index 01d4e8e8c..b87f54e05 100644 --- a/source/de/anomic/kelondro/kelondroTree.java +++ b/source/de/anomic/kelondro/kelondroTree.java @@ -74,10 +74,15 @@ public class kelondroTree extends kelondroCachedRecords implements kelondroIndex protected int readAheadChunkSize = 100; protected long lastIteratorCount = readAheadChunkSize; + /** + * Deprecated Class. Please use kelondroEcoTable instead + */ + @Deprecated public kelondroTree(final File file, final boolean useNodeCache, final long preloadTime, final kelondroRow rowdef) throws IOException { this(file, useNodeCache, preloadTime, rowdef, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */); } + @Deprecated public kelondroTree(final File file, final boolean useNodeCache, final long preloadTime, final kelondroRow rowdef, final int txtProps, final int txtPropsWidth) throws IOException { // opens an existing tree file or creates a new tree file super(file, useNodeCache, preloadTime, @@ -91,10 +96,7 @@ public class kelondroTree extends kelondroCachedRecords implements kelondroIndex this.loopDetectionOrder = new kelondroByteOrder.StringOrder(rowdef.objectOrder); } - public static final kelondroTree open(final File file, final boolean useNodeCache, final long preloadTime, final kelondroRow rowdef) { - return open(file, useNodeCache, preloadTime, rowdef, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */); - } - + @Deprecated public static final kelondroTree open(final File file, final boolean useNodeCache, final long preloadTime, final kelondroRow rowdef, final int txtProps, final int txtPropsWidth) { // opens new or existing file; in case that any error occur the file is deleted again and it is tried to create the file again // if that fails, the method returns null @@ -113,33 +115,6 @@ public class kelondroTree extends kelondroCachedRecords implements kelondroIndex } } - public kelondroTree(final kelondroRA ra, final String filename, final boolean useNodeCache, final long preloadTime, final kelondroRow rowdef, final boolean exitOnFail) { - // this creates a new tree within a kelondroRA - this(ra, filename, useNodeCache, preloadTime, rowdef, new kelondroNaturalOrder(true), rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail); - } - - public kelondroTree(final kelondroRA ra, final String filename, final boolean useNodeCache, final long preloadTime, final kelondroRow rowdef, final kelondroByteOrder objectOrder, final int txtProps, final int txtPropsWidth, final boolean exitOnFail) { - // this creates a new tree within a kelondroRA - super(ra, filename, useNodeCache, preloadTime, - thisOHBytes, thisOHHandles, rowdef, - thisFHandles, txtProps, txtPropsWidth, exitOnFail); - try { - setHandle(root, null); // define the root value - } catch (final IOException e) { - super.logFailure("cannot set root handle / " + e.getMessage()); - if (exitOnFail) System.exit(-1); - throw new RuntimeException("cannot set root handle / " + e.getMessage()); - } - super.setLogger(log); - this.loopDetectionOrder = new kelondroByteOrder.StringOrder(rowdef.objectOrder); - } - - public kelondroTree(final kelondroRA ra, final String filename, final boolean useNodeCache, final long preloadTime) throws IOException { - // this opens a file with an existing tree in a kelondroRA - super(ra, filename, useNodeCache, preloadTime); - super.setLogger(log); - } - public void clear() throws IOException { super.clear(); setHandle(root, null); diff --git a/source/de/anomic/plasma/plasmaRankingCRProcess.java b/source/de/anomic/plasma/plasmaRankingCRProcess.java index 80be062ad..b847c0b80 100644 --- a/source/de/anomic/plasma/plasmaRankingCRProcess.java +++ b/source/de/anomic/plasma/plasmaRankingCRProcess.java @@ -34,7 +34,7 @@ import de.anomic.kelondro.kelondroAttrSeq; import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroBitfield; import de.anomic.kelondro.kelondroCollectionIndex; -import de.anomic.kelondro.kelondroFlexTable; +import de.anomic.kelondro.kelondroEcoTable; import de.anomic.kelondro.kelondroIndex; import de.anomic.kelondro.kelondroMicroDate; import de.anomic.kelondro.kelondroRow; @@ -242,7 +242,7 @@ public class plasmaRankingCRProcess { kelondroCollectionIndex newseq = null; if (newdb) { final File path = to_file.getParentFile(); // path to storage place - newacc = new kelondroFlexTable(path, CRG_accname, CRG_accrow, 0, false); + newacc = new kelondroEcoTable(new File(path, CRG_accname), CRG_accrow, kelondroEcoTable.tailCacheUsageAuto, 0, 0); newseq = new kelondroCollectionIndex(path, CRG_seqname, 12, kelondroBase64Order.enhancedCoder, 2, 9, CRG_colrow, false); } else { if (!(to_file.exists())) {