diff --git a/source/dbtest.java b/source/dbtest.java index 83dced322..1e3573a9b 100644 --- a/source/dbtest.java +++ b/source/dbtest.java @@ -207,7 +207,7 @@ public class dbtest { } if (dbe.equals("kelondroFlexTable")) { File tablepath = new File(tablename).getParentFile(); - table = new kelondroFlexTable(tablepath, new File(tablename).getName(), preload, testRow, true); + table = new kelondroFlexTable(tablepath, new File(tablename).getName(), preload, testRow, 0, true); } if (dbe.equals("kelondroFlexSplitTable")) { File tablepath = new File(tablename).getParentFile(); diff --git a/source/de/anomic/kelondro/kelondroCollectionIndex.java b/source/de/anomic/kelondro/kelondroCollectionIndex.java index b1c90035c..6010c2559 100644 --- a/source/de/anomic/kelondro/kelondroCollectionIndex.java +++ b/source/de/anomic/kelondro/kelondroCollectionIndex.java @@ -123,24 +123,46 @@ public class kelondroCollectionIndex { this.commonsPath = new File(path, filenameStub + "." + fillZ(Integer.toHexString(rowdef.objectsize).toUpperCase(), 4) + ".commons"); this.commonsPath.mkdirs(); - boolean ramIndexGeneration = false; - boolean fileIndexGeneration = !(new File(path, filenameStub + ".index").exists()); - if (ramIndexGeneration) index = new kelondroRowSet(indexRow(keyLength, indexOrder), 0); - if (fileIndexGeneration) index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRow(keyLength, indexOrder), true); - - // open array files - this.arrays = new HashMap(); // all entries will be dynamically created with getArray() - if ((fileIndexGeneration) || (ramIndexGeneration)) { - serverLog.logFine("STARTUP", "STARTED INITIALIZATION OF NEW COLLECTION INDEX. THIS WILL TAKE SOME TIME"); - } else { - if (index == null) index = openIndexFile(path, filenameStub, indexOrder, preloadTime, loadfactor, rowdef); + if (new File(path, filenameStub + ".index").exists()) { serverLog.logFine("STARTUP", "OPENING COLLECTION INDEX"); + + // open index and array files + this.arrays = new HashMap(); // all entries will be dynamically created with getArray() + index = openIndexFile(path, filenameStub, indexOrder, preloadTime, loadfactor, rowdef, 0); + openAllArrayFiles(false, indexOrder); + } else { + // calculate initialSpace + String[] list = this.path.list(); + kelondroFixedWidthArray array; + int initialSpace = 0; + for (int i = 0; i < list.length; i++) if (list[i].endsWith(".kca")) { + // open array + int pos = list[i].indexOf('.'); + if (pos < 0) continue; + int partitionNumber = Integer.parseInt(list[i].substring(pos + 9, pos + 11), 16); + int serialNumber = Integer.parseInt(list[i].substring(pos + 12, pos + 14), 16); + try { + array = openArrayFile(partitionNumber, serialNumber, indexOrder, true); + initialSpace += array.size(); + array.close(); + } catch (IOException e) { + e.printStackTrace(); + continue; + } + } + serverLog.logFine("STARTUP", "STARTED INITIALIZATION OF NEW COLLECTION INDEX WITH " + initialSpace + " ENTRIES. THIS WILL TAKE SOME TIME"); + + // initialize (new generation) index table from file + index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRow(keyLength, indexOrder), initialSpace, true); + + // open array files + this.arrays = new HashMap(); // all entries will be dynamically created with getArray() + openAllArrayFiles(true, indexOrder); } - openAllArrayFiles(((fileIndexGeneration) || (ramIndexGeneration)), indexOrder); - if (index == null) index = openIndexFile(path, filenameStub, indexOrder, preloadTime, loadfactor, rowdef); } private void openAllArrayFiles(boolean indexGeneration, kelondroOrder indexOrder) throws IOException { + String[] list = this.path.list(); kelondroFixedWidthArray array; @@ -155,7 +177,7 @@ public class kelondroCollectionIndex { int partitionNumber = Integer.parseInt(list[i].substring(pos + 9, pos + 11), 16); int serialNumber = Integer.parseInt(list[i].substring(pos + 12, pos + 14), 16); try { - array = openArrayFile(partitionNumber, serialNumber, true); + array = openArrayFile(partitionNumber, serialNumber, indexOrder, true); } catch (IOException e) { e.printStackTrace(); continue; @@ -201,9 +223,9 @@ public class kelondroCollectionIndex { } private kelondroIndex openIndexFile(File path, String filenameStub, kelondroOrder indexOrder, - long preloadTime, int loadfactor, kelondroRow rowdef) throws IOException { + long preloadTime, int loadfactor, kelondroRow rowdef, int initialSpace) throws IOException { // open/create index table - kelondroIndex theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRow(keylength, indexOrder), true), true, false); + kelondroIndex theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRow(keylength, indexOrder), initialSpace, true), true, false); //kelondroIndex theindex = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRow(keylength, indexOrder), true); // save/check property file for this array @@ -224,13 +246,13 @@ public class kelondroCollectionIndex { return theindex; } - private kelondroFixedWidthArray openArrayFile(int partitionNumber, int serialNumber, boolean create) throws IOException { + private kelondroFixedWidthArray openArrayFile(int partitionNumber, int serialNumber, kelondroOrder indexOrder, boolean create) throws IOException { File f = arrayFile(path, filenameStub, loadfactor, payloadrow.objectsize, partitionNumber, serialNumber); int load = arrayCapacity(partitionNumber); kelondroRow rowdef = new kelondroRow( "byte[] key-" + keylength + "," + "byte[] collection-" + (kelondroRowCollection.exportOverheadSize + load * this.payloadrow.objectsize), - index.row().objectOrder, + indexOrder, 0 ); if ((!(f.exists())) && (!create)) return null; @@ -239,12 +261,12 @@ public class kelondroCollectionIndex { return a; } - private kelondroFixedWidthArray getArray(int partitionNumber, int serialNumber, int chunksize) { + private kelondroFixedWidthArray getArray(int partitionNumber, int serialNumber, kelondroOrder indexOrder, int chunksize) { String accessKey = partitionNumber + "-" + chunksize; kelondroFixedWidthArray array = (kelondroFixedWidthArray) arrays.get(accessKey); if (array != null) return array; try { - array = openArrayFile(partitionNumber, serialNumber, true); + array = openArrayFile(partitionNumber, serialNumber, indexOrder, true); } catch (IOException e) { return null; } @@ -289,7 +311,7 @@ public class kelondroCollectionIndex { int oldRownumber) throws IOException { // we need a new slot, that means we must first delete the old entry // find array file - kelondroFixedWidthArray array = getArray(oldPartitionNumber, serialNumber, chunkSize); + kelondroFixedWidthArray array = getArray(oldPartitionNumber, serialNumber, index.row().objectOrder, chunkSize); // delete old entry array.remove(oldRownumber); @@ -300,7 +322,7 @@ public class kelondroCollectionIndex { // the collection is new int partitionNumber = arrayIndex(collection.size()); kelondroRow.Entry indexrow = index.row().newEntry(); - kelondroFixedWidthArray array = getArray(partitionNumber, serialNumber, this.payloadrow.objectsize); + kelondroFixedWidthArray array = getArray(partitionNumber, serialNumber, index.row().objectOrder, this.payloadrow.objectsize); // define row kelondroRow.Entry arrayEntry = array.row().newEntry(); @@ -329,7 +351,7 @@ public class kelondroCollectionIndex { int partitionNumber, int serialNumber, int chunkSize) throws IOException { // write a new entry in the other array - kelondroFixedWidthArray array = getArray(partitionNumber, serialNumber, chunkSize); + kelondroFixedWidthArray array = getArray(partitionNumber, serialNumber, index.row().objectOrder, chunkSize); // define new row kelondroRow.Entry arrayEntry = array.row().newEntry(); @@ -365,7 +387,7 @@ public class kelondroCollectionIndex { entry = (Map.Entry) i.next(); actionList = (ArrayList) entry.getValue(); partitionNumber = ((Integer) entry.getKey()).intValue(); - array = getArray(partitionNumber, serialNumber, chunkSize); + array = getArray(partitionNumber, serialNumber, index.row().objectOrder, chunkSize); j = actionList.iterator(); while (j.hasNext()) { @@ -401,7 +423,7 @@ public class kelondroCollectionIndex { // we don't need a new slot, just write collection into the old one // find array file - kelondroFixedWidthArray array = getArray(partitionNumber, serialNumber, chunkSize); + kelondroFixedWidthArray array = getArray(partitionNumber, serialNumber, index.row().objectOrder, chunkSize); // define new row kelondroRow.Entry arrayEntry = array.row().newEntry(); @@ -437,7 +459,7 @@ public class kelondroCollectionIndex { entry = (Map.Entry) i.next(); actionMap = (TreeMap) entry.getValue(); partitionNumber = ((Integer) entry.getKey()).intValue(); - array = getArray(partitionNumber, serialNumber, chunkSize); + array = getArray(partitionNumber, serialNumber, index.row().objectOrder, chunkSize); j = actionMap.entrySet().iterator(); while (j.hasNext()) { @@ -921,7 +943,7 @@ public class kelondroCollectionIndex { private synchronized kelondroRowSet getwithparams(kelondroRow.Entry indexrow, int chunksize, int chunkcount, int clusteridx, int rownumber, int serialnumber, boolean remove) throws IOException { // open array entry - kelondroFixedWidthArray array = getArray(clusteridx, serialnumber, chunksize); + kelondroFixedWidthArray array = getArray(clusteridx, serialnumber, index.row().objectOrder, chunksize); kelondroRow.Entry arrayrow = array.get(rownumber); if (arrayrow == null) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString(), "array does not contain expected row"); @@ -1053,7 +1075,7 @@ public class kelondroCollectionIndex { // printout of index collectionIndex.close(); - kelondroFlexTable index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, kelondroCollectionIndex.indexRow(9, kelondroNaturalOrder.naturalOrder), true); + kelondroFlexTable index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, kelondroCollectionIndex.indexRow(9, kelondroNaturalOrder.naturalOrder), 0, true); index.print(); index.close(); } catch (IOException e) { diff --git a/source/de/anomic/kelondro/kelondroDyn.java b/source/de/anomic/kelondro/kelondroDyn.java index 9407e96f1..3e66863fa 100644 --- a/source/de/anomic/kelondro/kelondroDyn.java +++ b/source/de/anomic/kelondro/kelondroDyn.java @@ -94,7 +94,7 @@ public class kelondroDyn { } } else { - fbi = new kelondroFlexTable(file.getParentFile(), file.getName(), 10000, rowdef, resetOnFail); + fbi = new kelondroFlexTable(file.getParentFile(), file.getName(), 10000, rowdef, 0, resetOnFail); } this.index = (useObjectCache) ? (kelondroIndex) new kelondroCache(fbi, true, writebuffer) : fbi; this.keylen = key; diff --git a/source/de/anomic/kelondro/kelondroFlexSplitTable.java b/source/de/anomic/kelondro/kelondroFlexSplitTable.java index 6b43849e7..0a0b6f56a 100644 --- a/source/de/anomic/kelondro/kelondroFlexSplitTable.java +++ b/source/de/anomic/kelondro/kelondroFlexSplitTable.java @@ -100,7 +100,7 @@ public class kelondroFlexSplitTable implements kelondroIndex { // open next biggest table t.remove(maxf); date = maxf.substring(tablename.length() + 1); - table = new kelondroCache(new kelondroFlexTable(path, maxf, preloadTime, rowdef, resetOnFail), true, false); + table = new kelondroCache(new kelondroFlexTable(path, maxf, preloadTime, rowdef, 0, resetOnFail), true, false); tables.put(date, table); } } @@ -213,7 +213,7 @@ public class kelondroFlexSplitTable implements kelondroIndex { kelondroIndex table = (kelondroIndex) tables.get(suffix); if (table == null) { // make new table - table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef, true); + table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef, 0, true); tables.put(suffix, table); } table.put(row); @@ -244,7 +244,7 @@ public class kelondroFlexSplitTable implements kelondroIndex { kelondroIndex table = (kelondroIndex) tables.get(suffix); if (table == null) { // make new table - table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef, true); + table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef, 0, true); tables.put(suffix, table); } table.addUnique(row); diff --git a/source/de/anomic/kelondro/kelondroFlexTable.java b/source/de/anomic/kelondro/kelondroFlexTable.java index 3ad01166c..072faed5d 100644 --- a/source/de/anomic/kelondro/kelondroFlexTable.java +++ b/source/de/anomic/kelondro/kelondroFlexTable.java @@ -46,8 +46,9 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr protected kelondroBytesIntMap index; private boolean RAMIndex; - public kelondroFlexTable(File path, String tablename, long preloadTime, kelondroRow rowdef, boolean resetOnFail) { + public kelondroFlexTable(File path, String tablename, long preloadTime, kelondroRow rowdef, int minimumSpace, boolean resetOnFail) { // the buffersize applies to a possible load of the ram-index + // the minimumSpace is a initial allocation space for the index; names the number of index slots // if the ram is not sufficient, a tree file is generated // if, and only if a tree file exists, the preload time is applied super(path, tablename, rowdef, resetOnFail); @@ -59,7 +60,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr } try { - long neededRAM = (long) ((super.row().column(0).cellwidth + 4) * super.size() * kelondroRowCollection.growfactor); + long neededRAM = (long) ((super.row().column(0).cellwidth + 4) * Math.max(super.size(), minimumSpace) * kelondroRowCollection.growfactor); File newpath = new File(path, tablename); File indexfile = new File(newpath, "col.000.index"); @@ -82,7 +83,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr // fill the index System.out.print("*** Loading RAM index for " + size() + " entries from "+ newpath); - index = initializeRamIndex(); + index = initializeRamIndex(minimumSpace); System.out.println(" -done-"); System.out.println(index.size() @@ -151,8 +152,8 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr return index.geti(key) >= 0; } - private kelondroBytesIntMap initializeRamIndex() { - int space = super.col[0].size() + 1; + private kelondroBytesIntMap initializeRamIndex(int initialSpace) { + int space = Math.max(super.col[0].size(), initialSpace) + 1; if (space < 0) throw new kelondroException("wrong space: " + space); kelondroBytesIntMap ri = new kelondroBytesIntMap(super.row().column(0).cellwidth, super.rowdef.objectOrder, space); Iterator content = super.col[0].contentNodes(-1); @@ -449,7 +450,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr String name = args[1]; kelondroRow row = new kelondroRow("Cardinal key-4 {b256}, byte[] x-64", kelondroNaturalOrder.naturalOrder, 0); try { - kelondroFlexTable t = new kelondroFlexTable(f, name, 0, row, true); + kelondroFlexTable t = new kelondroFlexTable(f, name, 0, row, 0, true); kelondroRow.Entry entry = row.newEntry(); entry.setCol(0, System.currentTimeMillis()); entry.setCol(1, "dummy".getBytes()); diff --git a/source/de/anomic/plasma/plasmaCrawlBalancer.java b/source/de/anomic/plasma/plasmaCrawlBalancer.java index fe5213953..ea377bb3a 100644 --- a/source/de/anomic/plasma/plasmaCrawlBalancer.java +++ b/source/de/anomic/plasma/plasmaCrawlBalancer.java @@ -121,7 +121,7 @@ public class plasmaCrawlBalancer { private void openFileIndex() { cacheStacksPath.mkdirs(); - urlFileIndex = new kelondroCache(new kelondroFlexTable(cacheStacksPath, stackname + indexSuffix, -1, plasmaCrawlEntry.rowdef, true), true, false); + urlFileIndex = new kelondroCache(new kelondroFlexTable(cacheStacksPath, stackname + indexSuffix, -1, plasmaCrawlEntry.rowdef, 0, true), true, false); } private void resetFileIndex() { diff --git a/source/de/anomic/plasma/plasmaCrawlStacker.java b/source/de/anomic/plasma/plasmaCrawlStacker.java index 2eaf51365..66ed2436b 100644 --- a/source/de/anomic/plasma/plasmaCrawlStacker.java +++ b/source/de/anomic/plasma/plasmaCrawlStacker.java @@ -299,13 +299,13 @@ public final class plasmaCrawlStacker extends Thread { String newCacheName = "urlNoticeStacker8.db"; cacheStacksPath.mkdirs(); try { - this.urlEntryCache = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime, plasmaCrawlEntry.rowdef, true), true, false); + this.urlEntryCache = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime, plasmaCrawlEntry.rowdef, 0, true), true, false); } catch (Exception e) { e.printStackTrace(); // kill DB and try again kelondroFlexTable.delete(cacheStacksPath, newCacheName); try { - this.urlEntryCache = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime, plasmaCrawlEntry.rowdef, true), true, false); + this.urlEntryCache = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime, plasmaCrawlEntry.rowdef, 0, true), true, false); } catch (Exception ee) { ee.printStackTrace(); System.exit(-1); diff --git a/source/de/anomic/plasma/plasmaCrawlZURL.java b/source/de/anomic/plasma/plasmaCrawlZURL.java index 3d6d08a3c..241a8b72d 100644 --- a/source/de/anomic/plasma/plasmaCrawlZURL.java +++ b/source/de/anomic/plasma/plasmaCrawlZURL.java @@ -61,7 +61,7 @@ public class plasmaCrawlZURL { // creates a new ZURL in a file cachePath.mkdirs(); if (startWithEmptyFile) kelondroFlexTable.delete(cachePath, tablename); - urlIndex = new kelondroFlexTable(cachePath, tablename, -1, rowdef, true); + urlIndex = new kelondroFlexTable(cachePath, tablename, -1, rowdef, 0, true); } public plasmaCrawlZURL() { diff --git a/source/de/anomic/plasma/plasmaRankingCRProcess.java b/source/de/anomic/plasma/plasmaRankingCRProcess.java index cd9ccaece..4661ab998 100644 --- a/source/de/anomic/plasma/plasmaRankingCRProcess.java +++ b/source/de/anomic/plasma/plasmaRankingCRProcess.java @@ -261,7 +261,7 @@ public class plasmaRankingCRProcess { kelondroCollectionIndex newseq = null; if (newdb) { File path = to_file.getParentFile(); // path to storage place - newacc = new kelondroFlexTable(path, CRG_accname, -1, CRG_accrow, false); + newacc = new kelondroFlexTable(path, CRG_accname, -1, CRG_accrow, 0, false); newseq = new kelondroCollectionIndex(path, CRG_seqname, 12, kelondroBase64Order.enhancedCoder, -1, 2, 9, CRG_colrow); } else { if (!(to_file.exists())) {