diff --git a/source/de/anomic/kelondro/kelondroCollectionIndex.java b/source/de/anomic/kelondro/kelondroCollectionIndex.java index 66019ec5b..d0ee1bf4f 100644 --- a/source/de/anomic/kelondro/kelondroCollectionIndex.java +++ b/source/de/anomic/kelondro/kelondroCollectionIndex.java @@ -154,12 +154,17 @@ public class kelondroCollectionIndex { } } serverLog.logFine("STARTUP", "STARTED INITIALIZATION OF NEW COLLECTION INDEX WITH " + initialSpace + " ENTRIES. THIS WILL TAKE SOME TIME"); - + kelondroRow indexRowdef = indexRow(keyLength, indexOrder); + long necessaryRAM4fullTable = minimumRAM4Eco + (indexRowdef.objectsize + 4) * initialSpace * 3 / 2; + long necessaryRAM4fullIndex = minimumRAM4Eco + (indexRowdef.primaryKeyLength + 4) * initialSpace * 3 / 2; + // initialize (new generation) index table from file - if (serverMemory.request(minimumRAM4Eco, false)) { - index = new kelondroEcoTable(f, indexRow(keyLength, indexOrder), true, EcoFSBufferSize); + if (serverMemory.request(necessaryRAM4fullTable, false)) { + index = new kelondroEcoTable(f, indexRowdef, true, EcoFSBufferSize); + } else if (serverMemory.request(necessaryRAM4fullIndex, false)) { + index = new kelondroEcoTable(f, indexRowdef, false, EcoFSBufferSize); } else { - index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRow(keyLength, indexOrder), initialSpace, true); + index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRowdef, initialSpace, true); } // open array files @@ -233,9 +238,11 @@ public class kelondroCollectionIndex { long preloadTime, int loadfactor, kelondroRow rowdef, int initialSpace) throws IOException { // open/create index table File f = new File(path, filenameStub + ".index"); + kelondroRow indexRowdef = indexRow(keylength, indexOrder); + if (f.isDirectory()) { // use a flextable - kelondroIndex theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRow(keylength, indexOrder), initialSpace, true)); + kelondroIndex theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRowdef, initialSpace, true)); // save/check property file for this array File propfile = propertyFile(path, filenameStub, loadfactor, rowdef.objectsize); @@ -255,7 +262,9 @@ public class kelondroCollectionIndex { return theindex; } else { // open a ecotable - return new kelondroEcoTable(f, indexRow(keylength, indexOrder), true, EcoFSBufferSize); + long records = f.length() / indexRowdef.objectsize; + long necessaryRAM4fullTable = minimumRAM4Eco + (indexRowdef.objectsize + 4) * records * 3 / 2; + return new kelondroEcoTable(f, indexRowdef, serverMemory.request(necessaryRAM4fullTable, false), EcoFSBufferSize); } } diff --git a/source/de/anomic/kelondro/kelondroEcoTable.java b/source/de/anomic/kelondro/kelondroEcoTable.java index f08ad6f28..85082768f 100644 --- a/source/de/anomic/kelondro/kelondroEcoTable.java +++ b/source/de/anomic/kelondro/kelondroEcoTable.java @@ -90,7 +90,7 @@ public class kelondroEcoTable implements kelondroIndex { // initialize index and copy table int records = file.size(); - long neededRAM4table = records * taildef.objectsize * 3 / 2; + long neededRAM4table = 10 * 1024 * 1024 + records * (rowdef.objectsize + 4) * 3 / 2; table = ((useTailCache) && (serverMemory.request(neededRAM4table, true))) ? new kelondroRowSet(taildef, records + 1) : null; index = new kelondroBytesIntMap(rowdef.primaryKeyLength, rowdef.objectOrder, records + 1); diff --git a/source/de/anomic/plasma/plasmaCrawlBalancer.java b/source/de/anomic/plasma/plasmaCrawlBalancer.java index 78098a6c3..a1e9f384a 100644 --- a/source/de/anomic/plasma/plasmaCrawlBalancer.java +++ b/source/de/anomic/plasma/plasmaCrawlBalancer.java @@ -80,6 +80,7 @@ public class plasmaCrawlBalancer { private File cacheStacksPath; private String stackname; private boolean top; // to alternate between top and bottom of the file stack + private boolean fullram; public static class domaccess { long time; @@ -100,7 +101,7 @@ public class plasmaCrawlBalancer { } } - public plasmaCrawlBalancer(File cachePath, String stackname) { + public plasmaCrawlBalancer(File cachePath, String stackname, boolean fullram) { this.cacheStacksPath = cachePath; this.stackname = stackname; File stackFile = new File(cachePath, stackname + stackSuffix); @@ -108,6 +109,7 @@ public class plasmaCrawlBalancer { this.domainStacks = new HashMap>(); this.urlRAMStack = new ArrayList(); this.top = true; + this.fullram = fullram; // create a stack for newly entered entries if (!(cachePath.exists())) cachePath.mkdir(); // make the path @@ -140,7 +142,7 @@ public class plasmaCrawlBalancer { private void openFileIndex() { cacheStacksPath.mkdirs(); - urlFileIndex = new kelondroEcoTable(new File(cacheStacksPath, stackname + indexSuffix), plasmaCrawlEntry.rowdef, true, EcoFSBufferSize); + urlFileIndex = new kelondroEcoTable(new File(cacheStacksPath, stackname + indexSuffix), plasmaCrawlEntry.rowdef, fullram, EcoFSBufferSize); } private void resetFileIndex() { diff --git a/source/de/anomic/plasma/plasmaCrawlNURL.java b/source/de/anomic/plasma/plasmaCrawlNURL.java index bea3725b1..235911564 100644 --- a/source/de/anomic/plasma/plasmaCrawlNURL.java +++ b/source/de/anomic/plasma/plasmaCrawlNURL.java @@ -74,10 +74,10 @@ public class plasmaCrawlNURL { public plasmaCrawlNURL(File cachePath) { super(); - coreStack = new plasmaCrawlBalancer(cachePath, "urlNoticeCoreStack"); - limitStack = new plasmaCrawlBalancer(cachePath, "urlNoticeLimitStack"); + coreStack = new plasmaCrawlBalancer(cachePath, "urlNoticeCoreStack", true); + limitStack = new plasmaCrawlBalancer(cachePath, "urlNoticeLimitStack", false); //overhangStack = new plasmaCrawlBalancer(overhangStackFile); - remoteStack = new plasmaCrawlBalancer(cachePath, "urlNoticeRemoteStack"); + remoteStack = new plasmaCrawlBalancer(cachePath, "urlNoticeRemoteStack", true); } public void close() { @@ -213,7 +213,7 @@ public class plasmaCrawlNURL { private plasmaCrawlEntry[] top(plasmaCrawlBalancer balancer, int count) { // this is a filo - top if (count > balancer.size()) count = balancer.size(); - ArrayList list = new ArrayList(count); + ArrayList list = new ArrayList(count); for (int i = 0; i < count; i++) { try { plasmaCrawlEntry entry = balancer.top(i); @@ -226,15 +226,15 @@ public class plasmaCrawlNURL { return (plasmaCrawlEntry[]) list.toArray(new plasmaCrawlEntry[list.size()]); } - public Iterator iterator(int stackType) { + public Iterator iterator(int stackType) { // returns an iterator of plasmaCrawlBalancerEntry Objects try {switch (stackType) { - case STACK_TYPE_CORE: return coreStack.iterator(); - case STACK_TYPE_LIMIT: return limitStack.iterator(); - case STACK_TYPE_REMOTE: return remoteStack.iterator(); - default: return null; + case STACK_TYPE_CORE: return coreStack.iterator(); + case STACK_TYPE_LIMIT: return limitStack.iterator(); + case STACK_TYPE_REMOTE: return remoteStack.iterator(); + default: return null; }} catch (IOException e) { - return new HashSet().iterator(); + return new HashSet().iterator(); } }