better RAM protection using eco tables

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4345 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/kelondro/kelondroCollectionIndex.java

@@ -154,12 +154,17 @@ public class kelondroCollectionIndex {
                 }
             }
             serverLog.logFine("STARTUP", "STARTED INITIALIZATION OF NEW COLLECTION INDEX WITH " + initialSpace + " ENTRIES. THIS WILL TAKE SOME TIME");
-
+            kelondroRow indexRowdef = indexRow(keyLength, indexOrder);
+            long necessaryRAM4fullTable = minimumRAM4Eco + (indexRowdef.objectsize + 4) * initialSpace * 3 / 2;
+            long necessaryRAM4fullIndex = minimumRAM4Eco + (indexRowdef.primaryKeyLength + 4) * initialSpace * 3 / 2;
+            
             // initialize (new generation) index table from file
-            if (serverMemory.request(minimumRAM4Eco, false)) {
-                index = new kelondroEcoTable(f, indexRow(keyLength, indexOrder), true, EcoFSBufferSize);
+            if (serverMemory.request(necessaryRAM4fullTable, false)) {
+                index = new kelondroEcoTable(f, indexRowdef, true, EcoFSBufferSize);
+            } else if (serverMemory.request(necessaryRAM4fullIndex, false)) {
+                index = new kelondroEcoTable(f, indexRowdef, false, EcoFSBufferSize);
             } else {
-                index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRow(keyLength, indexOrder), initialSpace, true);
+                index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRowdef, initialSpace, true);
             }
             
             // open array files
@@ -233,9 +238,11 @@ public class kelondroCollectionIndex {
             long preloadTime, int loadfactor, kelondroRow rowdef, int initialSpace) throws IOException {
         // open/create index table
         File f = new File(path, filenameStub + ".index");
+        kelondroRow indexRowdef = indexRow(keylength, indexOrder);
+        
         if (f.isDirectory()) {
             // use a flextable
-            kelondroIndex theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRow(keylength, indexOrder), initialSpace, true));
+            kelondroIndex theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRowdef, initialSpace, true));
         
             // save/check property file for this array
             File propfile = propertyFile(path, filenameStub, loadfactor, rowdef.objectsize);
@@ -255,7 +262,9 @@ public class kelondroCollectionIndex {
             return theindex;
         } else {
             // open a ecotable
-            return new kelondroEcoTable(f, indexRow(keylength, indexOrder), true, EcoFSBufferSize);
+            long records = f.length() / indexRowdef.objectsize;
+            long necessaryRAM4fullTable = minimumRAM4Eco + (indexRowdef.objectsize + 4) * records * 3 / 2;
+            return new kelondroEcoTable(f, indexRowdef, serverMemory.request(necessaryRAM4fullTable, false), EcoFSBufferSize);
         }
     }
     

source/de/anomic/kelondro/kelondroEcoTable.java

@@ -90,7 +90,7 @@ public class kelondroEcoTable implements kelondroIndex {
         
             // initialize index and copy table
             int records = file.size();
-            long neededRAM4table = records * taildef.objectsize * 3 / 2;
+            long neededRAM4table = 10 * 1024 * 1024 + records * (rowdef.objectsize + 4) * 3 / 2;
             table = ((useTailCache) && (serverMemory.request(neededRAM4table, true))) ? new kelondroRowSet(taildef, records + 1) : null;
             index = new kelondroBytesIntMap(rowdef.primaryKeyLength, rowdef.objectOrder, records + 1);
         

source/de/anomic/plasma/plasmaCrawlBalancer.java

@@ -80,6 +80,7 @@ public class plasmaCrawlBalancer {
     private File                                cacheStacksPath;
     private String                              stackname;
     private boolean                             top;             // to alternate between top and bottom of the file stack
+    private boolean                             fullram;
 
     public static class domaccess {
     	long time;
@@ -100,7 +101,7 @@ public class plasmaCrawlBalancer {
     	}
     }
     
-    public plasmaCrawlBalancer(File cachePath, String stackname) {
+    public plasmaCrawlBalancer(File cachePath, String stackname, boolean fullram) {
         this.cacheStacksPath = cachePath;
         this.stackname = stackname;
         File stackFile = new File(cachePath, stackname + stackSuffix);
@@ -108,6 +109,7 @@ public class plasmaCrawlBalancer {
         this.domainStacks   = new HashMap<String, LinkedList<String>>();
         this.urlRAMStack    = new ArrayList<String>();
         this.top            = true;
+        this.fullram        = fullram;
         
         // create a stack for newly entered entries
         if (!(cachePath.exists())) cachePath.mkdir(); // make the path
@@ -140,7 +142,7 @@ public class plasmaCrawlBalancer {
     
     private void openFileIndex() {
         cacheStacksPath.mkdirs();
-        urlFileIndex = new kelondroEcoTable(new File(cacheStacksPath, stackname + indexSuffix), plasmaCrawlEntry.rowdef, true, EcoFSBufferSize);
+        urlFileIndex = new kelondroEcoTable(new File(cacheStacksPath, stackname + indexSuffix), plasmaCrawlEntry.rowdef, fullram, EcoFSBufferSize);
     }
     
     private void resetFileIndex() {

source/de/anomic/plasma/plasmaCrawlNURL.java

@@ -74,10 +74,10 @@ public class plasmaCrawlNURL {
 
     public plasmaCrawlNURL(File cachePath) {
         super();
-        coreStack = new plasmaCrawlBalancer(cachePath, "urlNoticeCoreStack");
-        limitStack = new plasmaCrawlBalancer(cachePath, "urlNoticeLimitStack");
+        coreStack = new plasmaCrawlBalancer(cachePath, "urlNoticeCoreStack", true);
+        limitStack = new plasmaCrawlBalancer(cachePath, "urlNoticeLimitStack", false);
         //overhangStack = new plasmaCrawlBalancer(overhangStackFile);
-        remoteStack = new plasmaCrawlBalancer(cachePath, "urlNoticeRemoteStack");
+        remoteStack = new plasmaCrawlBalancer(cachePath, "urlNoticeRemoteStack", true);
     }
 
     public void close() {
@@ -213,7 +213,7 @@ public class plasmaCrawlNURL {
     private plasmaCrawlEntry[] top(plasmaCrawlBalancer balancer, int count) {
         // this is a filo - top
         if (count > balancer.size()) count = balancer.size();
-        ArrayList list = new ArrayList(count);
+        ArrayList<plasmaCrawlEntry> list = new ArrayList<plasmaCrawlEntry>(count);
         for (int i = 0; i < count; i++) {
             try {
                 plasmaCrawlEntry entry = balancer.top(i);
@@ -226,15 +226,15 @@ public class plasmaCrawlNURL {
         return (plasmaCrawlEntry[]) list.toArray(new plasmaCrawlEntry[list.size()]);
     }
     
-    public Iterator iterator(int stackType) {
+    public Iterator<plasmaCrawlEntry> iterator(int stackType) {
         // returns an iterator of plasmaCrawlBalancerEntry Objects
         try {switch (stackType) {
-        case STACK_TYPE_CORE:     return coreStack.iterator();
-        case STACK_TYPE_LIMIT:    return limitStack.iterator();
-        case STACK_TYPE_REMOTE:   return remoteStack.iterator();
-        default: return null;
+            case STACK_TYPE_CORE:     return coreStack.iterator();
+            case STACK_TYPE_LIMIT:    return limitStack.iterator();
+            case STACK_TYPE_REMOTE:   return remoteStack.iterator();
+            default: return null;
         }} catch (IOException e) {
-            return new HashSet().iterator();
+            return new HashSet<plasmaCrawlEntry>().iterator();
         }
     }