enhanced caching, removed bug causing outOfMemory

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@195 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/plasma/plasmaWordIndexAssortment.java

@@ -107,18 +107,18 @@ public final class plasmaWordIndexAssortment {
             // open existing singeton tree file
             try {
                 assortments = new kelondroTree(assortmentFile, bufferSize);
-                log.logSystem("Opened Assortment Database, " + assortments.size() + " entries, width " + assortmentCapacity); 
+                if (log != null) log.logSystem("Opened Assortment Database, " + assortments.size() + " entries, width " + assortmentCapacity + ", " + bufferkb + "kb buffer"); 
             } catch (IOException e){
-                log.logError("unable to open assortment database: " + e.getMessage());
+                if (log != null) log.logError("unable to open assortment database: " + e.getMessage());
                 e.printStackTrace();
             }
         } else {
             // create new sigleton tree file
             try {
                 assortments = new kelondroTree(assortmentFile, bufferSize, bufferStructure(assortmentCapacity));
-                log.logSystem("Created new Assortment Database, width " + assortmentCapacity); 
+                if (log != null) log.logSystem("Created new Assortment Database, width " + assortmentCapacity + ", " + bufferkb + "kb buffer"); 
             } catch (IOException e){
-                log.logError("unable to create assortment database: " + e.getMessage());
+                if (log != null) log.logError("unable to create assortment database: " + e.getMessage());
                 e.printStackTrace();
             }
         }

source/de/anomic/plasma/plasmaWordIndexAssortmentCluster.java

@@ -58,19 +58,30 @@ public final class plasmaWordIndexAssortmentCluster {
     private int clusterCapacity;
     private serverLog log;
     private plasmaWordIndexAssortment[] assortments;
-    private long completeBufferSize;
+    private long completeBufferKB;
 
     public plasmaWordIndexAssortmentCluster(File assortmentsPath, int clusterCapacity, int bufferkb, serverLog log) {
 	// set class variables
 	if (!(assortmentsPath.exists())) assortmentsPath.mkdirs();
 	this.clusterCapacity = clusterCapacity;
-        this.completeBufferSize = bufferkb * 1024;
+        this.completeBufferKB = bufferkb;
         this.log = log;
 	this.assortments = new plasmaWordIndexAssortment[clusterCapacity];
 
-	// initialize cluster
+        // open cluster and close it directly again to detect the element sizes
+        int[] sizes = new int[clusterCapacity];
+        int sumSizes = 1;
+        plasmaWordIndexAssortment testAssortment;
+        for (int i = 0; i < clusterCapacity; i++) {
+	    testAssortment = new plasmaWordIndexAssortment(assortmentsPath, i + 1, 0, null);
+            sizes[i] = testAssortment.size() + clusterCapacity - i;
+            sumSizes += sizes[i];
+            testAssortment.close();
+	}
+        
+	// initialize cluster using the cluster elements size for optimal buffer size
 	for (int i = 0; i < clusterCapacity; i++) {
-	    assortments[i] = new plasmaWordIndexAssortment(assortmentsPath, i + 1, (int) completeBufferSize / clusterCapacity, log);
+	    assortments[i] = new plasmaWordIndexAssortment(assortmentsPath, i + 1, (int) completeBufferKB * sizes[i] / sumSizes, log);
 	}
     }
 

source/de/anomic/plasma/plasmaWordIndexCache.java

@@ -56,8 +56,8 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
     private static final String oldSingletonFileName = "indexSingletons0.db";
     private static final String newSingletonFileName = "indexAssortment001.db";
     private static final String indexAssortmentClusterPath = "ACLUSTER";
-    private static final int assortmentLimit = 20;
-    private static final int ramcacheLimit = 60;
+    private static final int assortmentLimit = 50;
+    private static final int ramcacheLimit = 70;
     
     
     // class variables