re-implemented file index in kelondroFlex

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4323 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/kelondro/kelondroFlexTable.java

@@ -71,26 +71,43 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
         System.out.println("*** Last Startup time: " + stt + " milliseconds");
         long start = System.currentTimeMillis();
 
-        if (!serverMemory.request(neededRAM, true)) {
-            System.out.println("WARNING: NOT ENOUGH MEMORY FOR RAM INDEX " + new File(path, tablename).toString());
-        }
-        	
-        if (indexfile.exists()) {
-            // delete existing index file
-            System.out.println("*** Delete File index " + indexfile);
-            indexfile.delete();
-        }
-        
-        // fill the index
-        System.out.print("*** Loading RAM index for " + size() + " entries from "+ newpath);
-        index = initializeRamIndex(minimumSpace);
-            
-        System.out.println(" -done-");
-        System.out.println(index.size()
-                + " index entries initialized and sorted from "
-                + super.col[0].size() + " keys.");
-        RAMIndex = true;
-        tableTracker.put(this.filename(), this);
+        if (serverMemory.request(neededRAM, true)) {
+			// we can use a RAM index
+			if (indexfile.exists()) {
+				// delete existing index file
+				System.out.println("*** Delete File index " + indexfile);
+				indexfile.delete();
+			}
+
+			// fill the index
+			System.out.print("*** Loading RAM index for " + size() + " entries from " + newpath);
+			index = initializeRamIndex(minimumSpace);
+
+			System.out.println(" -done-");
+			System.out.println(index.size() + " index entries initialized and sorted from " + super.col[0].size() + " keys.");
+			RAMIndex = true;
+			tableTracker.put(this.filename(), this);
+		} else {
+			// too less ram for a ram index
+			kelondroIndex ki;
+			if (indexfile.exists()) {
+				// use existing index file
+				System.out.println("*** Using File index " + indexfile);
+				ki = new kelondroCache(kelondroTree.open(indexfile, true, preloadTime, treeIndexRow(rowdef.width(0), rowdef.objectOrder), 2, 80), true, false);
+				RAMIndex = false;
+			} else {
+				// generate new index file
+				System.out.println("*** Generating File index for " + size() + " entries from " + indexfile);
+				System.out.println("*** Cause: too less RAM (" + serverMemory.available() + " Bytes) configured. Assign at least " + (neededRAM / 1024 / 1024) + " MB more RAM to enable a RAM index.");
+				ki = initializeTreeIndex(indexfile, preloadTime, rowdef.objectOrder);
+
+				System.out.println(" -done-");
+				System.out.println(ki.size() + " entries indexed from " + super.col[0].size() + " keys.");
+				RAMIndex = false;
+			}
+			index = new kelondroBytesIntMap(ki);
+			assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
+		}
         
         // assign index to wrapper
         description = "stt=" + Long.toString(System.currentTimeMillis() - start) + ";";
@@ -156,6 +173,36 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
         return ri;
     }
     
+
+        private kelondroIndex initializeTreeIndex(File indexfile, long preloadTime, kelondroByteOrder objectOrder) throws IOException {
+		kelondroIndex treeindex = new kelondroCache(new kelondroTree(indexfile, true, preloadTime, treeIndexRow(rowdef.primaryKeyLength, objectOrder), 2, 80), true, false);
+		Iterator<kelondroNode> content = super.col[0].contentNodes(-1);
+		kelondroNode node;
+		kelondroRow.Entry indexentry;
+		int i, c = 0, all = super.col[0].size();
+		long start = System.currentTimeMillis();
+		long last = start;
+		while (content.hasNext()) {
+			node = content.next();
+			i = node.handle().hashCode();
+			indexentry = treeindex.row().newEntry();
+			indexentry.setCol(0, node.getValueRow());
+			indexentry.setCol(1, i);
+			treeindex.addUnique(indexentry);
+			c++;
+			if (System.currentTimeMillis() - last > 30000) {
+				System.out.println(".. generated " + c + "/" + all + " entries, " + ((System.currentTimeMillis() - start) / c * (all - c) / 60000) + " minutes remaining");
+				System.out.flush();
+				last = System.currentTimeMillis();
+			}
+		}
+		return treeindex;
+	}
+
+	private static final kelondroRow treeIndexRow(int keywidth, kelondroByteOrder objectOrder) {
+		return new kelondroRow("byte[] key-" + keywidth + ", int reference-4 {b256}", objectOrder, 0);
+	}
+    
     public synchronized kelondroRow.Entry get(byte[] key) throws IOException {
         if (index == null) return null; // case may happen during shutdown
 		int pos = index.geti(key);