From eafb5ecd22442ee1c175e9a84bc8d44d309a8991 Mon Sep 17 00:00:00 2001 From: orbiter Date: Sun, 5 Nov 2006 21:30:53 +0000 Subject: [PATCH] - better usage of memory resources for kelondroFlexSplit - kelondroFlexTables does always load a RAM cache if it has enough ram assigned. Othervise it creates a kelondroTree file-index. If more memory is re-assigned, the file-index is deleted again, and RAM is used. Beware that assignement of too less RAM forces creation of file indexes and start-up time may last for hours. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2923 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- .../kelondro/kelondroFlexSplitTable.java | 31 ++++++--- .../de/anomic/kelondro/kelondroFlexTable.java | 64 +++++++++++-------- .../kelondro/kelondroFlexWidthArray.java | 15 +++++ .../de/anomic/kelondro/kelondroRecords.java | 19 ++++++ 4 files changed, 96 insertions(+), 33 deletions(-) diff --git a/source/de/anomic/kelondro/kelondroFlexSplitTable.java b/source/de/anomic/kelondro/kelondroFlexSplitTable.java index 5c84f2893..530027525 100644 --- a/source/de/anomic/kelondro/kelondroFlexSplitTable.java +++ b/source/de/anomic/kelondro/kelondroFlexSplitTable.java @@ -32,6 +32,7 @@ import java.util.Calendar; import java.util.Date; import java.util.HashMap; import java.util.Iterator; +import java.util.Map; public class kelondroFlexSplitTable implements kelondroIndex { @@ -57,20 +58,34 @@ public class kelondroFlexSplitTable implements kelondroIndex { String[] dir = path.list(); String date; - // first pass: count tables - int count = 0; - for (int i = 0; i < dir.length; i++) if (dir[i].startsWith(tablename)) count++; - - // second pass: open tables + // first pass: find tables + HashMap t = new HashMap(); // file/Integer(size) relation + int size, sum = 0; for (int i = 0; i < dir.length; i++) { if ((dir[i].startsWith(tablename)) && (dir[i].charAt(tablename.length()) == '.') && (dir[i].length() == tablename.length() + 7)) { - // open table - date = dir[i].substring(tablename.length() + 1); - this.tables.put(date, new kelondroCache(new kelondroFlexTable(path, dir[i], buffersize / count / 2, preloadTime, rowdef, objectOrder), buffersize / count / 2, true, false)); + size = kelondroFlexTable.staticSize(path, dir[i]); + if (size > 0) { + t.put(dir[i], new Integer(size)); + sum += size; + } } } + + // second pass: open tables + Iterator i = t.entrySet().iterator(); + Map.Entry entry; + String f; + long bs; + while (i.hasNext()) { + entry = (Map.Entry) i.next(); + f = (String) entry.getKey(); + size = ((Integer) entry.getValue()).intValue(); + date = f.substring(tablename.length() + 1); + bs = buffersize * size / sum; + tables.put(date, new kelondroCache(new kelondroFlexTable(path, f, bs / 2, preloadTime, rowdef, objectOrder), bs / 2, true, false)); + } } private static final Calendar thisCalendar = Calendar.getInstance(); diff --git a/source/de/anomic/kelondro/kelondroFlexTable.java b/source/de/anomic/kelondro/kelondroFlexTable.java index 547454fef..69cb2e528 100644 --- a/source/de/anomic/kelondro/kelondroFlexTable.java +++ b/source/de/anomic/kelondro/kelondroFlexTable.java @@ -38,8 +38,13 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr private boolean RAMIndex; public kelondroFlexTable(File path, String tablename, long buffersize, long preloadTime, kelondroRow rowdef, kelondroOrder objectOrder) throws IOException { - super(path, tablename, rowdef); - File newpath = new File(path, tablename); + // the buffersize applies to a possible load of the ram-index + // if the ram is not sufficient, a tree file is generated + // if, and only if a tree file exists, the preload time is applied + super(path, tablename, rowdef); + long neededRAM = (super.row().column(0).cellwidth() + 4) * 12 / 10 * super.size(); + + File newpath = new File(path, tablename); File indexfile = new File(newpath, "col.000.index"); kelondroIndex ki = null; String description = new String(this.col[0].getDescription()); @@ -48,28 +53,16 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr System.out.println("*** Last Startup time: " + stt + " milliseconds"); long start = System.currentTimeMillis(); - if ((preloadTime < 0) && (indexfile.exists())) { - // delete existing index file - System.out.println("*** Delete File index " + indexfile); - indexfile.delete(); - } - if (indexfile.exists()) { - // use existing index file - System.out.println("*** Using File index " + indexfile); - ki = new kelondroCache(kelondroTree.open(indexfile, buffersize / 2, preloadTime, treeIndexRow(rowdef.width(0)), objectOrder, 2, 80), buffersize / 2, true, false); - RAMIndex = false; - } else if ((preloadTime >= 0) && (stt > preloadTime)) { - // generate new index file - System.out.print("*** Generating File index for " + size() + " entries from " + indexfile); - ki = initializeTreeIndex(indexfile, buffersize, preloadTime, objectOrder); - - System.out.println(" -done-"); - System.out.println(ki.size() - + " entries indexed from " - + super.col[0].size() + " keys."); - RAMIndex = false; - } else { - // fill the index + if (buffersize >= neededRAM) { + // we can use a RAM index + + if (indexfile.exists()) { + // delete existing index file + System.out.println("*** Delete File index " + indexfile); + indexfile.delete(); + } + + // fill the index System.out.print("*** Loading RAM index for " + size() + " entries from "+ newpath); ki = initializeRamIndex(objectOrder); @@ -78,6 +71,23 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr + " index entries initialized and sorted from " + super.col[0].size() + " keys."); RAMIndex = true; + } else { + // too less ram for a ram index + if (indexfile.exists()) { + // use existing index file + System.out.println("*** Using File index " + indexfile); + ki = new kelondroCache(kelondroTree.open(indexfile, buffersize / 3 * 2, preloadTime, treeIndexRow(rowdef.width(0)), objectOrder, 2, 80), buffersize / 3, true, false); + RAMIndex = false; + } else if ((preloadTime >= 0) && (stt > preloadTime)) { + // generate new index file + System.out.print("*** Generating File index for " + size() + " entries from " + indexfile); + System.out.print("*** Cause: too less RAM configured. Assign at least " + neededRAM + " bytes buffersize to enable a RAM index."); + ki = initializeTreeIndex(indexfile, buffersize, preloadTime, objectOrder); + + System.out.println(" -done-"); + System.out.println(ki.size() + " entries indexed from " + super.col[0].size() + " keys."); + RAMIndex = false; + } } // assign index to wrapper index = new kelondroBytesIntMap(ki); @@ -85,6 +95,10 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr super.col[0].setDescription(description.getBytes()); } + public static int staticSize(File path, String tablename) { + return kelondroFlexWidthArray.staticsize(path, tablename); + } + public boolean hasRAMIndex() { return RAMIndex; } @@ -127,7 +141,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr } private kelondroIndex initializeTreeIndex(File indexfile, long buffersize, long preloadTime, kelondroOrder objectOrder) throws IOException { - kelondroIndex treeindex = new kelondroCache(new kelondroTree(indexfile, buffersize / 2, preloadTime, treeIndexRow(rowdef.width(0)), objectOrder, 2, 80), buffersize / 2, true, false); + kelondroIndex treeindex = new kelondroCache(new kelondroTree(indexfile, buffersize / 3 * 2, preloadTime, treeIndexRow(rowdef.width(0)), objectOrder, 2, 80), buffersize / 3, true, false); Iterator content = super.col[0].contentNodes(-1); kelondroRecords.Node node; kelondroRow.Entry indexentry; diff --git a/source/de/anomic/kelondro/kelondroFlexWidthArray.java b/source/de/anomic/kelondro/kelondroFlexWidthArray.java index d16a55cda..c50d04b24 100644 --- a/source/de/anomic/kelondro/kelondroFlexWidthArray.java +++ b/source/de/anomic/kelondro/kelondroFlexWidthArray.java @@ -104,6 +104,21 @@ public class kelondroFlexWidthArray implements kelondroArray { } } + public static int staticsize(File path, String tablename) { + + // check if table directory exists + File tabledir = new File(path, tablename); + if (tabledir.exists()) { + if (!(tabledir.isDirectory())) return 0; + } else { + return 0; + } + + // open existing files + File file = new File(tabledir, "col.000.list"); + return kelondroRecords.staticsize(file); + } + public static void delete(File path, String tablename) { File tabledir = new File(path, tablename); if ((tabledir.exists()) && (!(tabledir.isDirectory()))) { diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java index 9643db938..d476445aa 100644 --- a/source/de/anomic/kelondro/kelondroRecords.java +++ b/source/de/anomic/kelondro/kelondroRecords.java @@ -69,6 +69,7 @@ package de.anomic.kelondro; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; import java.util.HashSet; import java.util.Iterator; @@ -192,6 +193,24 @@ public class kelondroRecords { return this.USEDC + this.FREEC; } } + + public static int staticsize(File file) { + if (!(file.exists())) return 0; + try { + kelondroRA ra = new kelondroFileRA(file.getCanonicalPath()); + kelondroIOChunks entryFile = new kelondroRAIOChunks(ra, ra.name()); + + int USEDC = entryFile.readInt(POS_USEDC); + entryFile.close(); + ra.close(); + return USEDC; + } catch (FileNotFoundException e) { + return 0; + } catch (IOException e) { + return 0; + } + } + public kelondroRecords(File file, long buffersize /* bytes */, long preloadTime, short ohbytec, short ohhandlec,