boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- better usage of memory resources for kelondroFlexSplit

Browse Source 
- kelondroFlexTables does always load a RAM cache if it has enough
  ram assigned. Othervise it creates a kelondroTree file-index.
  If more memory is re-assigned, the file-index is deleted again,
  and RAM is used. Beware that assignement of too less RAM forces
  creation of file indexes and start-up time may last for hours.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2923 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 18 years ago
parent b2d51be33c
commit eafb5ecd22
4 changed files with 96 additions and 33 deletions
Show Stats Download Patch File Download Diff File

31
source/de/anomic/kelondro/kelondroFlexSplitTable.java
Unescape View File

@ -32,6 +32,7 @@ import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
public class kelondroFlexSplitTable implements kelondroIndex {
@ -57,20 +58,34 @@ public class kelondroFlexSplitTable implements kelondroIndex {
String[] dir = path.list();
String date;
// first pass: count tables
int count = 0;
for (int i = 0; i < dir.length; i++) if (dir[i].startsWith(tablename)) count++;
// second pass: open tables
// first pass: find tables
HashMap t = new HashMap(); // file/Integer(size) relation
int size, sum = 0;
for (int i = 0; i < dir.length; i++) {
if ((dir[i].startsWith(tablename)) &&
(dir[i].charAt(tablename.length()) == '.') &&
(dir[i].length() == tablename.length() + 7)) {
// open table
date = dir[i].substring(tablename.length() + 1);
this.tables.put(date, new kelondroCache(new kelondroFlexTable(path, dir[i], buffersize / count / 2, preloadTime, rowdef, objectOrder), buffersize / count / 2, true, false));
size = kelondroFlexTable.staticSize(path, dir[i]);
if (size > 0) {
t.put(dir[i], new Integer(size));
sum += size;
}
}
}
// second pass: open tables
Iterator i = t.entrySet().iterator();
Map.Entry entry;
String f;
long bs;
while (i.hasNext()) {
entry = (Map.Entry) i.next();
f = (String) entry.getKey();
size = ((Integer) entry.getValue()).intValue();
date = f.substring(tablename.length() + 1);
bs = buffersize * size / sum;
tables.put(date, new kelondroCache(new kelondroFlexTable(path, f, bs / 2, preloadTime, rowdef, objectOrder), bs / 2, true, false));
}
}
private static final Calendar thisCalendar = Calendar.getInstance();

64
source/de/anomic/kelondro/kelondroFlexTable.java
Unescape View File

@ -38,8 +38,13 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
private boolean RAMIndex;
public kelondroFlexTable(File path, String tablename, long buffersize, long preloadTime, kelondroRow rowdef, kelondroOrder objectOrder) throws IOException {
super(path, tablename, rowdef);
File newpath = new File(path, tablename);
// the buffersize applies to a possible load of the ram-index
// if the ram is not sufficient, a tree file is generated
// if, and only if a tree file exists, the preload time is applied
super(path, tablename, rowdef);
long neededRAM = (super.row().column(0).cellwidth() + 4) * 12 / 10 * super.size();
File newpath = new File(path, tablename);
File indexfile = new File(newpath, "col.000.index");
kelondroIndex ki = null;
String description = new String(this.col[0].getDescription());
@ -48,28 +53,16 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
System.out.println("*** Last Startup time: " + stt + " milliseconds");
long start = System.currentTimeMillis();
if ((preloadTime < 0) && (indexfile.exists())) {
// delete existing index file
System.out.println("*** Delete File index " + indexfile);
indexfile.delete();
}
if (indexfile.exists()) {
// use existing index file
System.out.println("*** Using File index " + indexfile);
ki = new kelondroCache(kelondroTree.open(indexfile, buffersize / 2, preloadTime, treeIndexRow(rowdef.width(0)), objectOrder, 2, 80), buffersize / 2, true, false);
RAMIndex = false;
} else if ((preloadTime >= 0) && (stt > preloadTime)) {
// generate new index file
System.out.print("*** Generating File index for " + size() + " entries from " + indexfile);
ki = initializeTreeIndex(indexfile, buffersize, preloadTime, objectOrder);
System.out.println(" -done-");
System.out.println(ki.size()
+ " entries indexed from "
+ super.col[0].size() + " keys.");
RAMIndex = false;
} else {
// fill the index
if (buffersize >= neededRAM) {
// we can use a RAM index
if (indexfile.exists()) {
// delete existing index file
System.out.println("*** Delete File index " + indexfile);
indexfile.delete();
}
// fill the index
System.out.print("*** Loading RAM index for " + size() + " entries from "+ newpath);
ki = initializeRamIndex(objectOrder);
@ -78,6 +71,23 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
+ " index entries initialized and sorted from "
+ super.col[0].size() + " keys.");
RAMIndex = true;
} else {
// too less ram for a ram index
if (indexfile.exists()) {
// use existing index file
System.out.println("*** Using File index " + indexfile);
ki = new kelondroCache(kelondroTree.open(indexfile, buffersize / 3 * 2, preloadTime, treeIndexRow(rowdef.width(0)), objectOrder, 2, 80), buffersize / 3, true, false);
RAMIndex = false;
} else if ((preloadTime >= 0) && (stt > preloadTime)) {
// generate new index file
System.out.print("*** Generating File index for " + size() + " entries from " + indexfile);
System.out.print("*** Cause: too less RAM configured. Assign at least " + neededRAM + " bytes buffersize to enable a RAM index.");
ki = initializeTreeIndex(indexfile, buffersize, preloadTime, objectOrder);
System.out.println(" -done-");
System.out.println(ki.size() + " entries indexed from " + super.col[0].size() + " keys.");
RAMIndex = false;
}
}
// assign index to wrapper
index = new kelondroBytesIntMap(ki);
@ -85,6 +95,10 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
super.col[0].setDescription(description.getBytes());
}
public static int staticSize(File path, String tablename) {
return kelondroFlexWidthArray.staticsize(path, tablename);
}
public boolean hasRAMIndex() {
return RAMIndex;
}
@ -127,7 +141,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
}
private kelondroIndex initializeTreeIndex(File indexfile, long buffersize, long preloadTime, kelondroOrder objectOrder) throws IOException {
kelondroIndex treeindex = new kelondroCache(new kelondroTree(indexfile, buffersize / 2, preloadTime, treeIndexRow(rowdef.width(0)), objectOrder, 2, 80), buffersize / 2, true, false);
kelondroIndex treeindex = new kelondroCache(new kelondroTree(indexfile, buffersize / 3 * 2, preloadTime, treeIndexRow(rowdef.width(0)), objectOrder, 2, 80), buffersize / 3, true, false);
Iterator content = super.col[0].contentNodes(-1);
kelondroRecords.Node node;
kelondroRow.Entry indexentry;

15
source/de/anomic/kelondro/kelondroFlexWidthArray.java
Unescape View File

@ -104,6 +104,21 @@ public class kelondroFlexWidthArray implements kelondroArray {
}
}
public static int staticsize(File path, String tablename) {
// check if table directory exists
File tabledir = new File(path, tablename);
if (tabledir.exists()) {
if (!(tabledir.isDirectory())) return 0;
} else {
return 0;
}
// open existing files
File file = new File(tabledir, "col.000.list");
return kelondroRecords.staticsize(file);
}
public static void delete(File path, String tablename) {
File tabledir = new File(path, tablename);
if ((tabledir.exists()) && (!(tabledir.isDirectory()))) {

19
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -69,6 +69,7 @@
package de.anomic.kelondro;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.HashSet;
import java.util.Iterator;
@ -192,6 +193,24 @@ public class kelondroRecords {
return this.USEDC + this.FREEC;
}
}
public static int staticsize(File file) {
if (!(file.exists())) return 0;
try {
kelondroRA ra = new kelondroFileRA(file.getCanonicalPath());
kelondroIOChunks entryFile = new kelondroRAIOChunks(ra, ra.name());
int USEDC = entryFile.readInt(POS_USEDC);
entryFile.close();
ra.close();
return USEDC;
} catch (FileNotFoundException e) {
return 0;
} catch (IOException e) {
return 0;
}
}
public kelondroRecords(File file, long buffersize /* bytes */, long preloadTime,
short ohbytec, short ohhandlec,

Write Preview
Loading…
Cancel
Save