boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added memory measurement for index recreation to avoid OOM during index RAM space extension

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4267 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent 5a80359b0e
commit 48138952ff
9 changed files with 67 additions and 44 deletions
Show Stats Download Patch File Download Diff File

2
source/dbtest.java
Unescape View File

@ -207,7 +207,7 @@ public class dbtest {
}
if (dbe.equals("kelondroFlexTable")) {
File tablepath = new File(tablename).getParentFile();
table = new kelondroFlexTable(tablepath, new File(tablename).getName(), preload, testRow, true);
table = new kelondroFlexTable(tablepath, new File(tablename).getName(), preload, testRow, 0, true);
}
if (dbe.equals("kelondroFlexSplitTable")) {
File tablepath = new File(tablename).getParentFile();

74
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -123,24 +123,46 @@ public class kelondroCollectionIndex {
this.commonsPath = new File(path, filenameStub + "." + fillZ(Integer.toHexString(rowdef.objectsize).toUpperCase(), 4) + ".commons");
this.commonsPath.mkdirs();
boolean ramIndexGeneration = false;
boolean fileIndexGeneration = !(new File(path, filenameStub + ".index").exists());
if (ramIndexGeneration) index = new kelondroRowSet(indexRow(keyLength, indexOrder), 0);
if (fileIndexGeneration) index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRow(keyLength, indexOrder), true);
if (new File(path, filenameStub + ".index").exists()) {
serverLog.logFine("STARTUP", "OPENING COLLECTION INDEX");
// open array files
// open index and array files
this.arrays = new HashMap(); // all entries will be dynamically created with getArray()
if ((fileIndexGeneration) || (ramIndexGeneration)) {
serverLog.logFine("STARTUP", "STARTED INITIALIZATION OF NEW COLLECTION INDEX. THIS WILL TAKE SOME TIME");
index = openIndexFile(path, filenameStub, indexOrder, preloadTime, loadfactor, rowdef, 0);
openAllArrayFiles(false, indexOrder);
} else {
if (index == null) index = openIndexFile(path, filenameStub, indexOrder, preloadTime, loadfactor, rowdef);
serverLog.logFine("STARTUP", "OPENING COLLECTION INDEX");
// calculate initialSpace
String[] list = this.path.list();
kelondroFixedWidthArray array;
int initialSpace = 0;
for (int i = 0; i < list.length; i++) if (list[i].endsWith(".kca")) {
// open array
int pos = list[i].indexOf('.');
if (pos < 0) continue;
int partitionNumber = Integer.parseInt(list[i].substring(pos + 9, pos + 11), 16);
int serialNumber = Integer.parseInt(list[i].substring(pos + 12, pos + 14), 16);
try {
array = openArrayFile(partitionNumber, serialNumber, indexOrder, true);
initialSpace += array.size();
array.close();
} catch (IOException e) {
e.printStackTrace();
continue;
}
}
serverLog.logFine("STARTUP", "STARTED INITIALIZATION OF NEW COLLECTION INDEX WITH " + initialSpace + " ENTRIES. THIS WILL TAKE SOME TIME");
// initialize (new generation) index table from file
index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRow(keyLength, indexOrder), initialSpace, true);
// open array files
this.arrays = new HashMap(); // all entries will be dynamically created with getArray()
openAllArrayFiles(true, indexOrder);
}
openAllArrayFiles(((fileIndexGeneration) || (ramIndexGeneration)), indexOrder);
if (index == null) index = openIndexFile(path, filenameStub, indexOrder, preloadTime, loadfactor, rowdef);
}
private void openAllArrayFiles(boolean indexGeneration, kelondroOrder indexOrder) throws IOException {
String[] list = this.path.list();
kelondroFixedWidthArray array;
@ -155,7 +177,7 @@ public class kelondroCollectionIndex {
int partitionNumber = Integer.parseInt(list[i].substring(pos + 9, pos + 11), 16);
int serialNumber = Integer.parseInt(list[i].substring(pos + 12, pos + 14), 16);
try {
array = openArrayFile(partitionNumber, serialNumber, true);
array = openArrayFile(partitionNumber, serialNumber, indexOrder, true);
} catch (IOException e) {
e.printStackTrace();
continue;
@ -201,9 +223,9 @@ public class kelondroCollectionIndex {
}
private kelondroIndex openIndexFile(File path, String filenameStub, kelondroOrder indexOrder,
long preloadTime, int loadfactor, kelondroRow rowdef) throws IOException {
long preloadTime, int loadfactor, kelondroRow rowdef, int initialSpace) throws IOException {
// open/create index table
kelondroIndex theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRow(keylength, indexOrder), true), true, false);
kelondroIndex theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRow(keylength, indexOrder), initialSpace, true), true, false);
//kelondroIndex theindex = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRow(keylength, indexOrder), true);
// save/check property file for this array
@ -224,13 +246,13 @@ public class kelondroCollectionIndex {
return theindex;
}
private kelondroFixedWidthArray openArrayFile(int partitionNumber, int serialNumber, boolean create) throws IOException {
private kelondroFixedWidthArray openArrayFile(int partitionNumber, int serialNumber, kelondroOrder indexOrder, boolean create) throws IOException {
File f = arrayFile(path, filenameStub, loadfactor, payloadrow.objectsize, partitionNumber, serialNumber);
int load = arrayCapacity(partitionNumber);
kelondroRow rowdef = new kelondroRow(
"byte[] key-" + keylength + "," +
"byte[] collection-" + (kelondroRowCollection.exportOverheadSize + load * this.payloadrow.objectsize),
index.row().objectOrder,
indexOrder,
0
);
if ((!(f.exists())) && (!create)) return null;
@ -239,12 +261,12 @@ public class kelondroCollectionIndex {
return a;
}
private kelondroFixedWidthArray getArray(int partitionNumber, int serialNumber, int chunksize) {
private kelondroFixedWidthArray getArray(int partitionNumber, int serialNumber, kelondroOrder indexOrder, int chunksize) {
String accessKey = partitionNumber + "-" + chunksize;
kelondroFixedWidthArray array = (kelondroFixedWidthArray) arrays.get(accessKey);
if (array != null) return array;
try {
array = openArrayFile(partitionNumber, serialNumber, true);
array = openArrayFile(partitionNumber, serialNumber, indexOrder, true);
} catch (IOException e) {
return null;
}
@ -289,7 +311,7 @@ public class kelondroCollectionIndex {
int oldRownumber) throws IOException {
// we need a new slot, that means we must first delete the old entry
// find array file
kelondroFixedWidthArray array = getArray(oldPartitionNumber, serialNumber, chunkSize);
kelondroFixedWidthArray array = getArray(oldPartitionNumber, serialNumber, index.row().objectOrder, chunkSize);
// delete old entry
array.remove(oldRownumber);
@ -300,7 +322,7 @@ public class kelondroCollectionIndex {
// the collection is new
int partitionNumber = arrayIndex(collection.size());
kelondroRow.Entry indexrow = index.row().newEntry();
kelondroFixedWidthArray array = getArray(partitionNumber, serialNumber, this.payloadrow.objectsize);
kelondroFixedWidthArray array = getArray(partitionNumber, serialNumber, index.row().objectOrder, this.payloadrow.objectsize);
// define row
kelondroRow.Entry arrayEntry = array.row().newEntry();
@ -329,7 +351,7 @@ public class kelondroCollectionIndex {
int partitionNumber, int serialNumber, int chunkSize) throws IOException {
// write a new entry in the other array
kelondroFixedWidthArray array = getArray(partitionNumber, serialNumber, chunkSize);
kelondroFixedWidthArray array = getArray(partitionNumber, serialNumber, index.row().objectOrder, chunkSize);
// define new row
kelondroRow.Entry arrayEntry = array.row().newEntry();
@ -365,7 +387,7 @@ public class kelondroCollectionIndex {
entry = (Map.Entry) i.next();
actionList = (ArrayList) entry.getValue();
partitionNumber = ((Integer) entry.getKey()).intValue();
array = getArray(partitionNumber, serialNumber, chunkSize);
array = getArray(partitionNumber, serialNumber, index.row().objectOrder, chunkSize);
j = actionList.iterator();
while (j.hasNext()) {
@ -401,7 +423,7 @@ public class kelondroCollectionIndex {
// we don't need a new slot, just write collection into the old one
// find array file
kelondroFixedWidthArray array = getArray(partitionNumber, serialNumber, chunkSize);
kelondroFixedWidthArray array = getArray(partitionNumber, serialNumber, index.row().objectOrder, chunkSize);
// define new row
kelondroRow.Entry arrayEntry = array.row().newEntry();
@ -437,7 +459,7 @@ public class kelondroCollectionIndex {
entry = (Map.Entry) i.next();
actionMap = (TreeMap) entry.getValue();
partitionNumber = ((Integer) entry.getKey()).intValue();
array = getArray(partitionNumber, serialNumber, chunkSize);
array = getArray(partitionNumber, serialNumber, index.row().objectOrder, chunkSize);
j = actionMap.entrySet().iterator();
while (j.hasNext()) {
@ -921,7 +943,7 @@ public class kelondroCollectionIndex {
private synchronized kelondroRowSet getwithparams(kelondroRow.Entry indexrow, int chunksize, int chunkcount, int clusteridx, int rownumber, int serialnumber, boolean remove) throws IOException {
// open array entry
kelondroFixedWidthArray array = getArray(clusteridx, serialnumber, chunksize);
kelondroFixedWidthArray array = getArray(clusteridx, serialnumber, index.row().objectOrder, chunksize);
kelondroRow.Entry arrayrow = array.get(rownumber);
if (arrayrow == null) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString(), "array does not contain expected row");
@ -1053,7 +1075,7 @@ public class kelondroCollectionIndex {
// printout of index
collectionIndex.close();
kelondroFlexTable index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, kelondroCollectionIndex.indexRow(9, kelondroNaturalOrder.naturalOrder), true);
kelondroFlexTable index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, kelondroCollectionIndex.indexRow(9, kelondroNaturalOrder.naturalOrder), 0, true);
index.print();
index.close();
} catch (IOException e) {

2
source/de/anomic/kelondro/kelondroDyn.java
Unescape View File

@ -94,7 +94,7 @@ public class kelondroDyn {
}
} else {
fbi = new kelondroFlexTable(file.getParentFile(), file.getName(), 10000, rowdef, resetOnFail);
fbi = new kelondroFlexTable(file.getParentFile(), file.getName(), 10000, rowdef, 0, resetOnFail);
}
this.index = (useObjectCache) ? (kelondroIndex) new kelondroCache(fbi, true, writebuffer) : fbi;
this.keylen = key;

6
source/de/anomic/kelondro/kelondroFlexSplitTable.java
Unescape View File

@ -100,7 +100,7 @@ public class kelondroFlexSplitTable implements kelondroIndex {
// open next biggest table
t.remove(maxf);
date = maxf.substring(tablename.length() + 1);
table = new kelondroCache(new kelondroFlexTable(path, maxf, preloadTime, rowdef, resetOnFail), true, false);
table = new kelondroCache(new kelondroFlexTable(path, maxf, preloadTime, rowdef, 0, resetOnFail), true, false);
tables.put(date, table);
}
}
@ -213,7 +213,7 @@ public class kelondroFlexSplitTable implements kelondroIndex {
kelondroIndex table = (kelondroIndex) tables.get(suffix);
if (table == null) {
// make new table
table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef, true);
table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef, 0, true);
tables.put(suffix, table);
}
table.put(row);
@ -244,7 +244,7 @@ public class kelondroFlexSplitTable implements kelondroIndex {
kelondroIndex table = (kelondroIndex) tables.get(suffix);
if (table == null) {
// make new table
table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef, true);
table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef, 0, true);
tables.put(suffix, table);
}
table.addUnique(row);

13
source/de/anomic/kelondro/kelondroFlexTable.java
Unescape View File

@ -46,8 +46,9 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
protected kelondroBytesIntMap index;
private boolean RAMIndex;
public kelondroFlexTable(File path, String tablename, long preloadTime, kelondroRow rowdef, boolean resetOnFail) {
public kelondroFlexTable(File path, String tablename, long preloadTime, kelondroRow rowdef, int minimumSpace, boolean resetOnFail) {
// the buffersize applies to a possible load of the ram-index
// the minimumSpace is a initial allocation space for the index; names the number of index slots
// if the ram is not sufficient, a tree file is generated
// if, and only if a tree file exists, the preload time is applied
super(path, tablename, rowdef, resetOnFail);
@ -59,7 +60,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
}
try {
long neededRAM = (long) ((super.row().column(0).cellwidth + 4) * super.size() * kelondroRowCollection.growfactor);
long neededRAM = (long) ((super.row().column(0).cellwidth + 4) * Math.max(super.size(), minimumSpace) * kelondroRowCollection.growfactor);
File newpath = new File(path, tablename);
File indexfile = new File(newpath, "col.000.index");
@ -82,7 +83,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
// fill the index
System.out.print("*** Loading RAM index for " + size() + " entries from "+ newpath);
index = initializeRamIndex();
index = initializeRamIndex(minimumSpace);
System.out.println(" -done-");
System.out.println(index.size()
@ -151,8 +152,8 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
return index.geti(key) >= 0;
}
private kelondroBytesIntMap initializeRamIndex() {
int space = super.col[0].size() + 1;
private kelondroBytesIntMap initializeRamIndex(int initialSpace) {
int space = Math.max(super.col[0].size(), initialSpace) + 1;
if (space < 0) throw new kelondroException("wrong space: " + space);
kelondroBytesIntMap ri = new kelondroBytesIntMap(super.row().column(0).cellwidth, super.rowdef.objectOrder, space);
Iterator content = super.col[0].contentNodes(-1);
@ -449,7 +450,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
String name = args[1];
kelondroRow row = new kelondroRow("Cardinal key-4 {b256}, byte[] x-64", kelondroNaturalOrder.naturalOrder, 0);
try {
kelondroFlexTable t = new kelondroFlexTable(f, name, 0, row, true);
kelondroFlexTable t = new kelondroFlexTable(f, name, 0, row, 0, true);
kelondroRow.Entry entry = row.newEntry();
entry.setCol(0, System.currentTimeMillis());
entry.setCol(1, "dummy".getBytes());

2
source/de/anomic/plasma/plasmaCrawlBalancer.java
Unescape View File

@ -121,7 +121,7 @@ public class plasmaCrawlBalancer {
private void openFileIndex() {
cacheStacksPath.mkdirs();
urlFileIndex = new kelondroCache(new kelondroFlexTable(cacheStacksPath, stackname + indexSuffix, -1, plasmaCrawlEntry.rowdef, true), true, false);
urlFileIndex = new kelondroCache(new kelondroFlexTable(cacheStacksPath, stackname + indexSuffix, -1, plasmaCrawlEntry.rowdef, 0, true), true, false);
}
private void resetFileIndex() {

4
source/de/anomic/plasma/plasmaCrawlStacker.java
Unescape View File

@ -299,13 +299,13 @@ public final class plasmaCrawlStacker extends Thread {
String newCacheName = "urlNoticeStacker8.db";
cacheStacksPath.mkdirs();
try {
this.urlEntryCache = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime, plasmaCrawlEntry.rowdef, true), true, false);
this.urlEntryCache = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime, plasmaCrawlEntry.rowdef, 0, true), true, false);
} catch (Exception e) {
e.printStackTrace();
// kill DB and try again
kelondroFlexTable.delete(cacheStacksPath, newCacheName);
try {
this.urlEntryCache = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime, plasmaCrawlEntry.rowdef, true), true, false);
this.urlEntryCache = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime, plasmaCrawlEntry.rowdef, 0, true), true, false);
} catch (Exception ee) {
ee.printStackTrace();
System.exit(-1);

2
source/de/anomic/plasma/plasmaCrawlZURL.java
Unescape View File

@ -61,7 +61,7 @@ public class plasmaCrawlZURL {
// creates a new ZURL in a file
cachePath.mkdirs();
if (startWithEmptyFile) kelondroFlexTable.delete(cachePath, tablename);
urlIndex = new kelondroFlexTable(cachePath, tablename, -1, rowdef, true);
urlIndex = new kelondroFlexTable(cachePath, tablename, -1, rowdef, 0, true);
}
public plasmaCrawlZURL() {

2
source/de/anomic/plasma/plasmaRankingCRProcess.java
Unescape View File

@ -261,7 +261,7 @@ public class plasmaRankingCRProcess {
kelondroCollectionIndex newseq = null;
if (newdb) {
File path = to_file.getParentFile(); // path to storage place
newacc = new kelondroFlexTable(path, CRG_accname, -1, CRG_accrow, false);
newacc = new kelondroFlexTable(path, CRG_accname, -1, CRG_accrow, 0, false);
newseq = new kelondroCollectionIndex(path, CRG_seqname, 12, kelondroBase64Order.enhancedCoder, -1, 2, 9, CRG_colrow);
} else {
if (!(to_file.exists())) {

Write Preview
Loading…
Cancel
Save