boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added option to configure the start-up delay time for kelondro database files.

Browse Source 
the start-up delay is used to pre-load the database node cache

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2276 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent ce9dd3e76d
commit 92f4cb4d73
47 changed files with 371 additions and 318 deletions
Show Stats Download Patch File Download Diff File

2
htroot/IndexImport_p.java
Unescape View File

@ -97,7 +97,7 @@ public final class IndexImport_p {
if (startImport) { if (startImport) {
dbImporter importerThread = switchboard.dbImportManager.getNewImporter(importType); dbImporter importerThread = switchboard.dbImportManager.getNewImporter(importType);
if (importerThread != null) { if (importerThread != null) {
importerThread.init(new File(importPath),cacheSize); importerThread.init(new File(importPath), cacheSize, 3000);
importerThread.startIt(); importerThread.startIt();
} }
prop.put("LOCATION",""); prop.put("LOCATION","");

12
source/dbtest.java
Unescape View File

@ -30,6 +30,7 @@ public class dbtest {
public final static int valuelength = 223; // sum of all data length as defined in plasmaURL public final static int valuelength = 223; // sum of all data length as defined in plasmaURL
//public final static long buffer = 0; //public final static long buffer = 0;
public final static long buffer = 8192 * 1024; // 8 MB buffer public final static long buffer = 8192 * 1024; // 8 MB buffer
public final static long preload = 1000; // 1 second
public static byte[] dummyvalue2 = new byte[valuelength]; public static byte[] dummyvalue2 = new byte[valuelength];
static { static {
// fill the dummy value // fill the dummy value
@ -171,23 +172,22 @@ public class dbtest {
if (dbe.equals("kelondroTree")) { if (dbe.equals("kelondroTree")) {
File tablefile = new File(tablename + ".kelondro.db"); File tablefile = new File(tablename + ".kelondro.db");
if (tablefile.exists()) { if (tablefile.exists()) {
table = new kelondroTree(tablefile, buffer, kelondroTree.defaultObjectCachePercent); table = new kelondroTree(tablefile, buffer, preload, kelondroTree.defaultObjectCachePercent);
} else { } else {
table = new kelondroTree(tablefile, buffer, kelondroTree.defaultObjectCachePercent, testRow, true); table = new kelondroTree(tablefile, buffer, preload, kelondroTree.defaultObjectCachePercent, testRow, true);
} }
} }
if (dbe.equals("kelondroSplittedTree")) { if (dbe.equals("kelondroSplittedTree")) {
File tablepath = new File(tablename).getParentFile(); File tablepath = new File(tablename).getParentFile();
tablename = new File(tablename).getName(); tablename = new File(tablename).getName();
table = kelondroSplittedTree.open(tablepath, tablename, kelondroBase64Order.enhancedCoder, table = kelondroSplittedTree.open(tablepath, tablename, kelondroBase64Order.enhancedCoder,
buffer, buffer, preload,
8, 8, testRow, 1, 80,
testRow, 1, 80,
true); true);
} }
if (dbe.equals("kelondroFlexTable")) { if (dbe.equals("kelondroFlexTable")) {
File tablepath = new File(tablename).getParentFile(); File tablepath = new File(tablename).getParentFile();
table = new kelondroFlexTable(tablepath, new File(tablename).getName(), buffer, testRow, true); table = new kelondroFlexTable(tablepath, new File(tablename).getName(), buffer, preload, testRow, true);
} }
if (dbe.equals("mysql")) { if (dbe.equals("mysql")) {
table = new dbTable("mysql", testRow); table = new dbTable("mysql", testRow);

8
source/de/anomic/data/blogBoard.java
Unescape View File

@ -80,15 +80,15 @@ public class blogBoard {
private kelondroMap datbase = null; private kelondroMap datbase = null;
public blogBoard(File actpath, int bufferkb) { public blogBoard(File actpath, int bufferkb, long preloadTime) {
new File(actpath.getParent()).mkdir(); new File(actpath.getParent()).mkdir();
if (datbase == null) { if (datbase == null) {
if (actpath.exists()) try { if (actpath.exists()) try {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x40, '_')); datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x40, preloadTime, '_'));
} catch (IOException e) { } catch (IOException e) {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, keyLength, recordSize, '_', true)); datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_', true));
} else { } else {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, keyLength, recordSize, '_', true)); datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_', true));
} }
} }
} }

20
source/de/anomic/data/bookmarksDB.java
Unescape View File

@ -119,7 +119,7 @@ public class bookmarksDB {
return new Date(); return new Date();
} }
public bookmarksDB(File bookmarksFile, File tagsFile, File datesFile, int bufferkb){ public bookmarksDB(File bookmarksFile, File tagsFile, File datesFile, int bufferkb, long preloadTime) {
//bookmarks //bookmarks
//check if database exists //check if database exists
tagCache=new HashMap(); tagCache=new HashMap();
@ -127,37 +127,37 @@ public class bookmarksDB {
if(bookmarksFile.exists()){ if(bookmarksFile.exists()){
try { try {
//open it //open it
this.bookmarksTable=new kelondroMap(new kelondroDyn(bookmarksFile, 1024*bufferkb, '_')); this.bookmarksTable=new kelondroMap(new kelondroDyn(bookmarksFile, 1024*bufferkb, preloadTime, '_'));
} catch (IOException e) { } catch (IOException e) {
//database reset :-(( //database reset :-((
bookmarksFile.delete(); bookmarksFile.delete();
bookmarksFile.getParentFile().mkdirs(); bookmarksFile.getParentFile().mkdirs();
//urlHash is 12 bytes long //urlHash is 12 bytes long
this.bookmarksTable = new kelondroMap(new kelondroDyn(bookmarksFile, bufferkb * 1024, 12, 256, '_', true)); this.bookmarksTable = new kelondroMap(new kelondroDyn(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true));
} }
}else{ }else{
//new database //new database
bookmarksFile.getParentFile().mkdirs(); bookmarksFile.getParentFile().mkdirs();
this.bookmarksTable = new kelondroMap(new kelondroDyn(bookmarksFile, bufferkb * 1024, 12, 256, '_', true)); this.bookmarksTable = new kelondroMap(new kelondroDyn(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true));
} }
//tags //tags
//check if database exists //check if database exists
if(tagsFile.exists()){ if(tagsFile.exists()){
try { try {
//open it //open it
this.tagsTable=new kelondroMap(new kelondroDyn(tagsFile, 1024*bufferkb, '_')); this.tagsTable=new kelondroMap(new kelondroDyn(tagsFile, 1024*bufferkb, preloadTime, '_'));
} catch (IOException e) { } catch (IOException e) {
//reset database //reset database
tagsFile.delete(); tagsFile.delete();
tagsFile.getParentFile().mkdirs(); tagsFile.getParentFile().mkdirs();
// max. 128 byte long tags // max. 128 byte long tags
this.tagsTable = new kelondroMap(new kelondroDyn(tagsFile, bufferkb * 1024, 12, 256, '_', true)); this.tagsTable = new kelondroMap(new kelondroDyn(tagsFile, bufferkb * 1024, preloadTime, 12, 256, '_', true));
rebuildTags(); rebuildTags();
} }
}else{ }else{
//new database //new database
tagsFile.getParentFile().mkdirs(); tagsFile.getParentFile().mkdirs();
this.tagsTable = new kelondroMap(new kelondroDyn(tagsFile, bufferkb * 1024, 12, 256, '_', true)); this.tagsTable = new kelondroMap(new kelondroDyn(tagsFile, bufferkb * 1024, preloadTime, 12, 256, '_', true));
rebuildTags(); rebuildTags();
} }
// dates // dates
@ -165,19 +165,19 @@ public class bookmarksDB {
if(datesFile.exists()){ if(datesFile.exists()){
try { try {
//open it //open it
this.datesTable=new kelondroMap(new kelondroDyn(datesFile, 1024*bufferkb, '_')); this.datesTable=new kelondroMap(new kelondroDyn(datesFile, 1024*bufferkb, preloadTime, '_'));
} catch (IOException e) { } catch (IOException e) {
//reset database //reset database
datesFile.delete(); datesFile.delete();
datesFile.getParentFile().mkdirs(); datesFile.getParentFile().mkdirs();
//YYYY-MM-DDTHH:mm:ssZ = 20 byte. currently used: YYYY-MM-DD = 10 bytes //YYYY-MM-DDTHH:mm:ssZ = 20 byte. currently used: YYYY-MM-DD = 10 bytes
this.datesTable = new kelondroMap(new kelondroDyn(datesFile, bufferkb * 1024, 20, 256, '_', true)); this.datesTable = new kelondroMap(new kelondroDyn(datesFile, bufferkb * 1024, preloadTime, 20, 256, '_', true));
rebuildDates(); rebuildDates();
} }
}else{ }else{
//new database //new database
datesFile.getParentFile().mkdirs(); datesFile.getParentFile().mkdirs();
this.datesTable = new kelondroMap(new kelondroDyn(datesFile, bufferkb * 1024, 20, 256, '_', true)); this.datesTable = new kelondroMap(new kelondroDyn(datesFile, bufferkb * 1024, preloadTime, 20, 256, '_', true));
rebuildDates(); rebuildDates();
} }
} }

8
source/de/anomic/data/messageBoard.java
Unescape View File

@ -67,16 +67,16 @@ public class messageBoard {
private kelondroMap database = null; private kelondroMap database = null;
private int sn = 0; private int sn = 0;
public messageBoard(File path, int bufferkb) { public messageBoard(File path, int bufferkb, long preloadTime) {
new File(path.getParent()).mkdir(); new File(path.getParent()).mkdir();
if (database == null) { if (database == null) {
if (path.exists()) try { if (path.exists()) try {
database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, '_')); database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, preloadTime, '_'));
} catch (IOException e) { } catch (IOException e) {
path.delete(); path.delete();
database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, categoryLength + dateFormat.length() + 2, recordSize, '_', true)); database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, preloadTime, categoryLength + dateFormat.length() + 2, recordSize, '_', true));
} else { } else {
database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, categoryLength + dateFormat.length() + 2, recordSize, '_', true)); database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, preloadTime, categoryLength + dateFormat.length() + 2, recordSize, '_', true));
} }
} }
sn = 0; sn = 0;

14
source/de/anomic/data/userDB.java
Unescape View File

@ -68,27 +68,29 @@ public final class userDB {
kelondroMap userTable; kelondroMap userTable;
private final File userTableFile; private final File userTableFile;
private final int bufferkb; private final int bufferkb;
private long preloadTime;
private HashMap ipUsers = new HashMap(); private HashMap ipUsers = new HashMap();
private HashMap cookieUsers = new HashMap(); private HashMap cookieUsers = new HashMap();
public userDB(File userTableFile, int bufferkb) { public userDB(File userTableFile, int bufferkb, long preloadTime) {
this.userTableFile = userTableFile; this.userTableFile = userTableFile;
this.bufferkb = bufferkb; this.bufferkb = bufferkb;
this.preloadTime = preloadTime;
if (userTableFile.exists()) { if (userTableFile.exists()) {
try { try {
this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, '_')); this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, preloadTime, '_'));
} catch (kelondroException e) { } catch (kelondroException e) {
userTableFile.delete(); userTableFile.delete();
userTableFile.getParentFile().mkdirs(); userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, 128, 256, '_', true)); this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, preloadTime, 128, 256, '_', true));
} catch (IOException e) { } catch (IOException e) {
userTableFile.delete(); userTableFile.delete();
userTableFile.getParentFile().mkdirs(); userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, 128, 256, '_', true)); this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, preloadTime, 128, 256, '_', true));
} }
} else { } else {
userTableFile.getParentFile().mkdirs(); userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, 128, 256, '_', true)); this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, preloadTime, 128, 256, '_', true));
} }
} }
@ -107,7 +109,7 @@ public final class userDB {
} catch (IOException e) {} } catch (IOException e) {}
if (!(userTableFile.delete())) throw new RuntimeException("cannot delete user database"); if (!(userTableFile.delete())) throw new RuntimeException("cannot delete user database");
userTableFile.getParentFile().mkdirs(); userTableFile.getParentFile().mkdirs();
userTable = new kelondroMap(new kelondroDyn(userTableFile, this.bufferkb, 256, 512, '_', true)); userTable = new kelondroMap(new kelondroDyn(userTableFile, this.bufferkb, preloadTime, 256, 512, '_', true));
} }
public void close() { public void close() {

14
source/de/anomic/data/wikiBoard.java
Unescape View File

@ -69,25 +69,25 @@ public class wikiBoard {
private kelondroMap bkpbase = null; private kelondroMap bkpbase = null;
private static HashMap authors = new HashMap(); private static HashMap authors = new HashMap();
public wikiBoard(File actpath, File bkppath, int bufferkb) { public wikiBoard(File actpath, File bkppath, int bufferkb, long preloadTime) {
new File(actpath.getParent()).mkdir(); new File(actpath.getParent()).mkdir();
if (datbase == null) { if (datbase == null) {
if (actpath.exists()) try { if (actpath.exists()) try {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x40, '_')); datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x40, preloadTime, '_'));
} catch (IOException e) { } catch (IOException e) {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, keyLength, recordSize, '_', true)); datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_', true));
} else { } else {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, keyLength, recordSize, '_', true)); datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_', true));
} }
} }
new File(bkppath.getParent()).mkdir(); new File(bkppath.getParent()).mkdir();
if (bkpbase == null) { if (bkpbase == null) {
if (bkppath.exists()) try { if (bkppath.exists()) try {
bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400, '_')); bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400, preloadTime, '_'));
} catch (IOException e) { } catch (IOException e) {
bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400, keyLength + dateFormat.length(), recordSize, '_', true)); bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400, preloadTime, keyLength + dateFormat.length(), recordSize, '_', true));
} else { } else {
bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400, keyLength + dateFormat.length(), recordSize, '_', true)); bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400, preloadTime, keyLength + dateFormat.length(), recordSize, '_', true));
} }
} }
} }

4
source/de/anomic/index/indexCollectionRI.java
Unescape View File

@ -12,12 +12,12 @@ public class indexCollectionRI extends indexAbstractRI implements indexRI {
kelondroCollectionIndex collectionIndex; kelondroCollectionIndex collectionIndex;
public indexCollectionRI(File path, String filenameStub, long buffersize) throws IOException { public indexCollectionRI(File path, String filenameStub, long buffersize, long preloadTime) throws IOException {
kelondroRow rowdef = new kelondroRow(new int[]{}); kelondroRow rowdef = new kelondroRow(new int[]{});
collectionIndex = new kelondroCollectionIndex( collectionIndex = new kelondroCollectionIndex(
path, filenameStub, 9 /*keyLength*/, path, filenameStub, 9 /*keyLength*/,
kelondroNaturalOrder.naturalOrder, buffersize, kelondroNaturalOrder.naturalOrder, buffersize, preloadTime,
4 /*loadfactor*/, rowdef, 8 /*partitions*/); 4 /*loadfactor*/, rowdef, 8 /*partitions*/);
} }

5
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -57,7 +57,8 @@ public class kelondroCollectionIndex {
return (int) (time / day) - 10957; return (int) (time / day) - 10957;
} }
public kelondroCollectionIndex(File path, String filenameStub, int keyLength, kelondroOrder indexOrder, long buffersize, public kelondroCollectionIndex(File path, String filenameStub, int keyLength, kelondroOrder indexOrder,
long buffersize, long preloadTime,
int loadfactor, kelondroRow rowdef, int partitions) throws IOException { int loadfactor, kelondroRow rowdef, int partitions) throws IOException {
this.path = path; this.path = path;
this.filenameStub = filenameStub; this.filenameStub = filenameStub;
@ -73,7 +74,7 @@ public class kelondroCollectionIndex {
columns[2] = 4; // chunkcount (number of chunks in this collection) columns[2] = 4; // chunkcount (number of chunks in this collection)
columns[3] = 4; // index (position in index file) columns[3] = 4; // index (position in index file)
columns[4] = 2; // update time in days since 1.1.2000 columns[4] = 2; // update time in days since 1.1.2000
index = new kelondroSplittedTree(path, filenameStub, indexOrder, buffersize, 8, new kelondroRow(columns), 1, 80, true); index = new kelondroSplittedTree(path, filenameStub, indexOrder, buffersize, preloadTime, 8, new kelondroRow(columns), 1, 80, true);
// create array files // create array files
this.array = new kelondroFixedWidthArray[partitions]; this.array = new kelondroFixedWidthArray[partitions];

21
source/de/anomic/kelondro/kelondroDyn.java
Unescape View File

@ -68,15 +68,15 @@ public class kelondroDyn extends kelondroTree {
private char fillChar; private char fillChar;
private kelondroObjectBuffer buffer; private kelondroObjectBuffer buffer;
public kelondroDyn(File file, long buffersize /*bytes*/, int key, int nodesize, char fillChar, boolean exitOnFail) { public kelondroDyn(File file, long buffersize /*bytes*/, long preloadTime, int key, int nodesize, char fillChar, boolean exitOnFail) {
this(file, buffersize, key, nodesize, fillChar, new kelondroNaturalOrder(true), exitOnFail); this(file, buffersize, preloadTime, key, nodesize, fillChar, new kelondroNaturalOrder(true), exitOnFail);
} }
public kelondroDyn(File file, long buffersize /* bytes */, int key, public kelondroDyn(File file, long buffersize /* bytes */, long preloadTime, int key,
int nodesize, char fillChar, kelondroOrder objectOrder, int nodesize, char fillChar, kelondroOrder objectOrder,
boolean exitOnFail) { boolean exitOnFail) {
// creates a new dynamic tree // creates a new dynamic tree
super(file, buffersize, kelondroTree.defaultObjectCachePercent, new kelondroRow(new int[] { key + counterlen, nodesize }), objectOrder, 1, 8, exitOnFail); super(file, buffersize, preloadTime, kelondroTree.defaultObjectCachePercent, new kelondroRow(new int[] { key + counterlen, nodesize }), objectOrder, 1, 8, exitOnFail);
this.keylen = row().width(0) - counterlen; this.keylen = row().width(0) - counterlen;
this.reclen = row().width(1); this.reclen = row().width(1);
this.fillChar = fillChar; this.fillChar = fillChar;
@ -85,9 +85,9 @@ public class kelondroDyn extends kelondroTree {
buffer = new kelondroObjectBuffer(file.toString()); buffer = new kelondroObjectBuffer(file.toString());
} }
public kelondroDyn(File file, long buffersize, char fillChar) throws IOException { public kelondroDyn(File file, long buffersize, long preloadTime, char fillChar) throws IOException {
// this opens a file with an existing dynamic tree // this opens a file with an existing dynamic tree
super(file, buffersize, kelondroTree.defaultObjectCachePercent); super(file, buffersize, preloadTime, kelondroTree.defaultObjectCachePercent);
this.keylen = row().width(0) - counterlen; this.keylen = row().width(0) - counterlen;
this.reclen = row().width(1); this.reclen = row().width(1);
this.fillChar = fillChar; this.fillChar = fillChar;
@ -434,8 +434,7 @@ public class kelondroDyn extends kelondroTree {
} else if (args.length == 1) { } else if (args.length == 1) {
// open a db and list keys // open a db and list keys
try { try {
kelondroDyn kd = new kelondroDyn(new File(args[0]), 0x100000, kelondroDyn kd = new kelondroDyn(new File(args[0]), 0x100000, 0, '_');
'_');
System.out.println(kd.size() + " elements in DB"); System.out.println(kd.size() + " elements in DB");
Iterator i = kd.dynKeys(true, false); Iterator i = kd.dynKeys(true, false);
while (i.hasNext()) while (i.hasNext())
@ -453,9 +452,9 @@ public class kelondroDyn extends kelondroTree {
kelondroDyn kd; kelondroDyn kd;
try { try {
if (db.exists()) if (db.exists())
kd = new kelondroDyn(db, 0x100000, '_'); kd = new kelondroDyn(db, 0x100000, 0, '_');
else else
kd = new kelondroDyn(db, 0x100000, 80, 200, '_', true); kd = new kelondroDyn(db, 0x100000, 0, 80, 200, '_', true);
if (writeFile) if (writeFile)
kd.readFile(key, f); kd.readFile(key, f);
else else
@ -479,7 +478,7 @@ public class kelondroDyn extends kelondroTree {
int steps = 0; int steps = 0;
while (true) { while (true) {
if (testFile.exists()) testFile.delete(); if (testFile.exists()) testFile.delete();
tt = new kelondroDyn(testFile, 0, 4 ,100, '_', true); tt = new kelondroDyn(testFile, 0, 0, 4 ,100, '_', true);
steps = ((int) System.currentTimeMillis() % 7) * (((int) System.currentTimeMillis() + 17) % 11); steps = ((int) System.currentTimeMillis() % 7) * (((int) System.currentTimeMillis() + 17) % 11);
t = s; t = s;
d = ""; d = "";

21
source/de/anomic/kelondro/kelondroDynTree.java
Unescape View File

@ -64,33 +64,38 @@ public class kelondroDynTree {
//private long maxageCache = 60000, cycletimeCache = 10000; //private long maxageCache = 60000, cycletimeCache = 10000;
private long maxageBuffer = 60000, cycletimeBuffer = 10000; private long maxageBuffer = 60000, cycletimeBuffer = 10000;
private long buffersize = 0; private long buffersize = 0;
private long preloadTime = 0;
// data structures for the cache and buffer // data structures for the cache and buffer
private Hashtable buffer, cache; private Hashtable buffer, cache;
private long cycleBuffer; private long cycleBuffer;
public kelondroDynTree(File file, long buffersize, int keylength, int nodesize, kelondroRow rowdef, char fillChar, boolean exitOnFail) { public kelondroDynTree(File file, long buffersize, long preloadTime, int keylength, int nodesize, kelondroRow rowdef, char fillChar, boolean exitOnFail) {
// creates a new DynTree // creates a new DynTree
this.file = file; this.file = file;
this.buffersize = buffersize;
this.preloadTime = preloadTime;
this.rowdef = rowdef; this.rowdef = rowdef;
this.buffer = new Hashtable(); this.buffer = new Hashtable();
this.cache = new Hashtable(); this.cache = new Hashtable();
//this.cycleCache = Long.MIN_VALUE; //this.cycleCache = Long.MIN_VALUE;
this.cycleBuffer = Long.MIN_VALUE; this.cycleBuffer = Long.MIN_VALUE;
if (file.exists()) file.delete(); if (file.exists()) file.delete();
this.table = new kelondroDyn(file, buffersize, keylength, nodesize, fillChar, exitOnFail); this.table = new kelondroDyn(file, buffersize, preloadTime, keylength, nodesize, fillChar, exitOnFail);
this.treeRAHandles = new Hashtable(); this.treeRAHandles = new Hashtable();
} }
public kelondroDynTree(File file, long buffersize, char fillChar) throws IOException { public kelondroDynTree(File file, long buffersize, long preloadTime, char fillChar) throws IOException {
// opens an existing DynTree // opens an existing DynTree
this.file = file; this.file = file;
this.buffersize = buffersize;
this.preloadTime = preloadTime;
this.buffer = new Hashtable(); this.buffer = new Hashtable();
this.cache = new Hashtable(); this.cache = new Hashtable();
//this.cycleCache = Long.MIN_VALUE; //this.cycleCache = Long.MIN_VALUE;
this.cycleBuffer = Long.MIN_VALUE; this.cycleBuffer = Long.MIN_VALUE;
if (!(file.exists())) throw new IOException("DynTree " + file.toString() + " does not exist"); if (!(file.exists())) throw new IOException("DynTree " + file.toString() + " does not exist");
this.table = new kelondroDyn(file, buffersize, fillChar); this.table = new kelondroDyn(file, buffersize, preloadTime, fillChar);
// read one element to measure the size of columns // read one element to measure the size of columns
if (table.size() == 0) throw new IOException("DynTree " + file.toString() + " is empty. Should not."); if (table.size() == 0) throw new IOException("DynTree " + file.toString() + " is empty. Should not.");
this.treeRAHandles = new Hashtable(); this.treeRAHandles = new Hashtable();
@ -127,7 +132,7 @@ public class kelondroDynTree {
kelondroRA ra = table.getRA(key); // works always, even with no-existing entry kelondroRA ra = table.getRA(key); // works always, even with no-existing entry
treeRAHandles.put(key, ra); treeRAHandles.put(key, ra);
try { try {
return new kelondroTree(ra, buffersize, kelondroTree.defaultObjectCachePercent, rowdef, false); return new kelondroTree(ra, buffersize, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, false);
} catch (RuntimeException e) { } catch (RuntimeException e) {
throw new IOException(e.getMessage()); throw new IOException(e.getMessage());
} }
@ -137,7 +142,7 @@ public class kelondroDynTree {
if (table.existsDyn(key)) { if (table.existsDyn(key)) {
kelondroRA ra = table.getRA(key); kelondroRA ra = table.getRA(key);
treeRAHandles.put(key, ra); treeRAHandles.put(key, ra);
return new kelondroTree(ra, buffersize, kelondroTree.defaultObjectCachePercent); return new kelondroTree(ra, buffersize, preloadTime, kelondroTree.defaultObjectCachePercent);
} else { } else {
return null; return null;
} }
@ -319,10 +324,10 @@ public class kelondroDynTree {
System.out.println("start"); System.out.println("start");
File file = new File("D:\\bin\\testDyn.db"); File file = new File("D:\\bin\\testDyn.db");
if (file.exists()) { if (file.exists()) {
kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, '_'); kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, 0, '_');
System.out.println("opened: table keylength=" + dt.table.row().width(0) + ", sectorsize=" + dt.table.row().width(1) + ", " + dt.table.size() + " entries."); System.out.println("opened: table keylength=" + dt.table.row().width(0) + ", sectorsize=" + dt.table.row().width(1) + ", " + dt.table.size() + " entries.");
} else { } else {
kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, 16, 512, new kelondroRow(new int[] {10,20,30}), '_', true); kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, 0, 16, 512, new kelondroRow(new int[] {10,20,30}), '_', true);
String name; String name;
kelondroTree t; kelondroTree t;
kelondroRow.Entry line; kelondroRow.Entry line;

4
source/de/anomic/kelondro/kelondroFixedWidthArray.java
Unescape View File

@ -57,7 +57,7 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
public kelondroFixedWidthArray(File file, kelondroRow rowdef, int intprops, boolean exitOnFail) { public kelondroFixedWidthArray(File file, kelondroRow rowdef, int intprops, boolean exitOnFail) {
// this creates a new array // this creates a new array
super(file, 0, thisOHBytes, thisOHHandles, rowdef, intprops, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail); super(file, 0, 0, thisOHBytes, thisOHHandles, rowdef, intprops, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
for (int i = 0; i < intprops; i++) try { for (int i = 0; i < intprops; i++) try {
setHandle(i, new Handle(0)); setHandle(i, new Handle(0));
} catch (IOException e) { } catch (IOException e) {
@ -69,7 +69,7 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
public kelondroFixedWidthArray(File file) throws IOException{ public kelondroFixedWidthArray(File file) throws IOException{
// this opens a file with an existing array // this opens a file with an existing array
super(file, 0); super(file, 0, 0);
} }
public synchronized kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException { public synchronized kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException {

10
source/de/anomic/kelondro/kelondroFlexTable.java
Unescape View File

@ -33,7 +33,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
private kelondroBytesIntMap index; private kelondroBytesIntMap index;
public kelondroFlexTable(File path, String tablename, long buffersize, kelondroRow rowdef, boolean exitOnFail) throws IOException { public kelondroFlexTable(File path, String tablename, long buffersize, long preloadTime, kelondroRow rowdef, boolean exitOnFail) throws IOException {
super(path, tablename, rowdef, exitOnFail); super(path, tablename, rowdef, exitOnFail);
File newpath = new File(path, tablename + ".table"); File newpath = new File(path, tablename + ".table");
File indexfile = new File(newpath, "col.000.index"); File indexfile = new File(newpath, "col.000.index");
@ -45,11 +45,11 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
if (indexfile.exists()) { if (indexfile.exists()) {
// use existing index file // use existing index file
System.out.println("*** Using File index " + indexfile); System.out.println("*** Using File index " + indexfile);
ki = new kelondroTree(indexfile, buffersize, 10); ki = new kelondroTree(indexfile, buffersize, preloadTime, 10);
} else if (size() > 100000) { } else if (size() > 100000) {
// generate new index file // generate new index file
System.out.print("*** Generating File index for " + size() + " entries from " + indexfile); System.out.print("*** Generating File index for " + size() + " entries from " + indexfile);
ki = initializeTreeIndex(indexfile, buffersize); ki = initializeTreeIndex(indexfile, buffersize, preloadTime);
System.out.println(" -done-"); System.out.println(" -done-");
System.out.println(ki.size() System.out.println(ki.size()
@ -98,8 +98,8 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
} }
private kelondroIndex initializeTreeIndex(File indexfile, long buffersize) throws IOException { private kelondroIndex initializeTreeIndex(File indexfile, long buffersize, long preloadTime) throws IOException {
kelondroTree index = new kelondroTree(indexfile, buffersize, 10, rowdef.width(0), 4, true); kelondroTree index = new kelondroTree(indexfile, buffersize, preloadTime, 10, rowdef.width(0), 4, true);
Iterator content = super.col[0].contentNodes(); Iterator content = super.col[0].contentNodes();
kelondroRecords.Node node; kelondroRecords.Node node;
kelondroRow.Entry indexentry; kelondroRow.Entry indexentry;

29
source/de/anomic/kelondro/kelondroMapTable.java
Unescape View File

@ -61,45 +61,52 @@ public class kelondroMapTable {
if (!(tablesPath.exists())) tablesPath.mkdirs(); if (!(tablesPath.exists())) tablesPath.mkdirs();
} }
public void declareMaps(String tablename, int keysize, int nodesize, char fillChar, boolean exitOnFail) { public void declareMaps(
String tablename, int keysize, int nodesize,
char fillChar, boolean exitOnFail) {
declareMaps(tablename, keysize, nodesize, null, null, fillChar, exitOnFail); declareMaps(tablename, keysize, nodesize, null, null, fillChar, exitOnFail);
} }
public void declareMaps(String tablename, int keysize, int nodesize, String[] sortfields, String[] accfields, char fillChar, boolean exitOnFail) { public void declareMaps(
declareMaps(tablename, keysize, nodesize, sortfields, accfields, fillChar, 0x800, exitOnFail); String tablename, int keysize, int nodesize,
String[] sortfields, String[] accfields, char fillChar, boolean exitOnFail) {
declareMaps(tablename, keysize, nodesize, sortfields, accfields, fillChar, 0x800, 0, exitOnFail);
} }
public void declareMaps(String tablename, int keysize, int nodesize, String[] sortfields, String[] accfields, char fillChar, long buffersize /*bytes*/, boolean exitOnFail) { public void declareMaps(
String tablename, int keysize, int nodesize,
String[] sortfields, String[] accfields, char fillChar,
long buffersize /*bytes*/, long preloadTime, boolean exitOnFail) {
if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared twice."); if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared twice.");
if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared already in other context."); if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared already in other context.");
File tablefile = new File(tablesPath, "table." + tablename + ".mdb"); File tablefile = new File(tablesPath, "table." + tablename + ".mdb");
kelondroDyn dyn; kelondroDyn dyn;
if (tablefile.exists()) try { if (tablefile.exists()) try {
dyn = new kelondroDyn(tablefile, buffersize, fillChar); dyn = new kelondroDyn(tablefile, buffersize, preloadTime, fillChar);
} catch (IOException e) { } catch (IOException e) {
tablefile.getParentFile().mkdirs(); tablefile.getParentFile().mkdirs();
dyn = new kelondroDyn(tablefile, buffersize, keysize, nodesize, fillChar, exitOnFail); dyn = new kelondroDyn(tablefile, buffersize, preloadTime, keysize, nodesize, fillChar, exitOnFail);
} else { } else {
tablefile.getParentFile().mkdirs(); tablefile.getParentFile().mkdirs();
dyn = new kelondroDyn(tablefile, buffersize, keysize, nodesize, fillChar, exitOnFail); dyn = new kelondroDyn(tablefile, buffersize, preloadTime, keysize, nodesize, fillChar, exitOnFail);
} }
kelondroMap map = new kelondroMap(dyn, sortfields, accfields); kelondroMap map = new kelondroMap(dyn, sortfields, accfields);
mTables.put(tablename, map); mTables.put(tablename, map);
} }
public void declareTree(String tablename, kelondroRow rowdef, long buffersize /*bytes*/, boolean exitOnFail) { public void declareTree(String tablename, kelondroRow rowdef, long buffersize /*bytes*/, long preloadTime, boolean exitOnFail) {
if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared already in other context."); if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared already in other context.");
if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared twice."); if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared twice.");
File tablefile = new File(tablesPath, "table." + tablename + ".tdb"); File tablefile = new File(tablesPath, "table." + tablename + ".tdb");
kelondroTree Tree; kelondroTree Tree;
if (tablefile.exists()) try { if (tablefile.exists()) try {
Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent); Tree = new kelondroTree(tablefile, buffersize, preloadTime, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) { } catch (IOException e) {
tablefile.getParentFile().mkdirs(); tablefile.getParentFile().mkdirs();
Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent, rowdef, exitOnFail); Tree = new kelondroTree(tablefile, buffersize, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, exitOnFail);
} else { } else {
tablefile.getParentFile().mkdirs(); tablefile.getParentFile().mkdirs();
Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent, rowdef, exitOnFail); Tree = new kelondroTree(tablefile, buffersize, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, exitOnFail);
} }
tTables.put(tablename, Tree); tTables.put(tablename, Tree);
} }

23
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -82,7 +82,6 @@ public class kelondroRecords {
private static final int NUL = Integer.MIN_VALUE; // the meta value for the kelondroRecords' NUL abstraction private static final int NUL = Integer.MIN_VALUE; // the meta value for the kelondroRecords' NUL abstraction
private static final long memBlock = 500000; // do not fill cache further if the amount of available memory is less that this private static final long memBlock = 500000; // do not fill cache further if the amount of available memory is less that this
public final static boolean useWriteBuffer = false; public final static boolean useWriteBuffer = false;
public final static long preloadCacheTime = 500; // time that can be wasted to initialize the node cache
// memory calculation // memory calculation
private static final int element_in_cache = 4; // for kelondroCollectionObjectMap: 4; for HashMap: 52 private static final int element_in_cache = 4; // for kelondroCollectionObjectMap: 4; for HashMap: 52
@ -186,7 +185,7 @@ public class kelondroRecords {
} }
} }
public kelondroRecords(File file, long buffersize /* bytes */, public kelondroRecords(File file, long buffersize /* bytes */, long preloadTime,
short ohbytec, short ohhandlec, short ohbytec, short ohhandlec,
kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth, kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth,
boolean exitOnFail) { boolean exitOnFail) {
@ -209,10 +208,10 @@ public class kelondroRecords {
if (exitOnFail) if (exitOnFail)
System.exit(-1); System.exit(-1);
} }
initCache(buffersize / 10 * 9); initCache(buffersize / 10 * 9, preloadTime);
} }
public kelondroRecords(kelondroRA ra, long buffersize /* bytes */, public kelondroRecords(kelondroRA ra, long buffersize /* bytes */, long preloadTime,
short ohbytec, short ohhandlec, short ohbytec, short ohhandlec,
kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth, kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth,
boolean exitOnFail) { boolean exitOnFail) {
@ -223,7 +222,7 @@ public class kelondroRecords {
logFailure("cannot create / " + e.getMessage()); logFailure("cannot create / " + e.getMessage());
if (exitOnFail) System.exit(-1); if (exitOnFail) System.exit(-1);
} }
initCache(buffersize / 10 * 9); initCache(buffersize / 10 * 9, preloadTime);
} }
private void initNewFile(kelondroRA ra, short ohbytec, short ohhandlec, private void initNewFile(kelondroRA ra, short ohbytec, short ohhandlec,
@ -344,7 +343,7 @@ public class kelondroRecords {
this.USAGE.write(); this.USAGE.write();
} }
public kelondroRecords(File file, long buffersize) throws IOException{ public kelondroRecords(File file, long buffersize, long preloadTime) throws IOException{
// opens an existing tree // opens an existing tree
assert (file.exists()): "file " + file.getAbsoluteFile().toString() + " does not exist"; assert (file.exists()): "file " + file.getAbsoluteFile().toString() + " does not exist";
this.filename = file.getCanonicalPath(); this.filename = file.getCanonicalPath();
@ -353,13 +352,13 @@ public class kelondroRecords {
//kelondroRA raf = new kelondroCachedRA(new kelondroFileRA(this.filename), 5000000, 1000); //kelondroRA raf = new kelondroCachedRA(new kelondroFileRA(this.filename), 5000000, 1000);
//kelondroRA raf = new kelondroNIOFileRA(this.filename, (file.length() < 4000000), 10000); //kelondroRA raf = new kelondroNIOFileRA(this.filename, (file.length() < 4000000), 10000);
initExistingFile(raf, buffersize / 10); initExistingFile(raf, buffersize / 10);
initCache(buffersize / 10 * 9); initCache(buffersize / 10 * 9, preloadTime);
} }
public kelondroRecords(kelondroRA ra, long buffersize) throws IOException{ public kelondroRecords(kelondroRA ra, long buffersize, long preloadTime) throws IOException{
this.filename = null; this.filename = null;
initExistingFile(ra, buffersize / 10); initExistingFile(ra, buffersize / 10);
initCache(buffersize / 10 * 9); initCache(buffersize / 10 * 9, preloadTime);
} }
private void initExistingFile(kelondroRA ra, long writeBufferSize) throws IOException { private void initExistingFile(kelondroRA ra, long writeBufferSize) throws IOException {
@ -410,7 +409,7 @@ public class kelondroRecords {
this.tailchunksize = this.recordsize - this.headchunksize; this.tailchunksize = this.recordsize - this.headchunksize;
} }
private void initCache(long buffersize) { private void initCache(long buffersize, long preloadTime) {
if (buffersize <= 0) { if (buffersize <= 0) {
this.cacheSize = 0; this.cacheSize = 0;
this.cacheHeaders = null; this.cacheHeaders = null;
@ -427,8 +426,8 @@ public class kelondroRecords {
this.cacheFlush = 0; this.cacheFlush = 0;
// pre-load node cache // pre-load node cache
if ((preloadCacheTime > 0) && (cacheSize > 0)) { if ((preloadTime > 0) && (cacheSize > 0)) {
long stop = System.currentTimeMillis() + preloadCacheTime; long stop = System.currentTimeMillis() + preloadTime;
Iterator i = contentNodes(); Iterator i = contentNodes();
Node n; Node n;
int count = 0; int count = 0;

29
source/de/anomic/kelondro/kelondroSplittedTree.java
Unescape View File

@ -73,10 +73,8 @@ public class kelondroSplittedTree implements kelondroIndex {
} }
public kelondroSplittedTree(File pathToFiles, String filenameStub, kelondroOrder objectOrder, public kelondroSplittedTree(File pathToFiles, String filenameStub, kelondroOrder objectOrder,
long buffersize, long buffersize, long preloadTime,
int forkfactor, int forkfactor, kelondroRow rowdef, int txtProps, int txtPropsWidth,
kelondroRow rowdef,
int txtProps, int txtPropsWidth,
boolean exitOnFail) { boolean exitOnFail) {
ktfs = new kelondroTree[forkfactor]; ktfs = new kelondroTree[forkfactor];
File f; File f;
@ -84,15 +82,15 @@ public class kelondroSplittedTree implements kelondroIndex {
f = dbFile(pathToFiles, filenameStub, forkfactor, rowdef.columns(), i); f = dbFile(pathToFiles, filenameStub, forkfactor, rowdef.columns(), i);
if (f.exists()) { if (f.exists()) {
try { try {
ktfs[i] = new kelondroTree(f, buffersize/forkfactor, kelondroTree.defaultObjectCachePercent); ktfs[i] = new kelondroTree(f, buffersize/forkfactor, preloadTime / forkfactor, kelondroTree.defaultObjectCachePercent);
this.order = ktfs[i].order(); this.order = ktfs[i].order();
} catch (IOException e) { } catch (IOException e) {
ktfs[i] = new kelondroTree(f, buffersize/forkfactor, kelondroTree.defaultObjectCachePercent, ktfs[i] = new kelondroTree(f, buffersize/forkfactor, preloadTime / forkfactor, kelondroTree.defaultObjectCachePercent,
rowdef, objectOrder, txtProps, txtPropsWidth, exitOnFail); rowdef, objectOrder, txtProps, txtPropsWidth, exitOnFail);
this.order = objectOrder; this.order = objectOrder;
} }
} else { } else {
ktfs[i] = new kelondroTree(f, buffersize/forkfactor, kelondroTree.defaultObjectCachePercent, ktfs[i] = new kelondroTree(f, buffersize/forkfactor, preloadTime / forkfactor, kelondroTree.defaultObjectCachePercent,
rowdef, objectOrder, txtProps, txtPropsWidth, exitOnFail); rowdef, objectOrder, txtProps, txtPropsWidth, exitOnFail);
this.order = objectOrder; this.order = objectOrder;
} }
@ -101,29 +99,28 @@ public class kelondroSplittedTree implements kelondroIndex {
} }
public kelondroSplittedTree(File pathToFiles, String filenameStub, kelondroOrder objectOrder, public kelondroSplittedTree(File pathToFiles, String filenameStub, kelondroOrder objectOrder,
long buffersize, int forkfactor, int columns) throws IOException { long buffersize, long preloadTime, int forkfactor, int columns) throws IOException {
ktfs = new kelondroTree[forkfactor]; ktfs = new kelondroTree[forkfactor];
for (int i = 0; i < forkfactor; i++) { for (int i = 0; i < forkfactor; i++) {
ktfs[i] = new kelondroTree(dbFile(pathToFiles, filenameStub, forkfactor, columns, i), buffersize/forkfactor, kelondroTree.defaultObjectCachePercent); ktfs[i] = new kelondroTree(dbFile(pathToFiles, filenameStub, forkfactor, columns, i),
buffersize/forkfactor, preloadTime / forkfactor, kelondroTree.defaultObjectCachePercent);
} }
ff = forkfactor; ff = forkfactor;
this.order = objectOrder; this.order = objectOrder;
} }
public static kelondroSplittedTree open(File pathToFiles, String filenameStub, kelondroOrder objectOrder, public static kelondroSplittedTree open(File pathToFiles, String filenameStub, kelondroOrder objectOrder,
long buffersize, long buffersize, long preloadTime,
int forkfactor, int forkfactor, kelondroRow rowdef, int txtProps, int txtPropsWidth,
kelondroRow rowdef, int txtProps, int txtPropsWidth,
boolean exitOnFail) throws IOException { boolean exitOnFail) throws IOException {
// generated a new splittet tree if it not exists or // generated a new splittet tree if it not exists or
// opens an existing one // opens an existing one
if (existsAll(pathToFiles, filenameStub, forkfactor, rowdef.columns())) { if (existsAll(pathToFiles, filenameStub, forkfactor, rowdef.columns())) {
return new kelondroSplittedTree(pathToFiles, filenameStub, objectOrder, buffersize, forkfactor, rowdef.columns()); return new kelondroSplittedTree(pathToFiles, filenameStub, objectOrder, buffersize, preloadTime, forkfactor, rowdef.columns());
} else { } else {
return new kelondroSplittedTree(pathToFiles, filenameStub, objectOrder, return new kelondroSplittedTree(pathToFiles, filenameStub, objectOrder,
buffersize, buffersize, preloadTime,
forkfactor, forkfactor, rowdef, txtProps, txtPropsWidth,
rowdef, txtProps, txtPropsWidth,
exitOnFail); exitOnFail);
} }
} }

27
source/de/anomic/kelondro/kelondroStack.java
Unescape View File

@ -67,13 +67,13 @@ public final class kelondroStack extends kelondroRecords {
private static int root = 0; // pointer for FHandles-array: pointer to root node private static int root = 0; // pointer for FHandles-array: pointer to root node
private static int toor = 1; // pointer for FHandles-array: pointer to root node private static int toor = 1; // pointer for FHandles-array: pointer to root node
public kelondroStack(File file, long buffersize, int key, int value, boolean exitOnFail) { public kelondroStack(File file, int key, int value, boolean exitOnFail) {
this(file, buffersize, new kelondroRow(new int[] { key, value }), exitOnFail); this(file, new kelondroRow(new int[] { key, value }), exitOnFail);
} }
public kelondroStack(File file, long buffersize, kelondroRow rowdef, boolean exitOnFail) { public kelondroStack(File file, kelondroRow rowdef, boolean exitOnFail) {
// this creates a new stack // this creates a new stack
super(file, buffersize, thisOHBytes, thisOHHandles, rowdef, thisFHandles, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail); super(file, 0, 0, thisOHBytes, thisOHHandles, rowdef, thisFHandles, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
try { try {
setHandle(root, null); // define the root value setHandle(root, null); // define the root value
setHandle(toor, null); // define the toor value setHandle(toor, null); // define the toor value
@ -84,9 +84,9 @@ public final class kelondroStack extends kelondroRecords {
} }
} }
public kelondroStack(File file, long buffersize) throws IOException{ public kelondroStack(File file) throws IOException{
// this opens a file with an existing stack // this opens a file with an existing stack
super(file, buffersize); super(file, 0, 0);
if ((getHandle(root) == null) && (getHandle(toor) == null)) clear(); if ((getHandle(root) == null) && (getHandle(toor) == null)) clear();
} }
@ -99,7 +99,6 @@ public final class kelondroStack extends kelondroRecords {
public static kelondroStack reset(kelondroStack stack) { public static kelondroStack reset(kelondroStack stack) {
// memorize settings to this file // memorize settings to this file
File f = new File(stack.filename); File f = new File(stack.filename);
long bz = stack.cacheNodeStatus()[0] * stack.cacheNodeChunkSize();
kelondroRow row = stack.row(); kelondroRow row = stack.row();
// close and delete the file // close and delete the file
@ -107,7 +106,7 @@ public final class kelondroStack extends kelondroRecords {
if (f.exists()) f.delete(); if (f.exists()) f.delete();
// re-open a database with same settings as before // re-open a database with same settings as before
return new kelondroStack(f, bz, row, true); return new kelondroStack(f, row, true);
} }
public class Counter implements Iterator { public class Counter implements Iterator {
@ -367,12 +366,12 @@ public final class kelondroStack extends kelondroRecords {
System.err.println("( create, push, view, (g)pop, imp, shell)"); System.err.println("( create, push, view, (g)pop, imp, shell)");
System.exit(0); System.exit(0);
} else if (args.length == 2) { } else if (args.length == 2) {
kelondroStack fm = new kelondroStack(new File(args[1]), 0x100000); kelondroStack fm = new kelondroStack(new File(args[1]));
if (args[0].equals("-v")) { if (args[0].equals("-v")) {
fm.print(); fm.print();
ret = null; ret = null;
} else if (args[0].equals("-g")) { } else if (args[0].equals("-g")) {
fm = new kelondroStack(new File(args[1]), 0x100000); fm = new kelondroStack(new File(args[1]));
kelondroRow.Entry ret2 = fm.pop(); kelondroRow.Entry ret2 = fm.pop();
ret = ((ret2 == null) ? null : ret2.getColBytes(1)); ret = ((ret2 == null) ? null : ret2.getColBytes(1));
fm.close(); fm.close();
@ -380,7 +379,7 @@ public final class kelondroStack extends kelondroRecords {
fm.close(); fm.close();
} else if (args.length == 3) { } else if (args.length == 3) {
if (args[0].equals("-i")) { if (args[0].equals("-i")) {
kelondroStack fm = new kelondroStack(new File(args[2]), 0x100000); kelondroStack fm = new kelondroStack(new File(args[2]));
int i = fm.imp(new File(args[1]),";"); int i = fm.imp(new File(args[1]),";");
fm.close(); fm.close();
ret = (i + " records imported").getBytes(); ret = (i + " records imported").getBytes();
@ -404,7 +403,7 @@ public final class kelondroStack extends kelondroRecords {
if (f != null) try {f.close();}catch(Exception e) {} if (f != null) try {f.close();}catch(Exception e) {}
} }
} else if (args[0].equals("-g")) { } else if (args[0].equals("-g")) {
kelondroStack fm = new kelondroStack(new File(args[2]), 0x100000); kelondroStack fm = new kelondroStack(new File(args[2]));
kelondroRow.Entry ret2 = fm.pop(Integer.parseInt(args[1])); kelondroRow.Entry ret2 = fm.pop(Integer.parseInt(args[1]));
ret = ((ret2 == null) ? null : ret2.getColBytes(1)); ret = ((ret2 == null) ? null : ret2.getColBytes(1));
fm.close(); fm.close();
@ -415,10 +414,10 @@ public final class kelondroStack extends kelondroRecords {
File f = new File(args[3]); File f = new File(args[3]);
if (f.exists()) f.delete(); if (f.exists()) f.delete();
kelondroRow lens = new kelondroRow(new int[]{Integer.parseInt(args[1]), Integer.parseInt(args[2])}); kelondroRow lens = new kelondroRow(new int[]{Integer.parseInt(args[1]), Integer.parseInt(args[2])});
kelondroStack fm = new kelondroStack(f, 0x100000, lens, true); kelondroStack fm = new kelondroStack(f, lens, true);
fm.close(); fm.close();
} else if (args[0].equals("-p")) { } else if (args[0].equals("-p")) {
kelondroStack fm = new kelondroStack(new File(args[3]), 0x100000); kelondroStack fm = new kelondroStack(new File(args[3]));
fm.push(fm.row().newEntry(new byte[][] {args[1].getBytes(), args[2].getBytes()})); fm.push(fm.row().newEntry(new byte[][] {args[1].getBytes(), args[2].getBytes()}));
fm.close(); fm.close();
} }

52
source/de/anomic/kelondro/kelondroTree.java
Unescape View File

@ -92,19 +92,19 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
private kelondroObjectCache objectCache; private kelondroObjectCache objectCache;
public kelondroTree(File file, long buffersize, int objectCachePercent, int key, int value, boolean exitOnFail) { public kelondroTree(File file, long buffersize, long preloadTime, int objectCachePercent, int key, int value, boolean exitOnFail) {
this(file, buffersize, objectCachePercent, new kelondroRow(new int[] { key, value }), new kelondroNaturalOrder(true), 1, 8, exitOnFail); this(file, buffersize, preloadTime, objectCachePercent, new kelondroRow(new int[] { key, value }), new kelondroNaturalOrder(true), 1, 8, exitOnFail);
} }
public kelondroTree(File file, long buffersize, int objectCachePercent, kelondroRow rowdef, boolean exitOnFail) { public kelondroTree(File file, long buffersize, long preloadTime, int objectCachePercent, kelondroRow rowdef, boolean exitOnFail) {
// this creates a new tree file // this creates a new tree file
this(file, buffersize, objectCachePercent, rowdef, new kelondroNaturalOrder(true), rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail); this(file, buffersize, preloadTime, objectCachePercent, rowdef, new kelondroNaturalOrder(true), rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
} }
public kelondroTree(File file, long buffersize, int objectCachePercent, kelondroRow rowdef, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) { public kelondroTree(File file, long buffersize, long preloadTime, int objectCachePercent, kelondroRow rowdef, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) {
// this creates a new tree file // this creates a new tree file
super(file, super(file,
(100 - objectCachePercent) * buffersize / 100, (100 - objectCachePercent) * buffersize / 100, preloadTime,
thisOHBytes, thisOHHandles, rowdef, thisOHBytes, thisOHHandles, rowdef,
thisFHandles, txtProps, txtPropsWidth, exitOnFail); thisFHandles, txtProps, txtPropsWidth, exitOnFail);
try { try {
@ -120,15 +120,15 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
initObjectCache(buffersize, objectCachePercent); initObjectCache(buffersize, objectCachePercent);
} }
public kelondroTree(kelondroRA ra, long buffersize, int objectCachePercent, kelondroRow rowdef, boolean exitOnFail) { public kelondroTree(kelondroRA ra, long buffersize, long preloadTime, int objectCachePercent, kelondroRow rowdef, boolean exitOnFail) {
// this creates a new tree within a kelondroRA // this creates a new tree within a kelondroRA
this(ra, buffersize, objectCachePercent, rowdef, new kelondroNaturalOrder(true), rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail); this(ra, buffersize, preloadTime, objectCachePercent, rowdef, new kelondroNaturalOrder(true), rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
} }
public kelondroTree(kelondroRA ra, long buffersize, int objectCachePercent, kelondroRow rowdef, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) { public kelondroTree(kelondroRA ra, long buffersize, long preloadTime, int objectCachePercent, kelondroRow rowdef, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) {
// this creates a new tree within a kelondroRA // this creates a new tree within a kelondroRA
super(ra, super(ra,
(100 - objectCachePercent) * buffersize / 100, (100 - objectCachePercent) * buffersize / 100, preloadTime,
thisOHBytes, thisOHHandles, rowdef, thisOHBytes, thisOHHandles, rowdef,
thisFHandles, txtProps, txtPropsWidth, exitOnFail); thisFHandles, txtProps, txtPropsWidth, exitOnFail);
try { try {
@ -144,17 +144,17 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
initObjectCache(buffersize, objectCachePercent); initObjectCache(buffersize, objectCachePercent);
} }
public kelondroTree(File file, long buffersize, int objectCachePercent) throws IOException { public kelondroTree(File file, long buffersize, long preloadTime, int objectCachePercent) throws IOException {
// this opens a file with an existing tree file // this opens a file with an existing tree file
super(file, (100 - objectCachePercent) * buffersize / 100); super(file, (100 - objectCachePercent) * buffersize / 100, preloadTime);
readOrderType(); readOrderType();
super.setLogger(log); super.setLogger(log);
initObjectCache(buffersize, objectCachePercent); initObjectCache(buffersize, objectCachePercent);
} }
public kelondroTree(kelondroRA ra, long buffersize, int objectCachePercent) throws IOException { public kelondroTree(kelondroRA ra, long buffersize, long preloadTime, int objectCachePercent) throws IOException {
// this opens a file with an existing tree in a kelondroRA // this opens a file with an existing tree in a kelondroRA
super(ra, (100 - objectCachePercent) * buffersize / 100); super(ra, (100 - objectCachePercent) * buffersize / 100, preloadTime);
readOrderType(); readOrderType();
super.setLogger(log); super.setLogger(log);
initObjectCache(buffersize, objectCachePercent); initObjectCache(buffersize, objectCachePercent);
@ -1299,7 +1299,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
// test script // test script
File testFile = new File("test.db"); File testFile = new File("test.db");
while (testFile.exists()) testFile.delete(); while (testFile.exists()) testFile.delete();
kelondroTree fm = new kelondroTree(testFile, 0x100000, 10, 4, 4, true); kelondroTree fm = new kelondroTree(testFile, 0x100000, 0, 10, 4, 4, true);
byte[] dummy = "".getBytes(); byte[] dummy = "".getBytes();
fm.put("abc0".getBytes(), dummy); fm.put("bcd0".getBytes(), dummy); fm.put("abc0".getBytes(), dummy); fm.put("bcd0".getBytes(), dummy);
fm.put("def0".getBytes(), dummy); fm.put("bab0".getBytes(), dummy); fm.put("def0".getBytes(), dummy); fm.put("bab0".getBytes(), dummy);
@ -1321,7 +1321,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
ret = null; ret = null;
} }
} else if (args.length == 2) { } else if (args.length == 2) {
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10); kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10);
if (args[0].equals("-v")) { if (args[0].equals("-v")) {
fm.print(); fm.print();
ret = null; ret = null;
@ -1329,11 +1329,11 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
fm.close(); fm.close();
} else if (args.length == 3) { } else if (args.length == 3) {
if (args[0].equals("-d")) { if (args[0].equals("-d")) {
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10); kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10);
fm.remove(args[2].getBytes()); fm.remove(args[2].getBytes());
fm.close(); fm.close();
} else if (args[0].equals("-i")) { } else if (args[0].equals("-i")) {
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10); kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10);
int i = fm.imp(new File(args[1]),";"); int i = fm.imp(new File(args[1]),";");
fm.close(); fm.close();
ret = (i + " records imported").getBytes(); ret = (i + " records imported").getBytes();
@ -1356,12 +1356,12 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
if (f != null) try {f.close();}catch(Exception e){} if (f != null) try {f.close();}catch(Exception e){}
} }
} else if (args[0].equals("-g")) { } else if (args[0].equals("-g")) {
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10); kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10);
kelondroRow.Entry ret2 = fm.get(args[2].getBytes()); kelondroRow.Entry ret2 = fm.get(args[2].getBytes());
ret = ((ret2 == null) ? null : ret2.getColBytes(1)); ret = ((ret2 == null) ? null : ret2.getColBytes(1));
fm.close(); fm.close();
} else if (args[0].equals("-n")) { } else if (args[0].equals("-n")) {
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10); kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10);
//byte[][] keys = fm.getSequentialKeys(args[2].getBytes(), 500, true); //byte[][] keys = fm.getSequentialKeys(args[2].getBytes(), 500, true);
Iterator rowIt = fm.rows(true, false, (args[2].length() == 0) ? null : args[2].getBytes()); Iterator rowIt = fm.rows(true, false, (args[2].length() == 0) ? null : args[2].getBytes());
Vector v = new Vector(); Vector v = new Vector();
@ -1375,10 +1375,10 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
File f = new File(args[3]); File f = new File(args[3]);
if (f.exists()) f.delete(); if (f.exists()) f.delete();
kelondroRow lens = new kelondroRow(new int[]{Integer.parseInt(args[1]), Integer.parseInt(args[2])}); kelondroRow lens = new kelondroRow(new int[]{Integer.parseInt(args[1]), Integer.parseInt(args[2])});
kelondroTree fm = new kelondroTree(f, 0x100000, 10, lens, true); kelondroTree fm = new kelondroTree(f, 0x100000, 0, 10, lens, true);
fm.close(); fm.close();
} else if (args[0].equals("-u")) { } else if (args[0].equals("-u")) {
kelondroTree fm = new kelondroTree(new File(args[3]), 0x100000, 10); kelondroTree fm = new kelondroTree(new File(args[3]), 0x100000, 0, 10);
ret = fm.put(args[1].getBytes(), args[2].getBytes()); ret = fm.put(args[1].getBytes(), args[2].getBytes());
fm.close(); fm.close();
} }
@ -1441,7 +1441,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
int steps = 0; int steps = 0;
while (true) { while (true) {
if (testFile.exists()) testFile.delete(); if (testFile.exists()) testFile.delete();
tt = new kelondroTree(testFile, 200, 10, 4 ,4, true); tt = new kelondroTree(testFile, 200, 0, 10, 4 ,4, true);
steps = 10 + ((int) System.currentTimeMillis() % 7) * (((int) System.currentTimeMillis() + 17) % 11); steps = 10 + ((int) System.currentTimeMillis() % 7) * (((int) System.currentTimeMillis() + 17) % 11);
t = s; t = s;
d = ""; d = "";
@ -1507,7 +1507,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
File f = new File("test.db"); File f = new File("test.db");
if (f.exists()) f.delete(); if (f.exists()) f.delete();
try { try {
kelondroTree tt = new kelondroTree(f, 1000, 10, 4, 4, true); kelondroTree tt = new kelondroTree(f, 1000, 0, 10, 4, 4, true);
byte[] b; byte[] b;
b = testWord('B'); tt.put(b, b); //tt.print(); b = testWord('B'); tt.put(b, b); //tt.print();
b = testWord('C'); tt.put(b, b); //tt.print(); b = testWord('C'); tt.put(b, b); //tt.print();
@ -1541,7 +1541,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
File f = new File("test.db"); File f = new File("test.db");
if (f.exists()) f.delete(); if (f.exists()) f.delete();
try { try {
kelondroTree tt = new kelondroTree(f, 0, 10, 4, 4, true); kelondroTree tt = new kelondroTree(f, 0, 0, 10, 4, 4, true);
byte[] b; byte[] b;
for (int i = 0; i < 100; i++) { for (int i = 0; i < 100; i++) {
b = ("T" + i).getBytes(); tt.put(b, b); b = ("T" + i).getBytes(); tt.put(b, b);
@ -1570,7 +1570,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
public static kelondroTree testTree(File f, String testentities) throws IOException { public static kelondroTree testTree(File f, String testentities) throws IOException {
if (f.exists()) f.delete(); if (f.exists()) f.delete();
kelondroTree tt = new kelondroTree(f, 0, 10, 4, 4, true); kelondroTree tt = new kelondroTree(f, 0, 0, 10, 4, 4, true);
byte[] b; byte[] b;
for (int i = 0; i < testentities.length(); i++) { for (int i = 0; i < testentities.length(); i++) {
b = testWord(testentities.charAt(i)); b = testWord(testentities.charAt(i));

1
source/de/anomic/plasma/dbImport/AbstractImporter.java
Unescape View File

@ -16,6 +16,7 @@ public abstract class AbstractImporter extends Thread implements dbImporter{
protected plasmaSwitchboard sb; protected plasmaSwitchboard sb;
protected File importPath; protected File importPath;
protected int cacheSize; protected int cacheSize;
protected long preloadTime;
protected long globalStart = System.currentTimeMillis(); protected long globalStart = System.currentTimeMillis();
protected long globalEnd; protected long globalEnd;

4
source/de/anomic/plasma/dbImport/AssortmentImporter.java
Unescape View File

@ -22,7 +22,7 @@ public class AssortmentImporter extends AbstractImporter implements dbImporter{
this.jobType = "ASSORTMENT"; this.jobType = "ASSORTMENT";
} }
public void init(File theImportAssortmentFile, int theCacheSize) { public void init(File theImportAssortmentFile, int theCacheSize, long preloadTime) {
super.init(theImportAssortmentFile); super.init(theImportAssortmentFile);
this.importAssortmentFile = theImportAssortmentFile; this.importAssortmentFile = theImportAssortmentFile;
this.cacheSize = theCacheSize; this.cacheSize = theCacheSize;
@ -61,7 +61,7 @@ public class AssortmentImporter extends AbstractImporter implements dbImporter{
// initializing the import assortment db // initializing the import assortment db
this.log.logInfo("Initializing source assortment file"); this.log.logInfo("Initializing source assortment file");
this.assortmentFile = new plasmaWordIndexAssortment(importAssortmentPath,assortmentNr, this.cacheSize/1024, this.log); this.assortmentFile = new plasmaWordIndexAssortment(importAssortmentPath,assortmentNr, this.cacheSize/1024, preloadTime, this.log);
this.importStartSize = this.assortmentFile.size(); this.importStartSize = this.assortmentFile.size();
} }

2
source/de/anomic/plasma/dbImport/dbImporter.java
Unescape View File

@ -24,6 +24,6 @@ public interface dbImporter {
public String getError(); public String getError();
public String getStatus(); public String getStatus();
public void init(File importPath, int cacheSize); public void init(File importPath, int cacheSize, long preloadTime);
public void startIt(); public void startIt();
} }

10
source/de/anomic/plasma/dbImport/plasmaCrawlNURLImporter.java
Unescape View File

@ -11,8 +11,7 @@ import de.anomic.plasma.plasmaCrawlProfile;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.plasmaCrawlNURL.Entry; import de.anomic.plasma.plasmaCrawlNURL.Entry;
public class plasmaCrawlNURLImporter extends AbstractImporter implements public class plasmaCrawlNURLImporter extends AbstractImporter implements dbImporter {
dbImporter {
private HashSet importProfileHandleCache = new HashSet(); private HashSet importProfileHandleCache = new HashSet();
private plasmaCrawlProfile importProfileDB; private plasmaCrawlProfile importProfileDB;
@ -47,9 +46,10 @@ public class plasmaCrawlNURLImporter extends AbstractImporter implements
return theStatus.toString(); return theStatus.toString();
} }
public void init(File theImportPath, int theCacheSize) { public void init(File theImportPath, int theCacheSize, long preloadTime) {
super.init(theImportPath); super.init(theImportPath);
this.cacheSize = theCacheSize; this.cacheSize = theCacheSize;
this.preloadTime = preloadTime;
File noticeUrlDbFile = new File(this.importPath,"urlNotice1.db"); File noticeUrlDbFile = new File(this.importPath,"urlNotice1.db");
File profileDbFile = new File(this.importPath, "crawlProfiles0.db"); File profileDbFile = new File(this.importPath, "crawlProfiles0.db");
@ -89,13 +89,13 @@ public class plasmaCrawlNURLImporter extends AbstractImporter implements
// init noticeUrlDB // init noticeUrlDB
this.log.logInfo("Initializing the source noticeUrlDB"); this.log.logInfo("Initializing the source noticeUrlDB");
this.importNurlDB = new plasmaCrawlNURL(this.importPath, ((this.cacheSize*3)/4)/1024); this.importNurlDB = new plasmaCrawlNURL(this.importPath, ((this.cacheSize*3)/4)/1024, preloadTime);
this.importStartSize = this.importNurlDB.size(); this.importStartSize = this.importNurlDB.size();
//int stackSize = this.importNurlDB.stackSize(); //int stackSize = this.importNurlDB.stackSize();
// init profile DB // init profile DB
this.log.logInfo("Initializing the source profileDB"); this.log.logInfo("Initializing the source profileDB");
this.importProfileDB = new plasmaCrawlProfile(profileDbFile, ((this.cacheSize*1)/4)/1024); this.importProfileDB = new plasmaCrawlProfile(profileDbFile, ((this.cacheSize*1)/4)/1024, 300);
} }
public void run() { public void run() {

6
source/de/anomic/plasma/dbImport/plasmaDbImporter.java
Unescape View File

@ -51,7 +51,7 @@ public class plasmaDbImporter extends AbstractImporter implements dbImporter {
return theStatus.toString(); return theStatus.toString();
} }
public void init(File theImportPath, int theCacheSize) { public void init(File theImportPath, int theCacheSize, long preloadTime) {
super.init(theImportPath); super.init(theImportPath);
this.homeWordIndex = this.sb.wordIndex; this.homeWordIndex = this.sb.wordIndex;
@ -75,9 +75,9 @@ public class plasmaDbImporter extends AbstractImporter implements dbImporter {
} }
this.log.logFine("Initializing source word index db."); this.log.logFine("Initializing source word index db.");
this.importWordIndex = new plasmaWordIndex(this.importPath, (this.cacheSize/2)/1024, this.log); this.importWordIndex = new plasmaWordIndex(this.importPath, (this.cacheSize/2)/1024, preloadTime / 2, this.log);
this.log.logFine("Initializing import URL db."); this.log.logFine("Initializing import URL db.");
this.importUrlDB = new plasmaCrawlLURL(new File(this.importPath, "urlHash.db"), (this.cacheSize/2)/1024); this.importUrlDB = new plasmaCrawlLURL(new File(this.importPath, "urlHash.db"), (this.cacheSize/2)/1024, preloadTime / 2);
this.importStartSize = this.importWordIndex.size(); this.importStartSize = this.importWordIndex.size();
} }

8
source/de/anomic/plasma/plasmaCrawlBalancer.java
Unescape View File

@ -58,15 +58,15 @@ public class plasmaCrawlBalancer {
private kelondroStack stack; private kelondroStack stack;
private HashMap domainStacks; private HashMap domainStacks;
public plasmaCrawlBalancer(File stackFile, long buffersize) { public plasmaCrawlBalancer(File stackFile) {
if (stackFile.exists()) { if (stackFile.exists()) {
try { try {
stack = new kelondroStack(stackFile, buffersize); stack = new kelondroStack(stackFile);
} catch (IOException e) { } catch (IOException e) {
stack = new kelondroStack(stackFile, buffersize, new kelondroRow(new int[] {indexURL.urlHashLength}), true); stack = new kelondroStack(stackFile, new kelondroRow(new int[] {indexURL.urlHashLength}), true);
} }
} else { } else {
stack = new kelondroStack(stackFile, buffersize, new kelondroRow(new int[] {indexURL.urlHashLength}), true); stack = new kelondroStack(stackFile, new kelondroRow(new int[] {indexURL.urlHashLength}), true);
} }
domainStacks = new HashMap(); domainStacks = new HashMap();
} }

8
source/de/anomic/plasma/plasmaCrawlEURL.java
Unescape View File

@ -62,7 +62,7 @@ public class plasmaCrawlEURL extends indexURL {
private LinkedList rejectedStack = new LinkedList(); // strings: url private LinkedList rejectedStack = new LinkedList(); // strings: url
public plasmaCrawlEURL(File cachePath, int bufferkb) { public plasmaCrawlEURL(File cachePath, int bufferkb, long preloadTime) {
super(); super();
int[] ce = { int[] ce = {
urlHashLength, // the url's hash urlHashLength, // the url's hash
@ -79,14 +79,14 @@ public class plasmaCrawlEURL extends indexURL {
}; };
if (cachePath.exists()) try { if (cachePath.exists()) try {
// open existing cache // open existing cache
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent); urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) { } catch (IOException e) {
cachePath.delete(); cachePath.delete();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true); urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true);
} else { } else {
// create new cache // create new cache
cachePath.getParentFile().mkdirs(); cachePath.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true); urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true);
} }
} }

10
source/de/anomic/plasma/plasmaCrawlLURL.java
Unescape View File

@ -94,7 +94,7 @@ public final class plasmaCrawlLURL extends indexURL {
//public static Set damagedURLS = Collections.synchronizedSet(new HashSet()); //public static Set damagedURLS = Collections.synchronizedSet(new HashSet());
public plasmaCrawlLURL(File cachePath, int bufferkb) { public plasmaCrawlLURL(File cachePath, int bufferkb, long preloadTime) {
super(); super();
int[] ce = { int[] ce = {
urlHashLength, urlHashLength,
@ -116,15 +116,15 @@ public final class plasmaCrawlLURL extends indexURL {
if (cachePath.exists()) { if (cachePath.exists()) {
// open existing cache // open existing cache
try { try {
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent); urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) { } catch (IOException e) {
cachePath.getParentFile().mkdirs(); cachePath.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true); urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true);
} }
} else { } else {
// create new cache // create new cache
cachePath.getParentFile().mkdirs(); cachePath.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true); urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true);
} }
// init result stacks // init result stacks
@ -974,7 +974,7 @@ public final class plasmaCrawlLURL extends indexURL {
} catch (MalformedURLException e) {} } catch (MalformedURLException e) {}
if (args[0].equals("-l")) try { if (args[0].equals("-l")) try {
// arg 1 is path to URLCache // arg 1 is path to URLCache
final plasmaCrawlLURL urls = new plasmaCrawlLURL(new File(args[1]), 1); final plasmaCrawlLURL urls = new plasmaCrawlLURL(new File(args[1]), 1, 0);
final Iterator enu = urls.entries(true, false); final Iterator enu = urls.entries(true, false);
while (enu.hasNext()) { while (enu.hasNext()) {
((Entry) enu.next()).print(); ((Entry) enu.next()).print();

36
source/de/anomic/plasma/plasmaCrawlNURL.java
Unescape View File

@ -101,12 +101,14 @@ public class plasmaCrawlNURL extends indexURL {
private final HashSet stackIndex; // to find out if a specific link is already on any stack private final HashSet stackIndex; // to find out if a specific link is already on any stack
private File cacheStacksPath; private File cacheStacksPath;
private int bufferkb; private int bufferkb;
private long preloadTime;
initStackIndex initThead; initStackIndex initThead;
public plasmaCrawlNURL(File cacheStacksPath, int bufferkb) { public plasmaCrawlNURL(File cacheStacksPath, int bufferkb, long preloadTime) {
super(); super();
this.cacheStacksPath = cacheStacksPath; this.cacheStacksPath = cacheStacksPath;
this.bufferkb = bufferkb; this.bufferkb = bufferkb;
this.preloadTime = preloadTime;
// create a stack for newly entered entries // create a stack for newly entered entries
if (!(cacheStacksPath.exists())) cacheStacksPath.mkdir(); // make the path if (!(cacheStacksPath.exists())) cacheStacksPath.mkdir(); // make the path
@ -120,31 +122,31 @@ public class plasmaCrawlNURL extends indexURL {
File imageStackFile = new File(cacheStacksPath, "urlNoticeImage0.stack"); File imageStackFile = new File(cacheStacksPath, "urlNoticeImage0.stack");
File movieStackFile = new File(cacheStacksPath, "urlNoticeMovie0.stack"); File movieStackFile = new File(cacheStacksPath, "urlNoticeMovie0.stack");
File musicStackFile = new File(cacheStacksPath, "urlNoticeMusic0.stack"); File musicStackFile = new File(cacheStacksPath, "urlNoticeMusic0.stack");
coreStack = new plasmaCrawlBalancer(coreStackFile, 0); coreStack = new plasmaCrawlBalancer(coreStackFile);
limitStack = new plasmaCrawlBalancer(limitStackFile, 0); limitStack = new plasmaCrawlBalancer(limitStackFile);
overhangStack = new plasmaCrawlBalancer(overhangStackFile, 0); overhangStack = new plasmaCrawlBalancer(overhangStackFile);
remoteStack = new plasmaCrawlBalancer(remoteStackFile, 0); remoteStack = new plasmaCrawlBalancer(remoteStackFile);
kelondroRow rowdef = new kelondroRow(new int[] {indexURL.urlHashLength}); kelondroRow rowdef = new kelondroRow(new int[] {indexURL.urlHashLength});
if (imageStackFile.exists()) try { if (imageStackFile.exists()) try {
imageStack = new kelondroStack(imageStackFile, 0); imageStack = new kelondroStack(imageStackFile);
} catch (IOException e) { } catch (IOException e) {
imageStack = new kelondroStack(imageStackFile, 0, rowdef, true); imageStack = new kelondroStack(imageStackFile, rowdef, true);
} else { } else {
imageStack = new kelondroStack(imageStackFile, 0, rowdef, true); imageStack = new kelondroStack(imageStackFile, rowdef, true);
} }
if (movieStackFile.exists()) try { if (movieStackFile.exists()) try {
movieStack = new kelondroStack(movieStackFile, 0); movieStack = new kelondroStack(movieStackFile);
} catch (IOException e) { } catch (IOException e) {
movieStack = new kelondroStack(movieStackFile, 0, rowdef, true); movieStack = new kelondroStack(movieStackFile, rowdef, true);
} else { } else {
movieStack = new kelondroStack(movieStackFile, 0, rowdef, true); movieStack = new kelondroStack(movieStackFile, rowdef, true);
} }
if (musicStackFile.exists()) try { if (musicStackFile.exists()) try {
musicStack = new kelondroStack(musicStackFile, 0); musicStack = new kelondroStack(musicStackFile);
} catch (IOException e) { } catch (IOException e) {
musicStack = new kelondroStack(musicStackFile, 0, rowdef, true); musicStack = new kelondroStack(musicStackFile, rowdef, true);
} else { } else {
musicStack = new kelondroStack(musicStackFile, 0, rowdef, true); musicStack = new kelondroStack(musicStackFile, rowdef, true);
} }
// init stack Index // init stack Index
@ -166,14 +168,14 @@ public class plasmaCrawlNURL extends indexURL {
File cacheFile = new File(cacheStacksPath, "urlNotice1.db"); File cacheFile = new File(cacheStacksPath, "urlNotice1.db");
if (cacheFile.exists()) try { if (cacheFile.exists()) try {
// open existing cache // open existing cache
urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent); urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) { } catch (IOException e) {
cacheFile.delete(); cacheFile.delete();
urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true); urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true);
} else { } else {
// create new cache // create new cache
cacheFile.getParentFile().mkdirs(); cacheFile.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true); urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true);
} }
} }

13
source/de/anomic/plasma/plasmaCrawlProfile.java
Unescape View File

@ -61,18 +61,21 @@ public class plasmaCrawlProfile {
private HashMap domsCache; private HashMap domsCache;
private File profileTableFile; private File profileTableFile;
private int bufferkb; private int bufferkb;
private long preloadTime;
public plasmaCrawlProfile(File file, int bufferkb) { public plasmaCrawlProfile(File file, int bufferkb, long preloadTime) {
this.profileTableFile = file; this.profileTableFile = file;
this.bufferkb = bufferkb;
this.preloadTime = preloadTime;
kelondroDyn dyn = null; kelondroDyn dyn = null;
if (profileTableFile.exists()) try { if (profileTableFile.exists()) try {
dyn = new kelondroDyn(file, bufferkb * 1024, '#'); dyn = new kelondroDyn(file, bufferkb * 1024, preloadTime, '#');
} catch (IOException e) { } catch (IOException e) {
profileTableFile.delete(); profileTableFile.delete();
dyn = new kelondroDyn(file, bufferkb * 1024, indexURL.urlCrawlProfileHandleLength, 2000, '#', true); dyn = new kelondroDyn(file, bufferkb * 1024, preloadTime, indexURL.urlCrawlProfileHandleLength, 2000, '#', true);
} else { } else {
profileTableFile.getParentFile().mkdirs(); profileTableFile.getParentFile().mkdirs();
dyn = new kelondroDyn(file, bufferkb * 1024, indexURL.urlCrawlProfileHandleLength, 2000, '#', true); dyn = new kelondroDyn(file, bufferkb * 1024, preloadTime, indexURL.urlCrawlProfileHandleLength, 2000, '#', true);
} }
profileTable = new kelondroMap(dyn); profileTable = new kelondroMap(dyn);
domsCache = new HashMap(); domsCache = new HashMap();
@ -95,7 +98,7 @@ public class plasmaCrawlProfile {
if (profileTable != null) try { profileTable.close(); } catch (IOException e) {} if (profileTable != null) try { profileTable.close(); } catch (IOException e) {}
if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database"); if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database");
profileTableFile.getParentFile().mkdirs(); profileTableFile.getParentFile().mkdirs();
profileTable = new kelondroMap(new kelondroDyn(profileTableFile, bufferkb * 1024, indexURL.urlCrawlProfileHandleLength, 2000, '#', true)); profileTable = new kelondroMap(new kelondroDyn(profileTableFile, bufferkb * 1024, preloadTime, indexURL.urlCrawlProfileHandleLength, 2000, '#', true));
} }
public void close() { public void close() {

14
source/de/anomic/plasma/plasmaCrawlRobotsTxt.java
Unescape View File

@ -65,23 +65,25 @@ public class plasmaCrawlRobotsTxt {
kelondroMap robotsTable; kelondroMap robotsTable;
private final File robotsTableFile; private final File robotsTableFile;
private int bufferkb; private int bufferkb;
private long preloadTime;
public plasmaCrawlRobotsTxt(File robotsTableFile, int bufferkb) { public plasmaCrawlRobotsTxt(File robotsTableFile, int bufferkb, long preloadTime) {
this.robotsTableFile = robotsTableFile; this.robotsTableFile = robotsTableFile;
this.bufferkb = bufferkb; this.bufferkb = bufferkb;
this.preloadTime = preloadTime;
if (robotsTableFile.exists()) { if (robotsTableFile.exists()) {
try { try {
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, '_')); robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, preloadTime, '_'));
} catch (kelondroException e) { } catch (kelondroException e) {
robotsTableFile.delete(); robotsTableFile.delete();
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, 256, 512, '_', true)); robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, preloadTime, 256, 512, '_', true));
} catch (IOException e) { } catch (IOException e) {
robotsTableFile.delete(); robotsTableFile.delete();
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, 256, 512, '_', true)); robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, preloadTime, 256, 512, '_', true));
} }
} else { } else {
robotsTableFile.getParentFile().mkdirs(); robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, 256, 512, '_', true)); robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, preloadTime, 256, 512, '_', true));
} }
} }
@ -104,7 +106,7 @@ public class plasmaCrawlRobotsTxt {
} catch (IOException e) {} } catch (IOException e) {}
if (!(robotsTableFile.delete())) throw new RuntimeException("cannot delete robots.txt database"); if (!(robotsTableFile.delete())) throw new RuntimeException("cannot delete robots.txt database");
robotsTableFile.getParentFile().mkdirs(); robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, this.bufferkb, 256, 512, '_', true)); robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, this.bufferkb, preloadTime, 256, 512, '_', true));
} }
public void close() { public void close() {

16
source/de/anomic/plasma/plasmaCrawlStacker.java
Unescape View File

@ -79,10 +79,10 @@ public final class plasmaCrawlStacker {
//private boolean stopped = false; //private boolean stopped = false;
private stackCrawlQueue queue; private stackCrawlQueue queue;
public plasmaCrawlStacker(plasmaSwitchboard sb, File dbPath, int dbCacheSize) { public plasmaCrawlStacker(plasmaSwitchboard sb, File dbPath, int dbCacheSize, long preloadTime) {
this.sb = sb; this.sb = sb;
this.queue = new stackCrawlQueue(dbPath,dbCacheSize); this.queue = new stackCrawlQueue(dbPath, dbCacheSize, preloadTime);
this.log.logInfo(this.queue.size() + " entries in the stackCrawl queue."); this.log.logInfo(this.queue.size() + " entries in the stackCrawl queue.");
this.log.logInfo("STACKCRAWL thread initialized."); this.log.logInfo("STACKCRAWL thread initialized.");
@ -559,7 +559,7 @@ public final class plasmaCrawlStacker {
private final LinkedList urlEntryHashCache; private final LinkedList urlEntryHashCache;
private kelondroTree urlEntryCache; private kelondroTree urlEntryCache;
public stackCrawlQueue(File cacheStacksPath, int bufferkb) { public stackCrawlQueue(File cacheStacksPath, int bufferkb, long preloadTime) {
// init the read semaphore // init the read semaphore
this.readSync = new serverSemaphore (0); this.readSync = new serverSemaphore (0);
@ -576,10 +576,10 @@ public final class plasmaCrawlStacker {
if (cacheFile.exists()) { if (cacheFile.exists()) {
// open existing cache // open existing cache
try { try {
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent); this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) { } catch (IOException e) {
cacheFile.delete(); cacheFile.delete();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(plasmaCrawlNURL.ce), true); this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, new kelondroRow(plasmaCrawlNURL.ce), true);
} }
try { try {
// loop through the list and fill the messageList with url hashs // loop through the list and fill the messageList with url hashs
@ -601,7 +601,7 @@ public final class plasmaCrawlStacker {
// deleting old db and creating a new db // deleting old db and creating a new db
try {this.urlEntryCache.close();}catch(Exception ex){} try {this.urlEntryCache.close();}catch(Exception ex){}
cacheFile.delete(); cacheFile.delete();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(plasmaCrawlNURL.ce), true); this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, new kelondroRow(plasmaCrawlNURL.ce), true);
} catch (IOException e) { } catch (IOException e) {
/* if we have an error, we start with a fresh database */ /* if we have an error, we start with a fresh database */
plasmaCrawlStacker.this.log.logSevere("Unable to initialize crawl stacker queue, IOException:" + e.getMessage() + ". Reseting DB.\n",e); plasmaCrawlStacker.this.log.logSevere("Unable to initialize crawl stacker queue, IOException:" + e.getMessage() + ". Reseting DB.\n",e);
@ -609,12 +609,12 @@ public final class plasmaCrawlStacker {
// deleting old db and creating a new db // deleting old db and creating a new db
try {this.urlEntryCache.close();}catch(Exception ex){} try {this.urlEntryCache.close();}catch(Exception ex){}
cacheFile.delete(); cacheFile.delete();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(plasmaCrawlNURL.ce), true); this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, new kelondroRow(plasmaCrawlNURL.ce), true);
} }
} else { } else {
// create new cache // create new cache
cacheFile.getParentFile().mkdirs(); cacheFile.getParentFile().mkdirs();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(plasmaCrawlNURL.ce), true); this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, new kelondroRow(plasmaCrawlNURL.ce), true);
} }
} }

6
source/de/anomic/plasma/plasmaHTCache.java
Unescape View File

@ -97,7 +97,7 @@ public final class plasmaHTCache {
public final serverLog log; public final serverLog log;
public static final HashSet filesInUse = new HashSet(); // can we delete this file public static final HashSet filesInUse = new HashSet(); // can we delete this file
public plasmaHTCache(File htCachePath, long maxCacheSize, int bufferkb) { public plasmaHTCache(File htCachePath, long maxCacheSize, int bufferkb, long preloadTime) {
// this.switchboard = switchboard; // this.switchboard = switchboard;
this.log = new serverLog("HTCACHE"); this.log = new serverLog("HTCACHE");
@ -156,9 +156,9 @@ public final class plasmaHTCache {
File dbfile = new File(this.cachePath, "responseHeader.db"); File dbfile = new File(this.cachePath, "responseHeader.db");
try { try {
if (dbfile.exists()) if (dbfile.exists())
this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400, '#')); this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400, preloadTime, '#'));
else else
this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400, indexURL.urlHashLength, 150, '#', false)); this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400, preloadTime, indexURL.urlHashLength, 150, '#', false));
} catch (IOException e) { } catch (IOException e) {
this.log.logSevere("the request header database could not be opened: " + e.getMessage()); this.log.logSevere("the request header database could not be opened: " + e.getMessage());
System.exit(0); System.exit(0);

93
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -329,33 +329,44 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
} }
// read memory amount // read memory amount
int ramLURL = (int) getConfigLong("ramCacheLURL", 1024) / 1024; int ramLURL = (int) getConfigLong("ramCacheLURL", 1024) / 1024;
int ramNURL = (int) getConfigLong("ramCacheNURL", 1024) / 1024; long ramLURL_time = getConfigLong("ramCacheLURL_time", 1000);
int ramEURL = (int) getConfigLong("ramCacheEURL", 1024) / 1024; int ramNURL = (int) getConfigLong("ramCacheNURL", 1024) / 1024;
int ramRWI = (int) getConfigLong("ramCacheRWI", 1024) / 1024; long ramNURL_time = getConfigLong("ramCacheNURL_time", 1000);
int ramHTTP = (int) getConfigLong("ramCacheHTTP", 1024) / 1024; int ramEURL = (int) getConfigLong("ramCacheEURL", 1024) / 1024;
int ramMessage = (int) getConfigLong("ramCacheMessage", 1024) / 1024; long ramEURL_time = getConfigLong("ramCacheEURL_time", 1000);
int ramRobots = (int) getConfigLong("ramCacheRobots",1024) / 1024; int ramRWI = (int) getConfigLong("ramCacheRWI", 1024) / 1024;
int ramProfiles= (int) getConfigLong("ramCacheProfiles",1024) / 1024; long ramRWI_time = getConfigLong("ramCacheRWI_time", 1000);
int ramPreNURL = (int) getConfigLong("ramCachePreNURL", 1024) / 1024; int ramHTTP = (int) getConfigLong("ramCacheHTTP", 1024) / 1024;
int ramWiki = (int) getConfigLong("ramCacheWiki", 1024) / 1024; long ramHTTP_time = getConfigLong("ramCacheHTTP_time", 1000);
int ramBlog = (int) getConfigLong("ramCacheBlog", 1024) / 1024; int ramMessage = (int) getConfigLong("ramCacheMessage", 1024) / 1024;
this.log.logConfig("LURL Cache memory = " + ppRamString(ramLURL)); long ramMessage_time = getConfigLong("ramCacheMessage_time", 1000);
this.log.logConfig("NURL Cache memory = " + ppRamString(ramNURL)); int ramRobots = (int) getConfigLong("ramCacheRobots",1024) / 1024;
this.log.logConfig("EURL Cache memory = " + ppRamString(ramEURL)); long ramRobots_time = getConfigLong("ramCacheRobots_time",1000);
this.log.logConfig("RWI Cache memory = " + ppRamString(ramRWI)); int ramProfiles = (int) getConfigLong("ramCacheProfiles",1024) / 1024;
this.log.logConfig("HTTP Cache memory = " + ppRamString(ramHTTP)); long ramProfiles_time= getConfigLong("ramCacheProfiles_time", 1000);
this.log.logConfig("Message Cache memory = " + ppRamString(ramMessage)); int ramPreNURL = (int) getConfigLong("ramCachePreNURL", 1024) / 1024;
this.log.logConfig("Wiki Cache memory = " + ppRamString(ramWiki)); long ramPreNURL_time = getConfigLong("ramCachePreNURL_time", 1000);
this.log.logConfig("Blog Cache memory = " + ppRamString(ramBlog)); int ramWiki = (int) getConfigLong("ramCacheWiki", 1024) / 1024;
this.log.logConfig("Robots Cache memory = " + ppRamString(ramRobots)); long ramWiki_time = getConfigLong("ramCacheWiki_time", 1000);
this.log.logConfig("Profiles Cache memory = " + ppRamString(ramProfiles)); int ramBlog = (int) getConfigLong("ramCacheBlog", 1024) / 1024;
this.log.logConfig("PreNURL Cache memory = " + ppRamString(ramPreNURL)); long ramBlog_time = getConfigLong("ramCacheBlog_time", 1000);
this.log.logConfig("LURL Cache memory = " + ppRamString(ramLURL) + ", preloadTime = " + ramLURL_time);
this.log.logConfig("NURL Cache memory = " + ppRamString(ramNURL) + ", preloadTime = " + ramNURL_time);
this.log.logConfig("EURL Cache memory = " + ppRamString(ramEURL) + ", preloadTime = " + ramEURL_time);
this.log.logConfig("RWI Cache memory = " + ppRamString(ramRWI) + ", preloadTime = " + ramRWI_time);
this.log.logConfig("HTTP Cache memory = " + ppRamString(ramHTTP) + ", preloadTime = " + ramHTTP_time);
this.log.logConfig("Message Cache memory = " + ppRamString(ramMessage) + ", preloadTime = " + ramMessage_time);
this.log.logConfig("Wiki Cache memory = " + ppRamString(ramWiki) + ", preloadTime = " + ramWiki_time);
this.log.logConfig("Blog Cache memory = " + ppRamString(ramBlog) + ", preloadTime = " + ramBlog_time);
this.log.logConfig("Robots Cache memory = " + ppRamString(ramRobots) + ", preloadTime = " + ramRobots_time);
this.log.logConfig("Profiles Cache memory = " + ppRamString(ramProfiles) + ", preloadTime = " + ramProfiles_time);
this.log.logConfig("PreNURL Cache memory = " + ppRamString(ramPreNURL) + ", preloadTime = " + ramPreNURL_time);
// make crawl profiles database and default profiles // make crawl profiles database and default profiles
this.log.logConfig("Initializing Crawl Profiles"); this.log.logConfig("Initializing Crawl Profiles");
File profilesFile = new File(this.plasmaPath, "crawlProfiles0.db"); File profilesFile = new File(this.plasmaPath, "crawlProfiles0.db");
this.profiles = new plasmaCrawlProfile(profilesFile, ramProfiles); this.profiles = new plasmaCrawlProfile(profilesFile, ramProfiles, ramProfiles_time);
initProfiles(); initProfiles();
log.logConfig("Loaded profiles from file " + profilesFile.getName() + log.logConfig("Loaded profiles from file " + profilesFile.getName() +
", " + this.profiles.size() + " entries" + ", " + this.profiles.size() + " entries" +
@ -364,16 +375,16 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// loading the robots.txt db // loading the robots.txt db
this.log.logConfig("Initializing robots.txt DB"); this.log.logConfig("Initializing robots.txt DB");
File robotsDBFile = new File(this.plasmaPath, "crawlRobotsTxt.db"); File robotsDBFile = new File(this.plasmaPath, "crawlRobotsTxt.db");
robots = new plasmaCrawlRobotsTxt(robotsDBFile, ramRobots); robots = new plasmaCrawlRobotsTxt(robotsDBFile, ramRobots, ramRobots_time);
this.log.logConfig("Loaded robots.txt DB from file " + robotsDBFile.getName() + this.log.logConfig("Loaded robots.txt DB from file " + robotsDBFile.getName() +
", " + robots.size() + " entries" + ", " + robots.size() + " entries" +
", " + ppRamString(robotsDBFile.length()/1024)); ", " + ppRamString(robotsDBFile.length()/1024));
// start indexing management // start indexing management
log.logConfig("Starting Indexing Management"); log.logConfig("Starting Indexing Management");
urlPool = new plasmaURLPool(plasmaPath, ramLURL, ramNURL, ramEURL); urlPool = new plasmaURLPool(plasmaPath, ramLURL, ramNURL, ramEURL, ramLURL_time);
wordIndex = new plasmaWordIndex(plasmaPath, ramRWI, log); wordIndex = new plasmaWordIndex(plasmaPath, ramRWI, ramRWI_time, log);
int wordCacheMaxCount = (int) getConfigLong("wordCacheMaxCount", 10000); int wordCacheMaxCount = (int) getConfigLong("wordCacheMaxCount", 10000);
wordIndex.setMaxWordCount(wordCacheMaxCount); wordIndex.setMaxWordCount(wordCacheMaxCount);
@ -391,7 +402,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
} }
this.log.logInfo("HTCACHE Path = " + htCachePath.getAbsolutePath()); this.log.logInfo("HTCACHE Path = " + htCachePath.getAbsolutePath());
long maxCacheSize = 1024 * 1024 * Long.parseLong(getConfig("proxyCacheSize", "2")); // this is megabyte long maxCacheSize = 1024 * 1024 * Long.parseLong(getConfig("proxyCacheSize", "2")); // this is megabyte
this.cacheManager = new plasmaHTCache(htCachePath, maxCacheSize, ramHTTP); this.cacheManager = new plasmaHTCache(htCachePath, maxCacheSize, ramHTTP, ramHTTP_time);
// make parser // make parser
log.logConfig("Starting Parser"); log.logConfig("Starting Parser");
@ -460,18 +471,18 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
Boolean.valueOf(getConfig(CRAWLJOB_GLOBAL_CRAWL_TRIGGER + "_isPaused", "false"))}); Boolean.valueOf(getConfig(CRAWLJOB_GLOBAL_CRAWL_TRIGGER + "_isPaused", "false"))});
// starting board // starting board
initMessages(ramMessage); initMessages(ramMessage, ramMessage_time);
// starting wiki // starting wiki
initWiki(ramWiki); initWiki(ramWiki, ramWiki_time);
//starting blog //starting blog
initBlog(ramBlog); initBlog(ramBlog, ramBlog_time);
// Init User DB // Init User DB
this.log.logConfig("Loading User DB"); this.log.logConfig("Loading User DB");
File userDbFile = new File(getRootPath(), "DATA/SETTINGS/user.db"); File userDbFile = new File(getRootPath(), "DATA/SETTINGS/user.db");
this.userDB = new userDB(userDbFile, 512); this.userDB = new userDB(userDbFile, 512, 500);
this.log.logConfig("Loaded User DB from file " + userDbFile.getName() + this.log.logConfig("Loaded User DB from file " + userDbFile.getName() +
", " + this.userDB.size() + " entries" + ", " + this.userDB.size() + " entries" +
", " + ppRamString(userDbFile.length()/1024)); ", " + ppRamString(userDbFile.length()/1024));
@ -547,7 +558,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
serverInstantThread.oneTimeJob(yc, "loadSeeds", yacyCore.log, 3000); serverInstantThread.oneTimeJob(yc, "loadSeeds", yacyCore.log, 3000);
// initializing the stackCrawlThread // initializing the stackCrawlThread
this.sbStackCrawlThread = new plasmaCrawlStacker(this,this.plasmaPath,ramPreNURL); this.sbStackCrawlThread = new plasmaCrawlStacker(this, this.plasmaPath, ramPreNURL, ramPreNURL_time);
//this.sbStackCrawlThread = new plasmaStackCrawlThread(this,this.plasmaPath,ramPreNURL); //this.sbStackCrawlThread = new plasmaStackCrawlThread(this,this.plasmaPath,ramPreNURL);
//this.sbStackCrawlThread.start(); //this.sbStackCrawlThread.start();
@ -613,29 +624,28 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
} }
public void initMessages(int ramMessage) { public void initMessages(int ramMessage, long ramMessage_time) {
this.log.logConfig("Starting Message Board"); this.log.logConfig("Starting Message Board");
File messageDbFile = new File(workPath, "message.db"); File messageDbFile = new File(workPath, "message.db");
this.messageDB = new messageBoard(messageDbFile, ramMessage); this.messageDB = new messageBoard(messageDbFile, ramMessage, ramMessage_time);
this.log.logConfig("Loaded Message Board DB from file " + messageDbFile.getName() + this.log.logConfig("Loaded Message Board DB from file " + messageDbFile.getName() +
", " + this.messageDB.size() + " entries" + ", " + this.messageDB.size() + " entries" +
", " + ppRamString(messageDbFile.length()/1024)); ", " + ppRamString(messageDbFile.length()/1024));
} }
public void initWiki(int ramWiki) { public void initWiki(int ramWiki, long ramWiki_time) {
this.log.logConfig("Starting Wiki Board"); this.log.logConfig("Starting Wiki Board");
File wikiDbFile = new File(workPath, "wiki.db"); File wikiDbFile = new File(workPath, "wiki.db");
this.wikiDB = new wikiBoard(wikiDbFile, this.wikiDB = new wikiBoard(wikiDbFile, new File(workPath, "wiki-bkp.db"), ramWiki, ramWiki_time);
new File(workPath, "wiki-bkp.db"), ramWiki);
this.log.logConfig("Loaded Wiki Board DB from file " + wikiDbFile.getName() + this.log.logConfig("Loaded Wiki Board DB from file " + wikiDbFile.getName() +
", " + this.wikiDB.size() + " entries" + ", " + this.wikiDB.size() + " entries" +
", " + ppRamString(wikiDbFile.length()/1024)); ", " + ppRamString(wikiDbFile.length()/1024));
} }
public void initBlog(int ramBlog) { public void initBlog(int ramBlog, long ramBlog_time) {
this.log.logConfig("Starting Blog"); this.log.logConfig("Starting Blog");
File blogDbFile = new File(workPath, "blog.db"); File blogDbFile = new File(workPath, "blog.db");
this.blogDB = new blogBoard(blogDbFile, ramBlog); this.blogDB = new blogBoard(blogDbFile, ramBlog, ramBlog_time);
this.log.logConfig("Loaded Blog DB from file " + blogDbFile.getName() + this.log.logConfig("Loaded Blog DB from file " + blogDbFile.getName() +
", " + this.blogDB.size() + " entries" + ", " + this.blogDB.size() + " entries" +
", " + ppRamString(blogDbFile.length()/1024)); ", " + ppRamString(blogDbFile.length()/1024));
@ -645,7 +655,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
File bookmarksFile = new File(workPath, "bookmarks.db"); File bookmarksFile = new File(workPath, "bookmarks.db");
File tagsFile = new File(workPath, "bookmarkTags.db"); File tagsFile = new File(workPath, "bookmarkTags.db");
File datesFile = new File(workPath, "bookmarkDates.db"); File datesFile = new File(workPath, "bookmarkDates.db");
this.bookmarksDB = new bookmarksDB(bookmarksFile, tagsFile, datesFile, 512); this.bookmarksDB = new bookmarksDB(bookmarksFile, tagsFile, datesFile, 512, 500);
this.log.logConfig("Loaded Bookmarks DB from files "+ bookmarksFile.getName()+ ", "+tagsFile.getName()); this.log.logConfig("Loaded Bookmarks DB from files "+ bookmarksFile.getName()+ ", "+tagsFile.getName());
this.log.logConfig(this.bookmarksDB.tagsSize()+" Tag, "+this.bookmarksDB.bookmarksSize()+" Bookmarks"); this.log.logConfig(this.bookmarksDB.tagsSize()+" Tag, "+this.bookmarksDB.bookmarksSize()+" Bookmarks");
} }
@ -709,7 +719,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
final File pdb = new File(plasmaPath, "crawlProfiles0.db"); final File pdb = new File(plasmaPath, "crawlProfiles0.db");
if (pdb.exists()) pdb.delete(); if (pdb.exists()) pdb.delete();
int ramProfiles = (int) getConfigLong("ramCacheProfiles", 1024) / 1024; int ramProfiles = (int) getConfigLong("ramCacheProfiles", 1024) / 1024;
profiles = new plasmaCrawlProfile(pdb, ramProfiles); long ramProfiles_time = getConfigLong("ramCacheProfiles_time", 1000);
profiles = new plasmaCrawlProfile(pdb, ramProfiles, ramProfiles_time);
initProfiles(); initProfiles();
} }

6
source/de/anomic/plasma/plasmaSwitchboardQueue.java
Unescape View File

@ -91,12 +91,12 @@ public class plasmaSwitchboardQueue {
indexURL.urlDescrLength indexURL.urlDescrLength
}); });
if (sbQueueStackPath.exists()) try { if (sbQueueStackPath.exists()) try {
sbQueueStack = new kelondroStack(sbQueueStackPath, 0); sbQueueStack = new kelondroStack(sbQueueStackPath);
} catch (IOException e) { } catch (IOException e) {
sbQueueStackPath.delete(); sbQueueStackPath.delete();
sbQueueStack = new kelondroStack(sbQueueStackPath, 0, rowdef, true); sbQueueStack = new kelondroStack(sbQueueStackPath, rowdef, true);
} else { } else {
sbQueueStack = new kelondroStack(sbQueueStackPath, 0, rowdef, true); sbQueueStack = new kelondroStack(sbQueueStackPath, rowdef, true);
} }
} }

8
source/de/anomic/plasma/plasmaURLPool.java
Unescape View File

@ -57,10 +57,10 @@ public class plasmaURLPool {
public final plasmaCrawlNURL noticeURL; public final plasmaCrawlNURL noticeURL;
public final plasmaCrawlEURL errorURL; public final plasmaCrawlEURL errorURL;
public plasmaURLPool(File plasmaPath, int ramLURL, int ramNURL, int ramEURL) { public plasmaURLPool(File plasmaPath, int ramLURL, int ramNURL, int ramEURL, long preloadTime) {
loadedURL = new plasmaCrawlLURL(new File(plasmaPath, "urlHash.db"), ramLURL); loadedURL = new plasmaCrawlLURL(new File(plasmaPath, "urlHash.db"), ramLURL, preloadTime);
noticeURL = new plasmaCrawlNURL(plasmaPath, ramNURL); noticeURL = new plasmaCrawlNURL(plasmaPath, ramNURL, 0);
errorURL = new plasmaCrawlEURL(new File(plasmaPath, "urlErr0.db"), ramEURL); errorURL = new plasmaCrawlEURL(new File(plasmaPath, "urlErr0.db"), ramEURL, 0);
} }
public String exists(String hash) { public String exists(String hash) {

8
source/de/anomic/plasma/plasmaWordConnotation.java
Unescape View File

@ -58,13 +58,13 @@ public class plasmaWordConnotation {
private static final int nodesize = 4048; private static final int nodesize = 4048;
private kelondroDynTree refDB; private kelondroDynTree refDB;
public plasmaWordConnotation(File refDBfile, int bufferkb, char fillChar) { public plasmaWordConnotation(File refDBfile, int bufferkb, long preloadTime, char fillChar) {
if (refDBfile.exists()) try { if (refDBfile.exists()) try {
refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, fillChar); refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, preloadTime, fillChar);
} catch (IOException e) { } catch (IOException e) {
refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, wordlength, nodesize, new kelondroRow(new int[] {wordlength, countlength}), fillChar, true); refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, preloadTime, wordlength, nodesize, new kelondroRow(new int[] {wordlength, countlength}), fillChar, true);
} else { } else {
refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, wordlength, nodesize, new kelondroRow(new int[] {wordlength, countlength}), fillChar, true); refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, preloadTime, wordlength, nodesize, new kelondroRow(new int[] {wordlength, countlength}), fillChar, true);
} }
} }

6
source/de/anomic/plasma/plasmaWordIndex.java
Unescape View File

@ -85,7 +85,7 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI {
private final plasmaWordIndexFileCluster backend; private final plasmaWordIndexFileCluster backend;
private final kelondroOrder indexOrder = new kelondroNaturalOrder(true); private final kelondroOrder indexOrder = new kelondroNaturalOrder(true);
public plasmaWordIndex(File databaseRoot, int bufferkb, serverLog log) { public plasmaWordIndex(File databaseRoot, int bufferkb, long preloadTime, serverLog log) {
this.databaseRoot = databaseRoot; this.databaseRoot = databaseRoot;
this.backend = new plasmaWordIndexFileCluster(databaseRoot, log); this.backend = new plasmaWordIndexFileCluster(databaseRoot, log);
this.ramCache = new indexRAMCacheRI(databaseRoot, log); this.ramCache = new indexRAMCacheRI(databaseRoot, log);
@ -94,7 +94,7 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI {
File assortmentClusterPath = new File(databaseRoot, indexAssortmentClusterPath); File assortmentClusterPath = new File(databaseRoot, indexAssortmentClusterPath);
if (!(assortmentClusterPath.exists())) assortmentClusterPath.mkdirs(); if (!(assortmentClusterPath.exists())) assortmentClusterPath.mkdirs();
this.assortmentBufferSize = bufferkb; this.assortmentBufferSize = bufferkb;
this.assortmentCluster = new plasmaWordIndexAssortmentCluster(assortmentClusterPath, assortmentCount, assortmentBufferSize, log); this.assortmentCluster = new plasmaWordIndexAssortmentCluster(assortmentClusterPath, assortmentCount, assortmentBufferSize, preloadTime, log);
} }
public File getRoot() { public File getRoot() {
@ -673,7 +673,7 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI {
// System.out.println(kelondroMSetTools.fastStringComparator(true).compare("RwGeoUdyDQ0Y", "rwGeoUdyDQ0Y")); // System.out.println(kelondroMSetTools.fastStringComparator(true).compare("RwGeoUdyDQ0Y", "rwGeoUdyDQ0Y"));
// System.out.println(new Date(reverseMicroDateDays(microDateDays(System.currentTimeMillis())))); // System.out.println(new Date(reverseMicroDateDays(microDateDays(System.currentTimeMillis()))));
plasmaWordIndex index = new plasmaWordIndex(new File("D:\\dev\\proxy\\DATA\\PLASMADB"), 555, new serverLog("TESTAPP")); plasmaWordIndex index = new plasmaWordIndex(new File("D:\\dev\\proxy\\DATA\\PLASMADB"), 555, 1000, new serverLog("TESTAPP"));
try { try {
Iterator iter = index.wordHashes("5A8yhZMh_Kmv", plasmaWordIndex.RL_WORDFILES, true); Iterator iter = index.wordHashes("5A8yhZMh_Kmv", plasmaWordIndex.RL_WORDFILES, true);
while (iter.hasNext()) { while (iter.hasNext()) {

10
source/de/anomic/plasma/plasmaWordIndexAssortment.java
Unescape View File

@ -83,6 +83,7 @@ public final class plasmaWordIndexAssortment {
private serverLog log; private serverLog log;
private kelondroTree assortments; private kelondroTree assortments;
private long bufferSize; private long bufferSize;
private long preloadTime;
private static String intx(int x) { private static String intx(int x) {
String s = Integer.toString(x); String s = Integer.toString(x);
@ -102,17 +103,18 @@ public final class plasmaWordIndexAssortment {
return structure; return structure;
} }
public plasmaWordIndexAssortment(File storagePath, int assortmentLength, int bufferkb, serverLog log) { public plasmaWordIndexAssortment(File storagePath, int assortmentLength, int bufferkb, long preloadTime, serverLog log) {
if (!(storagePath.exists())) storagePath.mkdirs(); if (!(storagePath.exists())) storagePath.mkdirs();
this.assortmentFile = new File(storagePath, assortmentFileName + intx(assortmentLength) + ".db"); this.assortmentFile = new File(storagePath, assortmentFileName + intx(assortmentLength) + ".db");
this.assortmentLength = assortmentLength; this.assortmentLength = assortmentLength;
//this.bufferStructureLength = 3 + 2 * assortmentLength; //this.bufferStructureLength = 3 + 2 * assortmentLength;
this.bufferSize = bufferkb * 1024; this.bufferSize = bufferkb * 1024;
this.preloadTime = preloadTime;
this.log = log; this.log = log;
if (assortmentFile.exists()) { if (assortmentFile.exists()) {
// open existing assortment tree file // open existing assortment tree file
try { try {
assortments = new kelondroTree(assortmentFile, bufferSize, kelondroTree.defaultObjectCachePercent); assortments = new kelondroTree(assortmentFile, bufferSize, preloadTime, kelondroTree.defaultObjectCachePercent);
if (log != null) log.logConfig("Opened Assortment Database, " + assortments.size() + " entries, width " + assortmentLength + ", " + bufferkb + "kb buffer"); if (log != null) log.logConfig("Opened Assortment Database, " + assortments.size() + " entries, width " + assortmentLength + ", " + bufferkb + "kb buffer");
return; return;
} catch (IOException e){ } catch (IOException e){
@ -123,7 +125,7 @@ public final class plasmaWordIndexAssortment {
assortmentFile.delete(); // make space for new one assortmentFile.delete(); // make space for new one
} }
// create new assortment tree file // create new assortment tree file
assortments = new kelondroTree(assortmentFile, bufferSize, kelondroTree.defaultObjectCachePercent, new kelondroRow(bufferStructure(assortmentLength)), true); assortments = new kelondroTree(assortmentFile, bufferSize, preloadTime, kelondroTree.defaultObjectCachePercent, new kelondroRow(bufferStructure(assortmentLength)), true);
if (log != null) log.logConfig("Created new Assortment Database, width " + assortmentLength + ", " + bufferkb + "kb buffer"); if (log != null) log.logConfig("Created new Assortment Database, width " + assortmentLength + ", " + bufferkb + "kb buffer");
} }
@ -243,7 +245,7 @@ public final class plasmaWordIndexAssortment {
if (!(assortmentFile.delete())) throw new RuntimeException("cannot delete assortment database"); if (!(assortmentFile.delete())) throw new RuntimeException("cannot delete assortment database");
} }
if (assortmentFile.exists()) assortmentFile.delete(); if (assortmentFile.exists()) assortmentFile.delete();
assortments = new kelondroTree(assortmentFile, bufferSize, kelondroTree.defaultObjectCachePercent, new kelondroRow(bufferStructure(assortmentLength)), true); assortments = new kelondroTree(assortmentFile, bufferSize, preloadTime, kelondroTree.defaultObjectCachePercent, new kelondroRow(bufferStructure(assortmentLength)), true);
} }
public Iterator hashes(String startWordHash, boolean up, boolean rot) throws IOException { public Iterator hashes(String startWordHash, boolean up, boolean rot) throws IOException {

10
source/de/anomic/plasma/plasmaWordIndexAssortmentCluster.java
Unescape View File

@ -72,7 +72,7 @@ public final class plasmaWordIndexAssortmentCluster extends indexAbstractRI impl
private plasmaWordIndexAssortment[] assortments; private plasmaWordIndexAssortment[] assortments;
private long completeBufferKB; private long completeBufferKB;
public plasmaWordIndexAssortmentCluster(File assortmentsPath, int clusterCount, int bufferkb, serverLog log) { public plasmaWordIndexAssortmentCluster(File assortmentsPath, int clusterCount, int bufferkb, long preloadTime, serverLog log) {
// set class variables // set class variables
if (!(assortmentsPath.exists())) assortmentsPath.mkdirs(); if (!(assortmentsPath.exists())) assortmentsPath.mkdirs();
this.clusterCount = clusterCount; this.clusterCount = clusterCount;
@ -86,7 +86,7 @@ public final class plasmaWordIndexAssortmentCluster extends indexAbstractRI impl
int sumSizes = 1; int sumSizes = 1;
plasmaWordIndexAssortment testAssortment; plasmaWordIndexAssortment testAssortment;
for (int i = 0; i < clusterCount; i++) { for (int i = 0; i < clusterCount; i++) {
testAssortment = new plasmaWordIndexAssortment(assortmentsPath, i + 1, 0, null); testAssortment = new plasmaWordIndexAssortment(assortmentsPath, i + 1, 0, 0, null);
sizes[i] = testAssortment.size() + clusterCount - i; sizes[i] = testAssortment.size() + clusterCount - i;
sumSizes += sizes[i]; sumSizes += sizes[i];
testAssortment.close(); testAssortment.close();
@ -96,7 +96,11 @@ public final class plasmaWordIndexAssortmentCluster extends indexAbstractRI impl
// initialize cluster using the cluster elements size for optimal buffer // initialize cluster using the cluster elements size for optimal buffer
// size // size
for (int i = 0; i < clusterCount; i++) { for (int i = 0; i < clusterCount; i++) {
assortments[i] = new plasmaWordIndexAssortment(assortmentsPath, i + 1, (int) (completeBufferKB * (long) sizes[i] / (long) sumSizes), log); assortments[i] = new plasmaWordIndexAssortment(
assortmentsPath, i + 1,
(int) (completeBufferKB * (long) sizes[i] / (long) sumSizes),
preloadTime * (long) sizes[i] / (long) sumSizes,
log);
} }
} }

6
source/de/anomic/plasma/plasmaWordIndexFile.java
Unescape View File

@ -92,13 +92,13 @@ public final class plasmaWordIndexFile {
if (cacheSize > 1048576) cacheSize = 1048576; if (cacheSize > 1048576) cacheSize = 1048576;
if (theLocation.exists()) try { if (theLocation.exists()) try {
// open existing index file // open existing index file
kt = new kelondroTree(theLocation, cacheSize, kelondroTree.defaultObjectCachePercent); kt = new kelondroTree(theLocation, cacheSize, 1000, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) { } catch (IOException e) {
theLocation.delete(); theLocation.delete();
kt = new kelondroTree(theLocation, cacheSize, kelondroTree.defaultObjectCachePercent, indexURL.urlHashLength, indexURLEntry.encodedStringFormLength(), false); kt = new kelondroTree(theLocation, cacheSize, 1000, kelondroTree.defaultObjectCachePercent, indexURL.urlHashLength, indexURLEntry.encodedStringFormLength(), false);
} else { } else {
// create new index file // create new index file
kt = new kelondroTree(theLocation, cacheSize, kelondroTree.defaultObjectCachePercent, indexURL.urlHashLength, indexURLEntry.encodedStringFormLength(), false); kt = new kelondroTree(theLocation, cacheSize, 1000, kelondroTree.defaultObjectCachePercent, indexURL.urlHashLength, indexURLEntry.encodedStringFormLength(), false);
} }
return kt; // everyone who get this should close it when finished! return kt; // everyone who get this should close it when finished!
} }

6
source/de/anomic/yacy/yacyCore.java
Unescape View File

@ -173,18 +173,20 @@ public class yacyCore {
// create or init seed cache // create or init seed cache
int memDHT = Integer.parseInt(switchboard.getConfig("ramCacheDHT", "1024")) / 1024; int memDHT = Integer.parseInt(switchboard.getConfig("ramCacheDHT", "1024")) / 1024;
long memDHT_time = Long.parseLong(switchboard.getConfig("ramCacheDHT_time", "1000"));
log.logConfig("DHT Cache memory = " + memDHT + " KB"); log.logConfig("DHT Cache memory = " + memDHT + " KB");
seedDB = new yacySeedDB( seedDB = new yacySeedDB(
sb, sb,
new File(yacyDBPath, "seed.new.db"), new File(yacyDBPath, "seed.new.db"),
new File(yacyDBPath, "seed.old.db"), new File(yacyDBPath, "seed.old.db"),
new File(yacyDBPath, "seed.pot.db"), new File(yacyDBPath, "seed.pot.db"),
memDHT); memDHT, memDHT_time);
// create or init news database // create or init news database
int memNews = Integer.parseInt(switchboard.getConfig("ramCacheNews", "1024")) / 1024; int memNews = Integer.parseInt(switchboard.getConfig("ramCacheNews", "1024")) / 1024;
long memNews_time = Long.parseLong(switchboard.getConfig("ramCacheNews_time", "1000"));
log.logConfig("News Cache memory = " + memNews + " KB"); log.logConfig("News Cache memory = " + memNews + " KB");
newsPool = new yacyNewsPool(yacyDBPath, memNews); newsPool = new yacyNewsPool(yacyDBPath, memNews, memNews_time);
loadSeedUploadMethods(); loadSeedUploadMethods();

18
source/de/anomic/yacy/yacyNewsDB.java
Unescape View File

@ -61,6 +61,7 @@ public class yacyNewsDB {
private File path; private File path;
private int bufferkb; private int bufferkb;
private long preloadTime;
private kelondroTree news; private kelondroTree news;
public static final int attributesMaxLength = yacyNewsRecord.maxNewsRecordLength public static final int attributesMaxLength = yacyNewsRecord.maxNewsRecordLength
@ -69,16 +70,17 @@ public class yacyNewsDB {
- yacyCore.universalDateShortPattern.length() - yacyCore.universalDateShortPattern.length()
- 2; - 2;
public yacyNewsDB(File path, int bufferkb) { public yacyNewsDB(File path, int bufferkb, long preloadTime) {
this.path = path; this.path = path;
this.bufferkb = bufferkb; this.bufferkb = bufferkb;
this.preloadTime = preloadTime;
if (path.exists()) try { if (path.exists()) try {
news = new kelondroTree(path, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent); news = new kelondroTree(path, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) { } catch (IOException e) {
news = createDB(path, bufferkb); news = createDB(path, bufferkb, preloadTime);
} else { } else {
news = createDB(path, bufferkb); news = createDB(path, bufferkb, preloadTime);
} }
} }
@ -90,14 +92,14 @@ public class yacyNewsDB {
new kelondroColumn("", kelondroColumn.celltype_string, attributesMaxLength, kelondroColumn.encoder_string, attributesMaxLength, ""), new kelondroColumn("", kelondroColumn.celltype_string, attributesMaxLength, kelondroColumn.encoder_string, attributesMaxLength, ""),
}); });
private static kelondroTree createDB(File path, int bufferkb) { private static kelondroTree createDB(File path, int bufferkb, long preloadTime) {
return new kelondroTree(path, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, rowdef, true); return new kelondroTree(path, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, true);
} }
private void resetDB() { private void resetDB() {
try {close();} catch (Exception e) {} try {close();} catch (Exception e) {}
if (path.exists()) path.delete(); if (path.exists()) path.delete();
news = createDB(path, bufferkb); news = createDB(path, bufferkb, preloadTime);
} }
public int dbCacheNodeChunkSize() { public int dbCacheNodeChunkSize() {

4
source/de/anomic/yacy/yacyNewsPool.java
Unescape View File

@ -93,8 +93,8 @@ public class yacyNewsPool {
private int maxDistribution; private int maxDistribution;
public yacyNewsPool(File yacyDBPath, int bufferkb) { public yacyNewsPool(File yacyDBPath, int bufferkb, long preloadTime) {
newsDB = new yacyNewsDB(new File(yacyDBPath, "news1.db"), bufferkb); newsDB = new yacyNewsDB(new File(yacyDBPath, "news1.db"), bufferkb, preloadTime);
outgoingNews = new yacyNewsQueue(new File(yacyDBPath, "newsOut1.stack"), newsDB); outgoingNews = new yacyNewsQueue(new File(yacyDBPath, "newsOut1.stack"), newsDB);
publishedNews = new yacyNewsQueue(new File(yacyDBPath, "newsPublished1.stack"), newsDB); publishedNews = new yacyNewsQueue(new File(yacyDBPath, "newsPublished1.stack"), newsDB);
incomingNews = new yacyNewsQueue(new File(yacyDBPath, "newsIn1.stack"), newsDB); incomingNews = new yacyNewsQueue(new File(yacyDBPath, "newsIn1.stack"), newsDB);

4
source/de/anomic/yacy/yacyNewsQueue.java
Unescape View File

@ -64,7 +64,7 @@ public class yacyNewsQueue {
this.newsDB = newsDB; this.newsDB = newsDB;
if (path.exists()) try { if (path.exists()) try {
queueStack = new kelondroStack(path, 0); queueStack = new kelondroStack(path);
} catch (kelondroException e) { } catch (kelondroException e) {
path.delete(); path.delete();
queueStack = createStack(path); queueStack = createStack(path);
@ -82,7 +82,7 @@ public class yacyNewsQueue {
}); });
private static kelondroStack createStack(File path) { private static kelondroStack createStack(File path) {
return new kelondroStack(path, 0, rowdef, true); return new kelondroStack(path, rowdef, true);
} }
private void resetDB() { private void resetDB() {

8
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -93,6 +93,7 @@ public final class yacySeedDB {
private kelondroMap seedActiveDB, seedPassiveDB, seedPotentialDB; private kelondroMap seedActiveDB, seedPassiveDB, seedPotentialDB;
private int seedDBBufferKB; private int seedDBBufferKB;
private long preloadTime;
public final plasmaSwitchboard sb; public final plasmaSwitchboard sb;
public yacySeed mySeed; // my own seed public yacySeed mySeed; // my own seed
@ -104,7 +105,7 @@ public final class yacySeedDB {
File seedActiveDBFile, File seedActiveDBFile,
File seedPassiveDBFile, File seedPassiveDBFile,
File seedPotentialDBFile, File seedPotentialDBFile,
int bufferkb) { int bufferkb, long preloadTime) {
this.seedDBBufferKB = bufferkb; this.seedDBBufferKB = bufferkb;
this.seedActiveDBFile = seedActiveDBFile; this.seedActiveDBFile = seedActiveDBFile;
@ -112,6 +113,7 @@ public final class yacySeedDB {
this.seedPotentialDBFile = seedPotentialDBFile; this.seedPotentialDBFile = seedPotentialDBFile;
this.mySeed = null; // my own seed this.mySeed = null; // my own seed
this.sb = sb; this.sb = sb;
this.preloadTime = preloadTime;
// set up seed database // set up seed database
seedActiveDB = openSeedTable(seedActiveDBFile); seedActiveDB = openSeedTable(seedActiveDBFile);
@ -195,7 +197,7 @@ public final class yacySeedDB {
private synchronized kelondroMap openSeedTable(File seedDBFile) { private synchronized kelondroMap openSeedTable(File seedDBFile) {
if (seedDBFile.exists()) try { if (seedDBFile.exists()) try {
// open existing seed database // open existing seed database
return new kelondroMap(new kelondroDyn(seedDBFile, (seedDBBufferKB * 0x400) / 3, '#'), sortFields, accFields); return new kelondroMap(new kelondroDyn(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, '#'), sortFields, accFields);
} catch (kelondroException e) { } catch (kelondroException e) {
// if we have an error, we start with a fresh database // if we have an error, we start with a fresh database
if (seedDBFile.exists()) seedDBFile.delete(); if (seedDBFile.exists()) seedDBFile.delete();
@ -205,7 +207,7 @@ public final class yacySeedDB {
} }
// create new seed database // create new seed database
new File(seedDBFile.getParent()).mkdir(); new File(seedDBFile.getParent()).mkdir();
return new kelondroMap(new kelondroDyn(seedDBFile, (seedDBBufferKB * 0x400) / 3, commonHashLength, 480, '#', true), sortFields, accFields); return new kelondroMap(new kelondroDyn(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', true), sortFields, accFields);
} }
private synchronized kelondroMap resetSeedTable(kelondroMap seedDB, File seedDBFile) { private synchronized kelondroMap resetSeedTable(kelondroMap seedDB, File seedDBFile) {

4
source/migration.java
Unescape View File

@ -137,7 +137,7 @@ public class migration {
file.delete(); file.delete();
} catch (IOException e) {} } catch (IOException e) {}
} }
sb.initWiki((int) sb.getConfigLong("ramCacheWiki", 1024) / 1024); sb.initWiki((int) sb.getConfigLong("ramCacheWiki", 1024) / 1024, sb.getConfigLong("ramCacheWiki_time", 1000));
} }
@ -150,7 +150,7 @@ public class migration {
serverFileUtils.copy(file, file2); serverFileUtils.copy(file, file2);
file.delete(); file.delete();
} catch (IOException e) {} } catch (IOException e) {}
sb.initMessages((int) sb.getConfigLong("ramCacheMessage", 1024) / 1024); sb.initMessages((int) sb.getConfigLong("ramCacheMessage", 1024) / 1024, sb.getConfigLong("ramCacheMessage_time", 1000));
} }
} }

22
source/yacy.java
Unescape View File

@ -663,7 +663,7 @@ public final class yacy {
File dbroot = new File(new File(homePath), "DATA/PLASMADB"); File dbroot = new File(new File(homePath), "DATA/PLASMADB");
serverLog log = new serverLog("WORDMIGRATION"); serverLog log = new serverLog("WORDMIGRATION");
log.logInfo("STARTING MIGRATION"); log.logInfo("STARTING MIGRATION");
plasmaWordIndex wordIndexCache = new plasmaWordIndex(dbroot, 20000, log); plasmaWordIndex wordIndexCache = new plasmaWordIndex(dbroot, 20000, 10000, log);
enumerateFiles words = new enumerateFiles(new File(dbroot, "WORDS"), true, false, true, true); enumerateFiles words = new enumerateFiles(new File(dbroot, "WORDS"), true, false, true, true);
String wordhash; String wordhash;
File wordfile; File wordfile;
@ -708,16 +708,16 @@ public final class yacy {
// db containing all currently loades urls // db containing all currently loades urls
int cache = dbcache * 1024; // in KB int cache = dbcache * 1024; // in KB
log.logFine("URLDB-Caches: "+cache+" bytes"); log.logFine("URLDB-Caches: "+cache+" bytes");
plasmaCrawlLURL currentUrlDB = new plasmaCrawlLURL(new File(dbroot, "urlHash.db"), cache); plasmaCrawlLURL currentUrlDB = new plasmaCrawlLURL(new File(dbroot, "urlHash.db"), cache, 10000);
// db used to hold all neede urls // db used to hold all neede urls
plasmaCrawlLURL minimizedUrlDB = new plasmaCrawlLURL(new File(dbroot, "urlHash.temp.db"), cache); plasmaCrawlLURL minimizedUrlDB = new plasmaCrawlLURL(new File(dbroot, "urlHash.temp.db"), cache, 10000);
Runtime rt = Runtime.getRuntime(); Runtime rt = Runtime.getRuntime();
int cacheMem = (int)((rt.maxMemory()-rt.totalMemory())/1024)-(2*cache + 8*1024); int cacheMem = (int)((rt.maxMemory()-rt.totalMemory())/1024)-(2*cache + 8*1024);
if (cacheMem < 2048) throw new OutOfMemoryError("Not enough memory available to start clean up."); if (cacheMem < 2048) throw new OutOfMemoryError("Not enough memory available to start clean up.");
plasmaWordIndex wordIndex = new plasmaWordIndex(dbroot, cacheMem, log); plasmaWordIndex wordIndex = new plasmaWordIndex(dbroot, cacheMem, 10000, log);
Iterator wordHashIterator = wordIndex.wordHashes("------------", plasmaWordIndex.RL_WORDFILES, false); Iterator wordHashIterator = wordIndex.wordHashes("------------", plasmaWordIndex.RL_WORDFILES, false);
String wordhash; String wordhash;
@ -950,7 +950,7 @@ public final class yacy {
File root = new File(homePath); File root = new File(homePath);
try { try {
plasmaURLPool pool = new plasmaURLPool(new File(root, "DATA/PLASMADB"), 16000, 1000, 1000); plasmaURLPool pool = new plasmaURLPool(new File(root, "DATA/PLASMADB"), 16000, 1000, 1000, 10000);
Iterator eiter = pool.loadedURL.entries(true, false); Iterator eiter = pool.loadedURL.entries(true, false);
HashSet doms = new HashSet(); HashSet doms = new HashSet();
plasmaCrawlLURL.Entry entry; plasmaCrawlLURL.Entry entry;
@ -1016,7 +1016,7 @@ public final class yacy {
private static void urllist(String homePath, boolean html, String targetName) { private static void urllist(String homePath, boolean html, String targetName) {
File root = new File(homePath); File root = new File(homePath);
try { try {
plasmaURLPool pool = new plasmaURLPool(new File(root, "DATA/PLASMADB"), 16000, 1000, 1000); plasmaURLPool pool = new plasmaURLPool(new File(root, "DATA/PLASMADB"), 16000, 1000, 1000, 10000);
Iterator eiter = pool.loadedURL.entries(true, false); Iterator eiter = pool.loadedURL.entries(true, false);
plasmaCrawlLURL.Entry entry; plasmaCrawlLURL.Entry entry;
File file = new File(root, targetName); File file = new File(root, targetName);
@ -1058,7 +1058,7 @@ public final class yacy {
File dbroot = new File(root, "DATA/PLASMADB"); File dbroot = new File(root, "DATA/PLASMADB");
serverLog log = new serverLog("URLDBCLEANUP"); serverLog log = new serverLog("URLDBCLEANUP");
try { try {
plasmaCrawlLURL currentUrlDB = new plasmaCrawlLURL(new File(dbroot, "urlHash.db"), 4194304); plasmaCrawlLURL currentUrlDB = new plasmaCrawlLURL(new File(dbroot, "urlHash.db"), 4194304, 10000);
currentUrlDB.urldbcleanup(); currentUrlDB.urldbcleanup();
currentUrlDB.close(); currentUrlDB.close();
} catch (IOException e) { } catch (IOException e) {
@ -1077,14 +1077,14 @@ public final class yacy {
try { try {
Iterator WordHashIterator = null; Iterator WordHashIterator = null;
if (resource.equals("all")) { if (resource.equals("all")) {
WordIndex = new plasmaWordIndex(homeDBroot, 8*1024*1024, log); WordIndex = new plasmaWordIndex(homeDBroot, 8*1024*1024, 3000, log);
WordHashIterator = WordIndex.wordHashes(wordChunkStartHash, plasmaWordIndex.RL_WORDFILES, false); WordHashIterator = WordIndex.wordHashes(wordChunkStartHash, plasmaWordIndex.RL_WORDFILES, false);
} else if (resource.equals("assortments")) { } else if (resource.equals("assortments")) {
plasmaWordIndexAssortmentCluster assortmentCluster = new plasmaWordIndexAssortmentCluster(new File(homeDBroot, "ACLUSTER"), 64, 16*1024*1024, log); plasmaWordIndexAssortmentCluster assortmentCluster = new plasmaWordIndexAssortmentCluster(new File(homeDBroot, "ACLUSTER"), 64, 16*1024*1024, 3000, log);
WordHashIterator = assortmentCluster.wordHashes(wordChunkStartHash, true, false); WordHashIterator = assortmentCluster.wordHashes(wordChunkStartHash, true, false);
} else if (resource.startsWith("assortment")) { } else if (resource.startsWith("assortment")) {
int a = Integer.parseInt(resource.substring(10)); int a = Integer.parseInt(resource.substring(10));
plasmaWordIndexAssortment assortment = new plasmaWordIndexAssortment(new File(homeDBroot, "ACLUSTER"), a, 8*1024*1024, null); plasmaWordIndexAssortment assortment = new plasmaWordIndexAssortment(new File(homeDBroot, "ACLUSTER"), a, 8*1024*1024, 3000, null);
WordHashIterator = assortment.hashes(wordChunkStartHash, true, false); WordHashIterator = assortment.hashes(wordChunkStartHash, true, false);
} else if (resource.equals("words")) { } else if (resource.equals("words")) {
plasmaWordIndexFileCluster fileDB = new plasmaWordIndexFileCluster(homeDBroot, log); plasmaWordIndexFileCluster fileDB = new plasmaWordIndexFileCluster(homeDBroot, log);
@ -1147,7 +1147,7 @@ public final class yacy {
String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"}; String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"};
for (int i=0; i < dbFileNames.length; i++) { for (int i=0; i < dbFileNames.length; i++) {
File dbFile = new File(yacyDBPath,dbFileNames[i]); File dbFile = new File(yacyDBPath,dbFileNames[i]);
kelondroMap db = new kelondroMap(new kelondroDyn(dbFile, (1024 * 0x400) / 3, '#'), yacySeedDB.sortFields, yacySeedDB.accFields); kelondroMap db = new kelondroMap(new kelondroDyn(dbFile, (1024 * 0x400) / 3, 3000, '#'), yacySeedDB.sortFields, yacySeedDB.accFields);
kelondroMap.mapIterator it; kelondroMap.mapIterator it;
it = db.maps(true, false); it = db.maps(true, false);

39
yacy.init
Unescape View File

@ -496,43 +496,56 @@ xpstopw=true
# ram cache for database files # ram cache for database files
# ram cache for assortment cache cluster (for all 64 files) # ram cache for assortment cache cluster (for all 64 files)
ramCacheRWI = 8388608 ramCacheRWI = 8388608
ramCacheRWI_time = 30000
# ram cache for responseHeader.db # ram cache for responseHeader.db
ramCacheHTTP = 1048576 ramCacheHTTP = 1048576
ramCacheHTTP_time = 1000
# ram cache for urlHash.db # ram cache for urlHash.db
ramCacheLURL = 8388608 ramCacheLURL = 8388608
ramCacheLURL_time = 10000
# ram cache for urlNotice.db # ram cache for urlNotice.db
ramCacheNURL = 4194304 ramCacheNURL = 4194304
ramCacheNURL_time = 2000
# ram cache for urlErr.db # ram cache for urlErr.db
ramCacheEURL = 8192 ramCacheEURL = 8192
ramCacheEURL_time = 1000
# ram cache for seedDBs # ram cache for seedDBs
ramCacheDHT = 131072 ramCacheDHT = 131072
ramCacheDHT_time = 1000
# ram cache for message.db # ram cache for message.db
ramCacheMessage = 8192 ramCacheMessage = 8192
ramCacheMessage_time = 500
# ram cache for wiki.db # ram cache for wiki.db
ramCacheWiki = 8192 ramCacheWiki = 8192
ramCacheWiki_time = 500
# ram cache for blog.db # ram cache for blog.db
ramCacheBlog = 2048 ramCacheBlog = 2048
ramCacheBlog_time = 500
# ram cache for news1.db # ram cache for news1.db
ramCacheNews = 1048576 ramCacheNews = 1048576
ramCacheNews_time = 1000
# ram cache for robotsTxt.db # ram cache for robotsTxt.db
ramCacheRobots = 4194304 ramCacheRobots = 4194304
ramCacheRobots_time = 3000
# ram cache for crawlProfile.db # ram cache for crawlProfile.db
ramCacheProfiles = 8192 ramCacheProfiles = 8192
ramCacheProfiles_time= 500
# ram cache for stack crawl thread db # ram cache for stack crawl thread db
ramCachePreNURL = 4194304 ramCachePreNURL = 4194304
ramCachePreNURL_time = 3000
# default memory settings for startup of yacy # default memory settings for startup of yacy
# is only valid in unix/shell environments and # is only valid in unix/shell environments and

Write Preview
Loading…
Cancel
Save