boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- changed order of database initialisation (index first)

Browse Source 
- removed mainly unused init-time for databases (was only used for tree tables, which are not used any more)


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4496 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent 3441ec3928
commit 2327451653
31 changed files with 124 additions and 213 deletions
Show Stats Download Patch File Download Diff File

7
source/dbtest.java
Unescape View File

@ -34,7 +34,6 @@ public class dbtest {
public final static int keylength = 12;
public final static int valuelength = 223; // sum of all data length as defined in plasmaURL
public final static long preload = 1000; // 1 second
public static byte[] dummyvalue2 = new byte[valuelength];
static {
// fill the dummy value
@ -201,15 +200,15 @@ public class dbtest {
}
if (dbe.equals("kelondroTree")) {
File tablefile = new File(tablename + ".kelondro.db");
return new kelondroCache(new kelondroTree(tablefile, true, preload, testRow));
return new kelondroCache(new kelondroTree(tablefile, true, 0, testRow));
}
if (dbe.equals("kelondroFlexTable")) {
File tablepath = new File(tablename).getParentFile();
return new kelondroFlexTable(tablepath, new File(tablename).getName(), preload, testRow, 0, true);
return new kelondroFlexTable(tablepath, new File(tablename).getName(), testRow, 0, true);
}
if (dbe.equals("kelondroSplitTable")) {
File tablepath = new File(tablename).getParentFile();
return new kelondroSplitTable(tablepath, new File(tablename).getName(), preload, testRow, true);
return new kelondroSplitTable(tablepath, new File(tablename).getName(), testRow, true);
}
if (dbe.equals("kelondroEcoTable")) {
return new kelondroEcoTable(new File(tablename), testRow, kelondroEcoTable.tailCacheForceUsage, 1000, 0);

4
source/de/anomic/data/blogBoard.java
Unescape View File

@ -80,10 +80,10 @@ public class blogBoard {
kelondroMapObjects database = null;
public blogBoard(File actpath, long preloadTime) {
public blogBoard(File actpath) {
new File(actpath.getParent()).mkdir();
if (database == null) {
database = new kelondroMapObjects(new kelondroDyn(actpath, true, true, preloadTime, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
database = new kelondroMapObjects(new kelondroDyn(actpath, true, true, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
}
}
public int size() {

4
source/de/anomic/data/blogBoardComments.java
Unescape View File

@ -82,10 +82,10 @@ public class blogBoardComments {
SimpleFormatter.setTimeZone(TimeZone.getTimeZone("GMT"));
}
private kelondroMapObjects database = null;
public blogBoardComments(File actpath, long preloadTime) {
public blogBoardComments(File actpath) {
new File(actpath.getParent()).mkdir();
if (database == null) {
database = new kelondroMapObjects(new kelondroDyn(actpath, true, true, preloadTime, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, false, false, false), 500);
database = new kelondroMapObjects(new kelondroDyn(actpath, true, true, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, false, false, false), 500);
}
}
public int size() {

8
source/de/anomic/data/bookmarksDB.java
Unescape View File

@ -111,22 +111,22 @@ public class bookmarksDB {
// bookmarksDB's class constructor
// ------------------------------------
public bookmarksDB(File bookmarksFile, File tagsFile, File datesFile, long preloadTime) {
public bookmarksDB(File bookmarksFile, File tagsFile, File datesFile) {
// bookmarks
tagCache=new HashMap<String, Tag>();
bookmarksFile.getParentFile().mkdirs();
//this.bookmarksTable = new kelondroMap(kelondroDyn.open(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true, false));
this.bookmarksTable = new kelondroObjects(new kelondroDyn(bookmarksFile, true, true, preloadTime, 12, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 1000);
this.bookmarksTable = new kelondroObjects(new kelondroDyn(bookmarksFile, true, true, 12, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 1000);
// tags
tagsFile.getParentFile().mkdirs();
boolean tagsFileExisted = tagsFile.exists();
this.tagsTable = new kelondroMapObjects(new kelondroDyn(tagsFile, true, true, preloadTime, 12, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
this.tagsTable = new kelondroMapObjects(new kelondroDyn(tagsFile, true, true, 12, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
if (!tagsFileExisted) rebuildTags();
// dates
boolean datesExisted = datesFile.exists();
this.datesTable = new kelondroMapObjects(new kelondroDyn(datesFile, true, true, preloadTime, 20, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
this.datesTable = new kelondroMapObjects(new kelondroDyn(datesFile, true, true, 20, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
if (!datesExisted) rebuildDates();
}

4
source/de/anomic/data/messageBoard.java
Unescape View File

@ -69,10 +69,10 @@ public class messageBoard {
private kelondroMapObjects database = null;
private int sn = 0;
public messageBoard(File path, long preloadTime) {
public messageBoard(File path) {
new File(path.getParent()).mkdir();
if (database == null) {
database = new kelondroMapObjects(new kelondroDyn(path, true, true, preloadTime, categoryLength + dateFormat.length() + 2, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
database = new kelondroMapObjects(new kelondroDyn(path, true, true, categoryLength + dateFormat.length() + 2, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
}
sn = 0;
}

8
source/de/anomic/data/userDB.java
Unescape View File

@ -70,15 +70,13 @@ public final class userDB {
kelondroMapObjects userTable;
private final File userTableFile;
private long preloadTime;
private HashMap<String, String> ipUsers = new HashMap<String, String>();
private HashMap<String, Object> cookieUsers = new HashMap<String, Object>();
public userDB(File userTableFile, long preloadTime) {
public userDB(File userTableFile) {
this.userTableFile = userTableFile;
this.preloadTime = preloadTime;
userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMapObjects(new kelondroDyn(userTableFile, true, true, preloadTime, 128, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 10);
this.userTable = new kelondroMapObjects(new kelondroDyn(userTableFile, true, true, 128, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 10);
}
void resetDatabase() {
@ -86,7 +84,7 @@ public final class userDB {
if (userTable != null) userTable.close();
if (!(userTableFile.delete())) throw new RuntimeException("cannot delete user database");
userTableFile.getParentFile().mkdirs();
userTable = new kelondroMapObjects(new kelondroDyn(userTableFile, true, true, preloadTime, 256, 512, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 10);
userTable = new kelondroMapObjects(new kelondroDyn(userTableFile, true, true, 256, 512, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 10);
}
public void close() {

6
source/de/anomic/data/wikiBoard.java
Unescape View File

@ -70,14 +70,14 @@ public class wikiBoard {
private kelondroMapObjects bkpbase = null;
private static HashMap<String, String> authors = new HashMap<String, String>();
public wikiBoard(File actpath, File bkppath, long preloadTime) {
public wikiBoard(File actpath, File bkppath) {
new File(actpath.getParent()).mkdirs();
if (datbase == null) {
datbase = new kelondroMapObjects(new kelondroDyn(actpath, true, true, preloadTime, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
datbase = new kelondroMapObjects(new kelondroDyn(actpath, true, true, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
}
new File(bkppath.getParent()).mkdirs();
if (bkpbase == null) {
bkpbase = new kelondroMapObjects(new kelondroDyn(bkppath, true, true, preloadTime, keyLength + dateFormat.length(), recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
bkpbase = new kelondroMapObjects(new kelondroDyn(bkppath, true, true, keyLength + dateFormat.length(), recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
}
}

8
source/de/anomic/htmlFilter/htmlFilterContentScraper.java
Unescape View File

@ -375,7 +375,7 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
public String getDescription() {
String s = metas.get("description");
if (s == null) s = metas.get("DC.description");
if (s == null) s = metas.get("dc.description");
if (s == null) return ""; else return s;
}
@ -387,21 +387,21 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
public String getAuthor() {
String s = metas.get("author");
if (s == null) s = metas.get("copyright");
if (s == null) s = metas.get("DC.creator");
if (s == null) s = metas.get("dc.creator");
if (s == null) return "";
return s;
}
public String[] getContentLanguages() {
String s = metas.get("content-language");
if (s == null) s = metas.get("DC.language");
if (s == null) s = metas.get("dc.language");
if (s == null) s = "";
return s.split(" |,");
}
public String[] getKeywords() {
String s = metas.get("keywords");
if (s == null) s = metas.get("DC.description");
if (s == null) s = metas.get("dc.description");
if (s == null) s = "";
if (s.length() == 0) {
return getTitle().toLowerCase().split(splitrex);

3
source/de/anomic/index/indexCollectionRI.java
Unescape View File

@ -45,14 +45,13 @@ public class indexCollectionRI implements indexRI {
kelondroCollectionIndex collectionIndex;
public indexCollectionRI(File path, String filenameStub, long preloadTime, int maxpartition, kelondroRow payloadrow) {
public indexCollectionRI(File path, String filenameStub, int maxpartition, kelondroRow payloadrow) {
try {
collectionIndex = new kelondroCollectionIndex(
path,
filenameStub,
12 /*keyLength*/,
kelondroBase64Order.enhancedCoder,
preloadTime,
4 /*loadfactor*/,
maxpartition,
payloadrow);

15
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -114,7 +114,7 @@ public class kelondroCollectionIndex {
}
public kelondroCollectionIndex(File path, String filenameStub, int keyLength, kelondroByteOrder indexOrder,
long preloadTime, int loadfactor, int maxpartitions, kelondroRow rowdef) throws IOException {
int loadfactor, int maxpartitions, kelondroRow rowdef) throws IOException {
// the buffersize is number of bytes that are only used if the kelondroFlexTable is backed up with a kelondroTree
this.path = path;
this.filenameStub = filenameStub;
@ -131,7 +131,7 @@ public class kelondroCollectionIndex {
// open index and array files
this.arrays = new HashMap<String, kelondroFixedWidthArray>(); // all entries will be dynamically created with getArray()
index = openIndexFile(path, filenameStub, indexOrder, preloadTime, loadfactor, rowdef, 0);
index = openIndexFile(path, filenameStub, indexOrder, loadfactor, rowdef, 0);
openAllArrayFiles(false, indexOrder);
} else {
// calculate initialSpace
@ -164,7 +164,7 @@ public class kelondroCollectionIndex {
} else if (serverMemory.request(necessaryRAM4fullIndex, false)) {
index = new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, initialSpace);
} else {
index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRowdef, initialSpace, true);
index = new kelondroFlexTable(path, filenameStub + ".index", indexRowdef, initialSpace, true);
}
// open array files
@ -264,14 +264,14 @@ public class kelondroCollectionIndex {
}
private kelondroIndex openIndexFile(File path, String filenameStub, kelondroByteOrder indexOrder,
long preloadTime, int loadfactor, kelondroRow rowdef, int initialSpace) throws IOException {
int loadfactor, kelondroRow rowdef, int initialSpace) throws IOException {
// open/create index table
File f = new File(path, filenameStub + ".index");
kelondroRow indexRowdef = indexRow(keylength, indexOrder);
if (f.isDirectory()) {
// use a flextable
kelondroIndex theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRowdef, initialSpace, true));
kelondroIndex theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", indexRowdef, initialSpace, true));
// save/check property file for this array
File propfile = propertyFile(path, filenameStub, loadfactor, rowdef.objectsize);
@ -1087,12 +1087,11 @@ public class kelondroCollectionIndex {
File path = new File(args[0]);
String filenameStub = args[1];
long preloadTime = 10000;
try {
// initialize collection index
kelondroCollectionIndex collectionIndex = new kelondroCollectionIndex(
path, filenameStub, 9 /*keyLength*/,
kelondroNaturalOrder.naturalOrder, preloadTime,
kelondroNaturalOrder.naturalOrder,
4 /*loadfactor*/, 7, rowdef);
// fill index with values
@ -1120,7 +1119,7 @@ public class kelondroCollectionIndex {
// printout of index
collectionIndex.close();
kelondroFlexTable index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, kelondroCollectionIndex.indexRow(9, kelondroNaturalOrder.naturalOrder), 0, true);
kelondroFlexTable index = new kelondroFlexTable(path, filenameStub + ".index", kelondroCollectionIndex.indexRow(9, kelondroNaturalOrder.naturalOrder), 0, true);
index.print();
index.close();
} catch (IOException e) {

10
source/de/anomic/kelondro/kelondroDyn.java
Unescape View File

@ -71,14 +71,14 @@ public class kelondroDyn {
private kelondroObjectBuffer buffer;
private kelondroRow rowdef;
public kelondroDyn(File file, boolean useNodeCache, boolean useObjectCache, long preloadTime, int key,
public kelondroDyn(File file, boolean useNodeCache, boolean useObjectCache, int key,
int nodesize, char fillChar, kelondroByteOrder objectOrder, boolean usetree, boolean writebuffer, boolean resetOnFail) {
// creates or opens a dynamic tree
rowdef = new kelondroRow("byte[] key-" + (key + counterlen) + ", byte[] node-" + nodesize, objectOrder, 0);
kelondroIndex fbi;
if (usetree) {
try {
fbi = new kelondroTree(file, useNodeCache, preloadTime, rowdef, 1, 8);
fbi = new kelondroTree(file, useNodeCache, 0, rowdef, 1, 8);
} catch (IOException e) {
e.printStackTrace();
if (resetOnFail) {
@ -97,7 +97,7 @@ public class kelondroDyn {
} else {
if (file.exists()) {
if (file.isDirectory()) {
fbi = new kelondroFlexTable(file.getParentFile(), file.getName(), 10000, rowdef, 0, resetOnFail);
fbi = new kelondroFlexTable(file.getParentFile(), file.getName(), rowdef, 0, resetOnFail);
} else {
fbi = new kelondroEcoTable(file, rowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize, 0);
}
@ -482,7 +482,7 @@ public class kelondroDyn {
if (args.length == 1) {
// open a db and list keys
try {
kelondroDyn kd = new kelondroDyn(new File(args[0]), true, true, 0, 4 ,100, '_', kelondroNaturalOrder.naturalOrder, false, false, true);
kelondroDyn kd = new kelondroDyn(new File(args[0]), true, true, 4 ,100, '_', kelondroNaturalOrder.naturalOrder, false, false, true);
System.out.println(kd.sizeDyn() + " elements in DB");
Iterator<String> i = kd.dynKeys(true, false);
while (i.hasNext())
@ -499,7 +499,7 @@ public class kelondroDyn {
File f = new File(args[3]);
kelondroDyn kd;
try {
kd = new kelondroDyn(db, true, true, 0, 80, 200, '_', kelondroNaturalOrder.naturalOrder, false, false, true);
kd = new kelondroDyn(db, true, true, 80, 200, '_', kelondroNaturalOrder.naturalOrder, false, false, true);
if (writeFile)
kd.readFile(key, f);
else

14
source/de/anomic/kelondro/kelondroDynTree.java
Unescape View File

@ -62,23 +62,21 @@ public class kelondroDynTree {
private int maxcountBuffer = 1000, maxsizeBuffer = 100;
//private long maxageCache = 60000, cycletimeCache = 10000;
private long maxageBuffer = 60000, cycletimeBuffer = 10000;
private long preloadTime = 0;
// data structures for the cache and buffer
private HashMap<String, treeBuffer> buffer;
private HashMap<String, treeCache> cache;
private long cycleBuffer;
public kelondroDynTree(File file, long preloadTime, int keylength, int nodesize, kelondroRow rowdef, char fillChar, boolean resetOnFail) {
public kelondroDynTree(File file, int keylength, int nodesize, kelondroRow rowdef, char fillChar, boolean resetOnFail) {
// creates or opens a DynTree
this.file = file;
this.preloadTime = preloadTime;
this.rowdef = rowdef;
this.buffer = new HashMap<String, treeBuffer>();
this.cache = new HashMap<String, treeCache>();
//this.cycleCache = Long.MIN_VALUE;
this.cycleBuffer = Long.MIN_VALUE;
this.table = new kelondroDyn(file, true, true, preloadTime, keylength, nodesize, fillChar, rowdef.objectOrder, true, false, resetOnFail);
this.table = new kelondroDyn(file, true, true, keylength, nodesize, fillChar, rowdef.objectOrder, true, false, resetOnFail);
this.treeRAHandles = new HashMap<String, kelondroRA>();
}
@ -106,7 +104,7 @@ public class kelondroDynTree {
kelondroRA ra = table.getRA(key); // works always, even with no-existing entry
treeRAHandles.put(key, ra);
try {
return new kelondroTree(ra, this.file.getCanonicalPath() + "#" + key, true, preloadTime, rowdef, false);
return new kelondroTree(ra, this.file.getCanonicalPath() + "#" + key, true, 0, rowdef, false);
} catch (RuntimeException e) {
throw new IOException(e.getMessage());
}
@ -116,7 +114,7 @@ public class kelondroDynTree {
if (table.existsDyn(key)) {
kelondroRA ra = table.getRA(key);
treeRAHandles.put(key, ra);
return new kelondroTree(ra, this.file.getCanonicalPath() + "#" + key, true, preloadTime);
return new kelondroTree(ra, this.file.getCanonicalPath() + "#" + key, true, 0);
}
return null;
}
@ -297,10 +295,10 @@ public class kelondroDynTree {
System.out.println("start");
File file = new File("D:\\bin\\testDyn.db");
if (file.exists()) {
kelondroDynTree dt = new kelondroDynTree(file, 0, 16, 512, new kelondroRow("byte[] a-10, byte[] b-20, byte[] c-30", kelondroNaturalOrder.naturalOrder, 0), '_', true);
kelondroDynTree dt = new kelondroDynTree(file, 16, 512, new kelondroRow("byte[] a-10, byte[] b-20, byte[] c-30", kelondroNaturalOrder.naturalOrder, 0), '_', true);
System.out.println("opened: table keylength=" + dt.table.row().width(0) + ", sectorsize=" + dt.table.row().width(1) + ", " + dt.table.sizeDyn() + " entries.");
} else {
kelondroDynTree dt = new kelondroDynTree(file, 0, 16, 512, new kelondroRow("byte[] a-10, byte[] b-20, byte[] c-30", kelondroNaturalOrder.naturalOrder, 0), '_', true);
kelondroDynTree dt = new kelondroDynTree(file, 16, 512, new kelondroRow("byte[] a-10, byte[] b-20, byte[] c-30", kelondroNaturalOrder.naturalOrder, 0), '_', true);
String name;
kelondroTree t;
kelondroRow.Entry line;

8
source/de/anomic/kelondro/kelondroFlexTable.java
Unescape View File

@ -47,7 +47,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
protected kelondroBytesIntMap index;
private boolean RAMIndex;
public kelondroFlexTable(File path, String tablename, long preloadTime, kelondroRow rowdef, int minimumSpace, boolean resetOnFail) {
public kelondroFlexTable(File path, String tablename, kelondroRow rowdef, int minimumSpace, boolean resetOnFail) {
// the buffersize applies to a possible load of the ram-index
// the minimumSpace is a initial allocation space for the index; names the number of index slots
// if the ram is not sufficient, a tree file is generated
@ -94,13 +94,13 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
if (indexfile.exists()) {
// use existing index file
System.out.println("*** Using File index " + indexfile);
ki = new kelondroCache(kelondroTree.open(indexfile, true, preloadTime, treeIndexRow(rowdef.width(0), rowdef.objectOrder), 2, 80));
ki = new kelondroCache(kelondroTree.open(indexfile, true, 0, treeIndexRow(rowdef.width(0), rowdef.objectOrder), 2, 80));
RAMIndex = false;
} else {
// generate new index file
System.out.println("*** Generating File index for " + size() + " entries from " + indexfile);
System.out.println("*** Cause: too less RAM (" + serverMemory.available() + " Bytes) configured. Assign at least " + (neededRAM / 1024 / 1024) + " MB more RAM to enable a RAM index.");
ki = initializeTreeIndex(indexfile, preloadTime, rowdef.objectOrder);
ki = initializeTreeIndex(indexfile, 0, rowdef.objectOrder);
System.out.println(" -done-");
System.out.println(ki.size() + " entries indexed from " + super.col[0].size() + " keys.");
@ -490,7 +490,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
String name = args[1];
kelondroRow row = new kelondroRow("Cardinal key-4 {b256}, byte[] x-64", kelondroNaturalOrder.naturalOrder, 0);
try {
kelondroFlexTable t = new kelondroFlexTable(f, name, 0, row, 0, true);
kelondroFlexTable t = new kelondroFlexTable(f, name, row, 0, true);
kelondroRow.Entry entry = row.newEntry();
entry.setCol(0, System.currentTimeMillis());
entry.setCol(1, "dummy".getBytes());

10
source/de/anomic/kelondro/kelondroMapTable.java
Unescape View File

@ -69,22 +69,16 @@ public class kelondroMapTable {
declareMaps(tablename, keysize, objectOrder, nodesize, cacheslots, null, null, null, fillChar, resetOnFail);
}
public void declareMaps(
String tablename, int keysize, kelondroByteOrder objectOrder, int nodesize, int cacheslots,
String[] sortfields, String[] longaccfields, String[] doubleaccfields, char fillChar, boolean resetOnFail) {
declareMaps(tablename, keysize, objectOrder, nodesize, cacheslots, sortfields, longaccfields, doubleaccfields, fillChar, 0, resetOnFail);
}
public void declareMaps(
String tablename, int keysize, kelondroByteOrder objectOrder, int nodesize, int cacheslots,
String[] sortfields, String[] longaccfields, String[] doubleaccfields, char fillChar,
long preloadTime, boolean resetOnFail) {
boolean resetOnFail) {
if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared twice.");
if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared already in other context.");
File tablefile = new File(tablesPath, "table." + tablename + ".mdb");
kelondroDyn dyn;
if (!(tablefile.exists())) tablefile.getParentFile().mkdirs();
dyn = new kelondroDyn(tablefile, true, true, preloadTime, keysize, nodesize, fillChar, objectOrder, true, false, resetOnFail);
dyn = new kelondroDyn(tablefile, true, true, keysize, nodesize, fillChar, objectOrder, true, false, resetOnFail);
kelondroMapObjects map = new kelondroMapObjects(dyn, cacheslots, sortfields, longaccfields, doubleaccfields, null, null);
mTables.put(tablename, map);
}

16
source/de/anomic/kelondro/kelondroSplitTable.java
Unescape View File

@ -54,15 +54,15 @@ public class kelondroSplitTable implements kelondroIndex {
private String tablename;
private kelondroOrder<kelondroRow.Entry> entryOrder;
public kelondroSplitTable(File path, String tablename, long preloadTime, kelondroRow rowdef, boolean resetOnFail) {
public kelondroSplitTable(File path, String tablename, kelondroRow rowdef, boolean resetOnFail) {
this.path = path;
this.tablename = tablename;
this.rowdef = rowdef;
this.entryOrder = new kelondroRow.EntryComparator(rowdef.objectOrder);
init(preloadTime, resetOnFail);
init(resetOnFail);
}
public void init(long preloadTime, boolean resetOnFail) {
public void init(boolean resetOnFail) {
// initialized tables map
this.tables = new HashMap<String, kelondroIndex>();
@ -117,7 +117,7 @@ public class kelondroSplitTable implements kelondroIndex {
f = new File(path, maxf);
if (f.isDirectory()) {
// this is a kelonodroFlex table
table = new kelondroCache(new kelondroFlexTable(path, maxf, preloadTime, rowdef, 0, resetOnFail));
table = new kelondroCache(new kelondroFlexTable(path, maxf, rowdef, 0, resetOnFail));
} else {
table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize, 0);
}
@ -134,7 +134,7 @@ public class kelondroSplitTable implements kelondroIndex {
if (f.isDirectory()) kelondroFlexTable.delete(path, l[i]); else f.delete();
}
}
init(-1, true);
init(true);
}
public String filename() {
@ -226,7 +226,7 @@ public class kelondroSplitTable implements kelondroIndex {
if (f.exists()) {
if (f.isDirectory()) {
// open a flex table
table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef, 0, true);
table = new kelondroFlexTable(path, tablename + "." + suffix, rowdef, 0, true);
} else {
// open a eco table
table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0);
@ -238,7 +238,7 @@ public class kelondroSplitTable implements kelondroIndex {
table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0);
} else {
// use the flex table
table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef, 0, true);
table = new kelondroFlexTable(path, tablename + "." + suffix, rowdef, 0, true);
}
}
tables.put(suffix, table);
@ -276,7 +276,7 @@ public class kelondroSplitTable implements kelondroIndex {
table = new kelondroEcoTable(new File(path, tablename + "." + suffix), rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0);
} else {
// use the flex table
table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef, 0, true);
table = new kelondroFlexTable(path, tablename + "." + suffix, rowdef, 0, true);
}
tables.put(suffix, table);
}

2
source/de/anomic/plasma/dbImport/plasmaCrawlNURLImporter.java
Unescape View File

@ -106,7 +106,7 @@ public class plasmaCrawlNURLImporter extends AbstractImporter implements dbImpor
// init profile DB
this.log.logInfo("Initializing the source profileDB");
this.importProfileDB = new plasmaCrawlProfile(profileDbFile, 300);
this.importProfileDB = new plasmaCrawlProfile(profileDbFile);
}
@SuppressWarnings("unchecked")

4
source/de/anomic/plasma/dbImport/plasmaDbImporter.java
Unescape View File

@ -85,8 +85,6 @@ public class plasmaDbImporter extends AbstractImporter implements dbImporter {
this.cacheSize = Integer.valueOf((String)initParams.get("cacheSize")).intValue();
if (this.cacheSize < 2*1024*1024) this.cacheSize = 8*1024*1024;
this.preloadTime = Long.valueOf((String)initParams.get("preloadTime")).longValue();
// configure import DB
String errorMsg = null;
if (!this.importPrimaryPath.exists()) errorMsg = "Primary Import directory does not exist.";
@ -107,7 +105,7 @@ public class plasmaDbImporter extends AbstractImporter implements dbImporter {
}
this.log.logFine("Initializing source word index db.");
this.importWordIndex = new plasmaWordIndex(this.importPrimaryPath, this.importSecondaryPath, preloadTime / 2, this.log);
this.importWordIndex = new plasmaWordIndex(this.importPrimaryPath, this.importSecondaryPath, this.log);
this.importStartSize = this.importWordIndex.size();
}

6
source/de/anomic/plasma/plasmaCrawlLURL.java
Unescape View File

@ -97,10 +97,10 @@ public final class plasmaCrawlLURL {
// the class object
private kelondroIndex urlIndexFile;
public plasmaCrawlLURL(File indexPath, long preloadTime) {
public plasmaCrawlLURL(File indexPath) {
super();
urlIndexFile = new kelondroSplitTable(new File(indexPath, "PUBLIC/TEXT"), "urls", preloadTime, indexURLEntry.rowdef, false);
urlIndexFile = new kelondroSplitTable(new File(indexPath, "PUBLIC/TEXT"), "urls", indexURLEntry.rowdef, false);
// init result stacks
externResultStack = new LinkedList<String>();
@ -676,7 +676,7 @@ public final class plasmaCrawlLURL {
} catch (MalformedURLException e) {}
if (args[0].equals("-l")) try {
// arg 1 is path to URLCache
final plasmaCrawlLURL urls = new plasmaCrawlLURL(new File(args[2]), 0);
final plasmaCrawlLURL urls = new plasmaCrawlLURL(new File(args[2]));
final Iterator<indexURLEntry> enu = urls.entries(true, null);
while (enu.hasNext()) {
System.out.println(enu.next().toString());

8
source/de/anomic/plasma/plasmaCrawlProfile.java
Unescape View File

@ -64,13 +64,11 @@ public class plasmaCrawlProfile {
private kelondroMapObjects profileTable;
private File profileTableFile;
private long preloadTime;
public plasmaCrawlProfile(File file, long preloadTime) {
public plasmaCrawlProfile(File file) {
this.profileTableFile = file;
this.preloadTime = preloadTime;
profileTableFile.getParentFile().mkdirs();
kelondroDyn dyn = new kelondroDyn(profileTableFile, true, true, preloadTime, yacySeedDB.commonHashLength, 2000, '#', kelondroNaturalOrder.naturalOrder, false, false, true);
kelondroDyn dyn = new kelondroDyn(profileTableFile, true, true, yacySeedDB.commonHashLength, 2000, '#', kelondroNaturalOrder.naturalOrder, false, false, true);
profileTable = new kelondroMapObjects(dyn, 500);
}
@ -79,7 +77,7 @@ public class plasmaCrawlProfile {
if (profileTable != null) profileTable.close();
if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database");
profileTableFile.getParentFile().mkdirs();
kelondroDyn dyn = new kelondroDyn(profileTableFile, true, true, preloadTime, yacySeedDB.commonHashLength, 2000, '#', kelondroNaturalOrder.naturalOrder, false, false, true);
kelondroDyn dyn = new kelondroDyn(profileTableFile, true, true, yacySeedDB.commonHashLength, 2000, '#', kelondroNaturalOrder.naturalOrder, false, false, true);
profileTable = new kelondroMapObjects(dyn, 500);
}

8
source/de/anomic/plasma/plasmaCrawlRobotsTxt.java
Unescape View File

@ -65,13 +65,11 @@ public class plasmaCrawlRobotsTxt {
kelondroMapObjects robotsTable;
private final File robotsTableFile;
private long preloadTime;
public plasmaCrawlRobotsTxt(File robotsTableFile, long preloadTime) {
public plasmaCrawlRobotsTxt(File robotsTableFile) {
this.robotsTableFile = robotsTableFile;
this.preloadTime = preloadTime;
robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMapObjects(new kelondroDyn(robotsTableFile, true, true, preloadTime, 256, 512, '_', kelondroNaturalOrder.naturalOrder, false, false, true), 100);
robotsTable = new kelondroMapObjects(new kelondroDyn(robotsTableFile, true, true, 256, 512, '_', kelondroNaturalOrder.naturalOrder, false, false, true), 100);
}
private void resetDatabase() {
@ -79,7 +77,7 @@ public class plasmaCrawlRobotsTxt {
if (robotsTable != null) robotsTable.close();
if (!(robotsTableFile.delete())) throw new RuntimeException("cannot delete robots.txt database");
robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMapObjects(new kelondroDyn(robotsTableFile, true, true, preloadTime, 256, 512, '_', kelondroNaturalOrder.naturalOrder, false, false, true), 100);
robotsTable = new kelondroMapObjects(new kelondroDyn(robotsTableFile, true, true, 256, 512, '_', kelondroNaturalOrder.naturalOrder, false, false, true), 100);
}
public void close() {

6
source/de/anomic/plasma/plasmaCrawlStacker.java
Unescape View File

@ -85,7 +85,6 @@ public final class plasmaCrawlStacker extends Thread {
private final LinkedList<String> urlEntryHashCache;
private kelondroIndex urlEntryCache;
private File cacheStacksPath;
private long preloadTime;
private int dbtype;
private boolean prequeue;
private long dnsHit, dnsMiss;
@ -95,7 +94,7 @@ public final class plasmaCrawlStacker extends Thread {
// objects for the prefetch task
private ArrayList<String> dnsfetchHosts = new ArrayList<String>();
public plasmaCrawlStacker(plasmaSwitchboard sb, File dbPath, long preloadTime, int dbtype, boolean prequeue) {
public plasmaCrawlStacker(plasmaSwitchboard sb, File dbPath, int dbtype, boolean prequeue) {
this.sb = sb;
this.prequeue = prequeue;
this.dnsHit = 0;
@ -107,7 +106,6 @@ public final class plasmaCrawlStacker extends Thread {
// create a stack for newly entered entries
this.cacheStacksPath = dbPath;
this.preloadTime = preloadTime;
this.dbtype = dbtype;
openDB();
@ -321,7 +319,7 @@ public final class plasmaCrawlStacker extends Thread {
if (this.dbtype == QUEUE_DB_TYPE_TREE) {
File cacheFile = new File(cacheStacksPath, stackfile);
cacheFile.getParentFile().mkdirs();
this.urlEntryCache = new kelondroCache(kelondroTree.open(cacheFile, true, preloadTime, plasmaCrawlEntry.rowdef));
this.urlEntryCache = new kelondroCache(kelondroTree.open(cacheFile, true, 0, plasmaCrawlEntry.rowdef));
}
}

10
source/de/anomic/plasma/plasmaHTCache.java
Unescape View File

@ -229,7 +229,7 @@ public final class plasmaHTCache {
return doctype;
}
public static void init(File htCachePath, long CacheSizeMax, long preloadTime, String layout, boolean migration) {
public static void init(File htCachePath, long CacheSizeMax, String layout, boolean migration) {
cachePath = htCachePath;
cacheLayout = layout;
@ -287,7 +287,7 @@ public final class plasmaHTCache {
}
// open the response header database
openResponseHeaderDB(preloadTime);
openResponseHeaderDB();
// start the cache startup thread
// this will collect information about the current cache size and elements
@ -302,13 +302,13 @@ public final class plasmaHTCache {
if (responseHeaderDB != null) responseHeaderDB.close();
File dbfile = new File(cachePath, DB_NAME);
if (dbfile.exists()) dbfile.delete();
openResponseHeaderDB(0);
openResponseHeaderDB();
}
private static void openResponseHeaderDB(long preloadTime) {
private static void openResponseHeaderDB() {
// open the response header database
File dbfile = new File(cachePath, DB_NAME);
responseHeaderDB = new kelondroMapObjects(new kelondroDyn(dbfile, true, true, preloadTime, yacySeedDB.commonHashLength, 150, '#', kelondroBase64Order.enhancedCoder, false, false, true), 500);
responseHeaderDB = new kelondroMapObjects(new kelondroDyn(dbfile, true, true, yacySeedDB.commonHashLength, 150, '#', kelondroBase64Order.enhancedCoder, false, false, true), 500);
}
private static void deleteOldHTCache(File directory) {

8
source/de/anomic/plasma/plasmaRankingCRProcess.java
Unescape View File

@ -260,8 +260,8 @@ public class plasmaRankingCRProcess {
kelondroCollectionIndex newseq = null;
if (newdb) {
File path = to_file.getParentFile(); // path to storage place
newacc = new kelondroFlexTable(path, CRG_accname, -1, CRG_accrow, 0, false);
newseq = new kelondroCollectionIndex(path, CRG_seqname, 12, kelondroBase64Order.enhancedCoder, -1, 2, 9, CRG_colrow);
newacc = new kelondroFlexTable(path, CRG_accname, CRG_accrow, 0, false);
newseq = new kelondroCollectionIndex(path, CRG_seqname, 12, kelondroBase64Order.enhancedCoder, 2, 9, CRG_colrow);
} else {
if (!(to_file.exists())) {
acc = new kelondroAttrSeq("Global Ranking Accumulator File",
@ -390,8 +390,8 @@ public class plasmaRankingCRProcess {
public static int genrcix(File cr_path_in, File rci_path_out) throws IOException {
//kelondroFlexTable acc = new kelondroFlexTable(cr_path_in, CRG_accname, kelondroBase64Order.enhancedCoder, 128 * 1024 * 1024, -1, CRG_accrow, true);
kelondroCollectionIndex seq = new kelondroCollectionIndex(cr_path_in, CRG_seqname, 12, kelondroBase64Order.enhancedCoder, -1, 2, 9, CRG_colrow);
kelondroCollectionIndex rci = new kelondroCollectionIndex(rci_path_out, RCI_colname, 6, kelondroBase64Order.enhancedCoder, -1, 2, 9, RCI_coli);
kelondroCollectionIndex seq = new kelondroCollectionIndex(cr_path_in, CRG_seqname, 12, kelondroBase64Order.enhancedCoder, 2, 9, CRG_colrow);
kelondroCollectionIndex rci = new kelondroCollectionIndex(rci_path_out, RCI_colname, 6, kelondroBase64Order.enhancedCoder, 2, 9, RCI_coli);
// loop over all referees
int count = 0;

81
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -947,12 +947,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
this.acceptGlobalURLs = "global.any".indexOf(getConfig("network.unit.domain", "global")) >= 0;
this.acceptLocalURLs = "local.any".indexOf(getConfig("network.unit.domain", "global")) >= 0;
// start yacy core
log.logConfig("Starting YaCy Protocol Core");
this.yc = new yacyCore(this);
serverInstantThread.oneTimeJob(yacyCore.peerActions, "loadSeedLists", yacyCore.log, 0);
long startedSeedListAquisition = System.currentTimeMillis();
// load values from configs
this.plasmaPath = getConfigPath(DBPATH, DBPATH_DEFAULT);
this.log.logConfig("Plasma DB Path: " + this.plasmaPath.toString());
@ -970,6 +964,16 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
this.workPath = getConfigPath(WORK_PATH, WORK_PATH_DEFAULT);
this.log.logConfig("Work Path: " + this.workPath.toString());
// start indexing management
log.logConfig("Starting Indexing Management");
wordIndex = new plasmaWordIndex(indexPrimaryPath, indexSecondaryPath, log);
// start yacy core
log.logConfig("Starting YaCy Protocol Core");
this.yc = new yacyCore(this);
serverInstantThread.oneTimeJob(yacyCore.peerActions, "loadSeedLists", yacyCore.log, 0);
long startedSeedListAquisition = System.currentTimeMillis();
// set up local robots.txt
this.robotstxtConfig = httpdRobotsTxtConfig.init(this);
@ -1040,41 +1044,17 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
if (YBRPath.exists()) {
plasmaSearchRankingProcess.loadYBR(YBRPath, 15);
}
// read memory amount
long ramLURL_time = getConfigLong(RAM_CACHE_LURL_TIME, 1000);
long ramNURL_time = getConfigLong(RAM_CACHE_NURL_TIME, 1000);
long ramEURL_time = getConfigLong(RAM_CACHE_EURL_TIME, 1000);
long ramRWI_time = getConfigLong(RAM_CACHE_RWI_TIME, 1000);
long ramHTTP_time = getConfigLong(RAM_CACHE_HTTP_TIME, 1000);
long ramMessage_time = getConfigLong(RAM_CACHE_MESSAGE_TIME, 1000);
long ramRobots_time = getConfigLong(RAM_CACHE_ROBOTS_TIME, 1000);
long ramProfiles_time= getConfigLong(RAM_CACHE_PROFILES_TIME, 1000);
long ramPreNURL_time = getConfigLong(RAM_CACHE_PRE_NURL_TIME, 1000);
long ramWiki_time = getConfigLong(RAM_CACHE_WIKI_TIME, 1000);
long ramBlog_time = getConfigLong(RAM_CACHE_BLOG_TIME, 1000);
this.log.logConfig("LURL preloadTime = " + ramLURL_time);
this.log.logConfig("NURL preloadTime = " + ramNURL_time);
this.log.logConfig("EURL preloadTime = " + ramEURL_time);
this.log.logConfig("RWI preloadTime = " + ramRWI_time);
this.log.logConfig("HTTP preloadTime = " + ramHTTP_time);
this.log.logConfig("Message preloadTime = " + ramMessage_time);
this.log.logConfig("Wiki preloadTime = " + ramWiki_time);
this.log.logConfig("Blog preloadTime = " + ramBlog_time);
this.log.logConfig("Robots preloadTime = " + ramRobots_time);
this.log.logConfig("Profiles preloadTime = " + ramProfiles_time);
this.log.logConfig("PreNURL preloadTime = " + ramPreNURL_time);
// make crawl profiles database and default profiles
this.log.logConfig("Initializing Crawl Profiles");
File profilesActiveFile = new File(this.plasmaPath, DBFILE_ACTIVE_CRAWL_PROFILES);
this.profilesActiveCrawls = new plasmaCrawlProfile(profilesActiveFile, ramProfiles_time);
this.profilesActiveCrawls = new plasmaCrawlProfile(profilesActiveFile);
initActiveCrawlProfiles();
log.logConfig("Loaded active crawl profiles from file " + profilesActiveFile.getName() +
", " + this.profilesActiveCrawls.size() + " entries" +
", " + ppRamString(profilesActiveFile.length()/1024));
File profilesPassiveFile = new File(this.plasmaPath, DBFILE_PASSIVE_CRAWL_PROFILES);
this.profilesPassiveCrawls = new plasmaCrawlProfile(profilesPassiveFile, ramProfiles_time);
this.profilesPassiveCrawls = new plasmaCrawlProfile(profilesPassiveFile);
log.logConfig("Loaded passive crawl profiles from file " + profilesPassiveFile.getName() +
", " + this.profilesPassiveCrawls.size() + " entries" +
", " + ppRamString(profilesPassiveFile.length()/1024));
@ -1082,7 +1062,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// loading the robots.txt db
this.log.logConfig("Initializing robots.txt DB");
File robotsDBFile = new File(this.plasmaPath, DBFILE_CRAWL_ROBOTS);
robots = new plasmaCrawlRobotsTxt(robotsDBFile, ramRobots_time);
robots = new plasmaCrawlRobotsTxt(robotsDBFile);
this.log.logConfig("Loaded robots.txt DB from file " + robotsDBFile.getName() +
", " + robots.size() + " entries" +
", " + ppRamString(robotsDBFile.length()/1024));
@ -1096,7 +1076,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
long maxCacheSize = 1024 * 1024 * Long.parseLong(getConfig(PROXY_CACHE_SIZE, "2")); // this is megabyte
String cacheLayout = getConfig(PROXY_CACHE_LAYOUT, PROXY_CACHE_LAYOUT_TREE);
boolean cacheMigration = getConfigBool(PROXY_CACHE_MIGRATION, true);
plasmaHTCache.init(htCachePath, maxCacheSize, ramHTTP_time, cacheLayout, cacheMigration);
plasmaHTCache.init(htCachePath, maxCacheSize, cacheLayout, cacheMigration);
// create the release download directory
releasePath = getConfigPath(RELEASE_PATH, RELEASE_PATH_DEFAULT);
@ -1104,18 +1084,18 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
this.log.logInfo("RELEASE Path = " + releasePath.getAbsolutePath());
// starting message board
initMessages(ramMessage_time);
initMessages();
// starting wiki
initWiki(ramWiki_time);
initWiki();
//starting blog
initBlog(ramBlog_time);
initBlog();
// Init User DB
this.log.logConfig("Loading User DB");
File userDbFile = new File(getRootPath(), DBFILE_USER);
this.userDB = new userDB(userDbFile, 2000);
this.userDB = new userDB(userDbFile);
this.log.logConfig("Loaded User DB from file " + userDbFile.getName() +
", " + this.userDB.size() + " entries" +
", " + ppRamString(userDbFile.length()/1024));
@ -1123,10 +1103,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
//Init bookmarks DB
initBookmarks();
// start indexing management
log.logConfig("Starting Indexing Management");
wordIndex = new plasmaWordIndex(indexPrimaryPath, indexSecondaryPath, ramRWI_time, log);
// set a high maximum cache size to current size; this is adopted later automatically
int wordCacheMaxCount = Math.max((int) getConfigLong(WORDCACHE_INIT_COUNT, 30000),
(int) getConfigLong(WORDCACHE_MAX_COUNT, 20000));
@ -1285,7 +1261,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
}
// initializing the stackCrawlThread
this.crawlStacker = new plasmaCrawlStacker(this, this.plasmaPath, ramPreNURL_time, (int) getConfigLong("tableTypeForPreNURL", 0), (((int) getConfigLong("tableTypeForPreNURL", 0) == 0) && (getConfigLong(CRAWLSTACK_BUSYSLEEP, 0) <= 100)));
this.crawlStacker = new plasmaCrawlStacker(this, this.plasmaPath, (int) getConfigLong("tableTypeForPreNURL", 0), (((int) getConfigLong("tableTypeForPreNURL", 0) == 0) && (getConfigLong(CRAWLSTACK_BUSYSLEEP, 0) <= 100)));
//this.sbStackCrawlThread = new plasmaStackCrawlThread(this,this.plasmaPath,ramPreNURL);
//this.sbStackCrawlThread.start();
@ -1359,34 +1335,34 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
log.logConfig("Finished Switchboard Initialization");
}
public void initMessages(long ramMessage_time) {
public void initMessages() {
this.log.logConfig("Starting Message Board");
File messageDbFile = new File(workPath, DBFILE_MESSAGE);
this.messageDB = new messageBoard(messageDbFile, ramMessage_time);
this.messageDB = new messageBoard(messageDbFile);
this.log.logConfig("Loaded Message Board DB from file " + messageDbFile.getName() +
", " + this.messageDB.size() + " entries" +
", " + ppRamString(messageDbFile.length()/1024));
}
public void initWiki(long ramWiki_time) {
public void initWiki() {
this.log.logConfig("Starting Wiki Board");
File wikiDbFile = new File(workPath, DBFILE_WIKI);
this.wikiDB = new wikiBoard(wikiDbFile, new File(workPath, DBFILE_WIKI_BKP), ramWiki_time);
this.wikiDB = new wikiBoard(wikiDbFile, new File(workPath, DBFILE_WIKI_BKP));
this.log.logConfig("Loaded Wiki Board DB from file " + wikiDbFile.getName() +
", " + this.wikiDB.size() + " entries" +
", " + ppRamString(wikiDbFile.length()/1024));
}
public void initBlog(long ramBlog_time) {
public void initBlog() {
this.log.logConfig("Starting Blog");
File blogDbFile = new File(workPath, DBFILE_BLOG);
this.blogDB = new blogBoard(blogDbFile, ramBlog_time);
this.blogDB = new blogBoard(blogDbFile);
this.log.logConfig("Loaded Blog DB from file " + blogDbFile.getName() +
", " + this.blogDB.size() + " entries" +
", " + ppRamString(blogDbFile.length()/1024));
File blogCommentDbFile = new File(workPath, DBFILE_BLOGCOMMENTS);
this.blogCommentDB = new blogBoardComments(blogCommentDbFile, ramBlog_time);
this.blogCommentDB = new blogBoardComments(blogCommentDbFile);
this.log.logConfig("Loaded Blog-Comment DB from file " + blogCommentDbFile.getName() +
", " + this.blogCommentDB.size() + " entries" +
", " + ppRamString(blogCommentDbFile.length()/1024));
@ -1397,7 +1373,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
File bookmarksFile = new File(workPath, DBFILE_BOOKMARKS);
File tagsFile = new File(workPath, DBFILE_BOOKMARKS_TAGS);
File datesFile = new File(workPath, DBFILE_BOOKMARKS_DATES);
this.bookmarksDB = new bookmarksDB(bookmarksFile, tagsFile, datesFile, 2000);
this.bookmarksDB = new bookmarksDB(bookmarksFile, tagsFile, datesFile);
this.log.logConfig("Loaded Bookmarks DB from files "+ bookmarksFile.getName()+ ", "+tagsFile.getName());
this.log.logConfig(this.bookmarksDB.tagsSize()+" Tag, "+this.bookmarksDB.bookmarksSize()+" Bookmarks");
}
@ -1575,8 +1551,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
private void resetProfiles() {
final File pdb = new File(plasmaPath, DBFILE_ACTIVE_CRAWL_PROFILES);
if (pdb.exists()) pdb.delete();
long ramProfiles_time = getConfigLong(RAM_CACHE_PROFILES_TIME, 1000);
profilesActiveCrawls = new plasmaCrawlProfile(pdb, ramProfiles_time);
profilesActiveCrawls = new plasmaCrawlProfile(pdb);
initActiveCrawlProfiles();
}

4
source/de/anomic/plasma/plasmaWordConnotation.java
Unescape View File

@ -59,8 +59,8 @@ public class plasmaWordConnotation {
private static final int nodesize = 4048;
private kelondroDynTree refDB;
public plasmaWordConnotation(File refDBfile, long preloadTime, char fillChar) {
refDB = new kelondroDynTree(refDBfile, preloadTime, wordlength, nodesize, new kelondroRow("byte[] word-" + wordlength + ", Cardinal count-" + countlength, kelondroNaturalOrder.naturalOrder, 0), fillChar, true);
public plasmaWordConnotation(File refDBfile, char fillChar) {
refDB = new kelondroDynTree(refDBfile, wordlength, nodesize, new kelondroRow("byte[] word-" + wordlength + ", Cardinal count-" + countlength, kelondroNaturalOrder.naturalOrder, 0), fillChar, true);
}
private void addSingleRef(String word, String reference) throws IOException {

6
source/de/anomic/plasma/plasmaWordIndex.java
Unescape View File

@ -74,7 +74,7 @@ public final class plasmaWordIndex implements indexRI {
private int flushsize;
public final plasmaCrawlLURL loadedURL;
public plasmaWordIndex(File indexPrimaryRoot, File indexSecondaryRoot, long preloadTime, serverLog log) {
public plasmaWordIndex(File indexPrimaryRoot, File indexSecondaryRoot, serverLog log) {
File textindexcache = new File(indexPrimaryRoot, "PUBLIC/TEXT/RICACHE");
if (!(textindexcache.exists())) textindexcache.mkdirs();
this.dhtOutCache = new indexRAMRI(textindexcache, indexRWIRowEntry.urlEntryRow, wCacheMaxChunk, wCacheMaxAge, "dump1.array", log);
@ -83,10 +83,10 @@ public final class plasmaWordIndex implements indexRI {
// create collections storage path
File textindexcollections = new File(indexPrimaryRoot, "PUBLIC/TEXT/RICOLLECTION");
if (!(textindexcollections.exists())) textindexcollections.mkdirs();
this.collections = new indexCollectionRI(textindexcollections, "collection", preloadTime, maxCollectionPartition, indexRWIRowEntry.urlEntryRow);
this.collections = new indexCollectionRI(textindexcollections, "collection", maxCollectionPartition, indexRWIRowEntry.urlEntryRow);
// create LURL-db
loadedURL = new plasmaCrawlLURL(indexSecondaryRoot, preloadTime);
loadedURL = new plasmaCrawlLURL(indexSecondaryRoot);
// performance settings
busyCacheFlush = false;

4
source/de/anomic/yacy/yacyCore.java
Unescape View File

@ -129,13 +129,11 @@ public class yacyCore {
if (!yacyDBPath.exists()) { yacyDBPath.mkdir(); }
// create or init seed cache
long memDHT_time = Long.parseLong(switchboard.getConfig("ramCacheDHT_time", "1000"));
seedDB = new yacySeedDB(
sb,
new File(yacyDBPath, "seed2.new.db"),
new File(yacyDBPath, "seed2.old.db"),
new File(yacyDBPath, "seed2.pot.db"),
memDHT_time);
new File(yacyDBPath, "seed2.pot.db"));
// create or init news database
newsPool = new yacyNewsPool(yacyDBPath);

9
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -100,7 +100,6 @@ public final class yacySeedDB {
protected File seedActiveDBFile, seedPassiveDBFile, seedPotentialDBFile;
protected kelondroMapObjects seedActiveDB, seedPassiveDB, seedPotentialDB;
private long preloadTime;
private final plasmaSwitchboard sb;
private yacySeed mySeed; // my own seed
@ -111,15 +110,13 @@ public final class yacySeedDB {
public yacySeedDB(plasmaSwitchboard sb,
File seedActiveDBFile,
File seedPassiveDBFile,
File seedPotentialDBFile,
long preloadTime) {
File seedPotentialDBFile) {
this.seedActiveDBFile = seedActiveDBFile;
this.seedPassiveDBFile = seedPassiveDBFile;
this.seedPotentialDBFile = seedPotentialDBFile;
this.mySeed = null; // my own seed
this.sb = sb;
this.preloadTime = preloadTime;
// set up seed database
seedActiveDB = openSeedTable(seedActiveDBFile);
@ -224,11 +221,11 @@ public final class yacySeedDB {
initializeHandlerMethod = null;
}
try {
return new kelondroMapObjects(new kelondroDyn(seedDBFile, true, true, preloadTime / 3, commonHashLength, 480, '#', kelondroBase64Order.enhancedCoder, usetree, false, true), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
return new kelondroMapObjects(new kelondroDyn(seedDBFile, true, true, commonHashLength, 480, '#', kelondroBase64Order.enhancedCoder, usetree, false, true), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
} catch (Exception e) {
// try again
kelondroDyn.delete(seedDBFile, usetree);
return new kelondroMapObjects(new kelondroDyn(seedDBFile, true, true, preloadTime / 3, commonHashLength, 480, '#', kelondroBase64Order.enhancedCoder, usetree, false, true), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
return new kelondroMapObjects(new kelondroDyn(seedDBFile, true, true, commonHashLength, 480, '#', kelondroBase64Order.enhancedCoder, usetree, false, true), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
}
}

4
source/migration.java
Unescape View File

@ -163,7 +163,7 @@ public class migration {
file.delete();
} catch (IOException e) {}
}
sb.initWiki(sb.getConfigLong("ramCacheWiki_time", 1000));
sb.initWiki();
}
@ -176,7 +176,7 @@ public class migration {
serverFileUtils.copy(file, file2);
file.delete();
} catch (IOException e) {}
sb.initMessages(sb.getConfigLong("ramCacheMessage_time", 1000));
sb.initMessages();
}
}

12
source/yacy.java
Unescape View File

@ -598,16 +598,16 @@ public final class yacy {
log.logInfo("STARTING URL CLEANUP");
// db containing all currently loades urls
plasmaCrawlLURL currentUrlDB = new plasmaCrawlLURL(indexSecondaryRoot, 10000);
plasmaCrawlLURL currentUrlDB = new plasmaCrawlLURL(indexSecondaryRoot);
// db used to hold all neede urls
plasmaCrawlLURL minimizedUrlDB = new plasmaCrawlLURL(indexRoot2, 10000);
plasmaCrawlLURL minimizedUrlDB = new plasmaCrawlLURL(indexRoot2);
Runtime rt = Runtime.getRuntime();
int cacheMem = (int)(rt.maxMemory() - rt.totalMemory());
if (cacheMem < 2048000) throw new OutOfMemoryError("Not enough memory available to start clean up.");
plasmaWordIndex wordIndex = new plasmaWordIndex(indexPrimaryRoot, indexSecondaryRoot, 10000, log);
plasmaWordIndex wordIndex = new plasmaWordIndex(indexPrimaryRoot, indexSecondaryRoot, log);
Iterator<indexContainer> indexContainerIterator = wordIndex.wordContainers("AAAAAAAAAAAA", false, false);
long urlCounter = 0, wordCounter = 0;
@ -781,7 +781,7 @@ public final class yacy {
File root = homePath;
File indexroot = new File(root, "DATA/INDEX");
try {serverLog.configureLogging(homePath, new File(homePath, "DATA/LOG/yacy.logging"));} catch (Exception e) {}
plasmaCrawlLURL currentUrlDB = new plasmaCrawlLURL(indexroot, 10000);
plasmaCrawlLURL currentUrlDB = new plasmaCrawlLURL(indexroot);
currentUrlDB.urldbcleanup();
currentUrlDB.close();
}
@ -798,7 +798,7 @@ public final class yacy {
try {
Iterator<indexContainer> indexContainerIterator = null;
if (resource.equals("all")) {
WordIndex = new plasmaWordIndex(indexPrimaryRoot, indexSecondaryRoot, 3000, log);
WordIndex = new plasmaWordIndex(indexPrimaryRoot, indexSecondaryRoot, log);
indexContainerIterator = WordIndex.wordContainers(wordChunkStartHash, false, false);
}
int counter = 0;
@ -858,7 +858,7 @@ public final class yacy {
String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"};
for (int i=0; i < dbFileNames.length; i++) {
File dbFile = new File(yacyDBPath,dbFileNames[i]);
kelondroMapObjects db = new kelondroMapObjects(new kelondroDyn(dbFile, true, true, 3000, yacySeedDB.commonHashLength, 480, '#', kelondroBase64Order.enhancedCoder, true, false, true), 500, yacySeedDB.sortFields, yacySeedDB.longaccFields, yacySeedDB.doubleaccFields, null, null);
kelondroMapObjects db = new kelondroMapObjects(new kelondroDyn(dbFile, true, true, yacySeedDB.commonHashLength, 480, '#', kelondroBase64Order.enhancedCoder, true, false, true), 500, yacySeedDB.sortFields, yacySeedDB.longaccFields, yacySeedDB.doubleaccFields, null, null);
kelondroMapObjects.mapIterator it;
it = db.maps(true, false);

40
yacy.init
Unescape View File

@ -167,7 +167,7 @@ htDocsPath = DATA/HTDOCS
# The complete path to this file is created by combination with the rootPath
# you can set a list of defaults, separated by comma
# the first one is priorized
defaultFiles = ConfigBasic.html,index.html,default.html,search.html,console.html,control.html,welcome.html,wiki.html,forum.html,blog.html,email.html,content.html,monitor.html,share.html,dir.html,readme.txt
defaultFiles = ConfigBasic.html,index.html,index.htm,default.html,search.html,console.html,control.html,welcome.html,wiki.html,forum.html,blog.html,email.html,content.html,monitor.html,share.html,dir.html,readme.txt
# locale-options: YaCy supports localization.
# Web pages for special languages are located in the htLocalePath
@ -595,44 +595,6 @@ cleanup.deletionPublishedNews = true
# a cluster value of '2' would be appropriate
80_indexing_cluster=1
# ram cache init timeout for database files
# ram cache for collection index
ramCacheRWI_time = 30000
# ram cache for responseHeader.db
ramCacheHTTP_time = 1000
# ram cache for urlHash.db
ramCacheLURL_time = 10000
# ram cache for stack crawl thread db
ramCachePreNURL_time = 3000
# ram cache for urlNotice.db
ramCacheNURL_time = 2000
# ram cache for urlErr.db
ramCacheEURL_time = 1000
# ram cache for seedDBs
ramCacheDHT_time = 1000
# ram cache for message.db
ramCacheMessage_time = 500
# ram cache for wiki.db
ramCacheWiki_time = 500
# ram cache for blog.db
ramCacheBlog_time = 500
# ram cache for robotsTxt.db
ramCacheRobots_time = 0
# ram cache for crawlProfile.db
ramCacheProfiles_time= 500
# default memory settings for startup of yacy
# is valid in unix/shell and windows environments but
# not for first startup of YaCy

Write Preview
Loading…
Cancel
Save