boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

replaced tree database structure for seed db by flex data structure

Browse Source 
I don't know if this helps, we will find out...

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3177 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
(no author) 18 years ago
parent e5845b6b74
commit 37e53b4a6a
14 changed files with 39 additions and 34 deletions
Show Stats Download Patch File Download Diff File

2
source/de/anomic/data/blogBoard.java
Unescape View File

@ -83,7 +83,7 @@ public class blogBoard {
public blogBoard(File actpath, int bufferkb, long preloadTime) {
new File(actpath.getParent()).mkdir();
if (datbase == null) {
datbase = new kelondroMap(kelondroDyn.open(actpath, bufferkb / 2 * 0x40, preloadTime, keyLength, recordSize, '_'));
datbase = new kelondroMap(kelondroDyn.open(actpath, bufferkb / 2 * 0x40, preloadTime, keyLength, recordSize, '_', true));
}
}

6
source/de/anomic/data/bookmarksDB.java
Unescape View File

@ -128,17 +128,17 @@ public class bookmarksDB {
tagCache=new HashMap();
bookmarkCache=new HashMap();
bookmarksFile.getParentFile().mkdirs();
this.bookmarksTable = new kelondroMap(kelondroDyn.open(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_'));
this.bookmarksTable = new kelondroMap(kelondroDyn.open(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true));
// tags
tagsFile.getParentFile().mkdirs();
boolean tagsFileExisted = tagsFile.exists();
this.tagsTable = new kelondroMap(kelondroDyn.open(tagsFile, bufferkb * 1024, preloadTime, 12, 256, '_'));
this.tagsTable = new kelondroMap(kelondroDyn.open(tagsFile, bufferkb * 1024, preloadTime, 12, 256, '_', true));
if (!tagsFileExisted) rebuildTags();
// dates
boolean datesExisted = datesFile.exists();
this.datesTable = new kelondroMap(kelondroDyn.open(datesFile, bufferkb * 1024, preloadTime, 20, 256, '_'));
this.datesTable = new kelondroMap(kelondroDyn.open(datesFile, bufferkb * 1024, preloadTime, 20, 256, '_', true));
if (!datesExisted) rebuildDates();
}

2
source/de/anomic/data/messageBoard.java
Unescape View File

@ -70,7 +70,7 @@ public class messageBoard {
public messageBoard(File path, int bufferkb, long preloadTime) {
new File(path.getParent()).mkdir();
if (database == null) {
database = new kelondroMap(kelondroDyn.open(path, bufferkb * 0x400, preloadTime, categoryLength + dateFormat.length() + 2, recordSize, '_'));
database = new kelondroMap(kelondroDyn.open(path, bufferkb * 0x400, preloadTime, categoryLength + dateFormat.length() + 2, recordSize, '_', true));
}
sn = 0;
}

4
source/de/anomic/data/userDB.java
Unescape View File

@ -77,7 +77,7 @@ public final class userDB {
this.bufferkb = bufferkb;
this.preloadTime = preloadTime;
userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMap(kelondroDyn.open(userTableFile, bufferkb * 1024, preloadTime, 128, 256, '_'));
this.userTable = new kelondroMap(kelondroDyn.open(userTableFile, bufferkb * 1024, preloadTime, 128, 256, '_', true));
}
public int dbCacheNodeChunkSize() {
@ -95,7 +95,7 @@ public final class userDB {
} catch (IOException e) {}
if (!(userTableFile.delete())) throw new RuntimeException("cannot delete user database");
userTableFile.getParentFile().mkdirs();
userTable = new kelondroMap(kelondroDyn.open(userTableFile, this.bufferkb, preloadTime, 256, 512, '_'));
userTable = new kelondroMap(kelondroDyn.open(userTableFile, this.bufferkb, preloadTime, 256, 512, '_', true));
}
public void close() {

4
source/de/anomic/data/wikiBoard.java
Unescape View File

@ -72,11 +72,11 @@ public class wikiBoard {
public wikiBoard(File actpath, File bkppath, int bufferkb, long preloadTime) {
new File(actpath.getParent()).mkdirs();
if (datbase == null) {
datbase = new kelondroMap(kelondroDyn.open(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_'));
datbase = new kelondroMap(kelondroDyn.open(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_', true));
}
new File(bkppath.getParent()).mkdirs();
if (bkpbase == null) {
bkpbase = new kelondroMap(kelondroDyn.open(bkppath, bufferkb / 2 * 0x400, preloadTime, keyLength + dateFormat.length(), recordSize, '_'));
bkpbase = new kelondroMap(kelondroDyn.open(bkppath, bufferkb / 2 * 0x400, preloadTime, keyLength + dateFormat.length(), recordSize, '_', true));
}
}

29
source/de/anomic/kelondro/kelondroDyn.java
Unescape View File

@ -72,16 +72,21 @@ public class kelondroDyn {
private kelondroObjectBuffer buffer;
private kelondroRow rowdef;
public kelondroDyn(File file, long buffersize /*bytes*/, long preloadTime, int key, int nodesize, char fillChar) throws IOException {
this(file, buffersize, preloadTime, key, nodesize, fillChar, new kelondroNaturalOrder(true));
public kelondroDyn(File file, long buffersize /*bytes*/, long preloadTime, int key, int nodesize, char fillChar, boolean usetree) throws IOException {
this(file, buffersize, preloadTime, key, nodesize, fillChar, new kelondroNaturalOrder(true), usetree);
}
public kelondroDyn(File file, long buffersize /* bytes */, long preloadTime, int key,
int nodesize, char fillChar, kelondroOrder objectOrder) throws IOException {
int nodesize, char fillChar, kelondroOrder objectOrder, boolean usetree) throws IOException {
// creates or opens a dynamic tree
rowdef = new kelondroRow("byte[] key-" + (key + counterlen) + ", byte[] node-" + nodesize, objectOrder, 0);
kelondroTree tree = new kelondroTree(file, buffersize / 2, preloadTime, rowdef, 1, 8);
this.index = new kelondroCache(tree, buffersize / 2, true, false);
if (usetree) {
kelondroTree tree = new kelondroTree(file, buffersize / 2, preloadTime, rowdef, 1, 8);
this.index = new kelondroCache(tree, buffersize / 2, true, false);
} else {
kelondroFlexTable table = new kelondroFlexTable(file.getParentFile(), file.getName(), buffersize / 2, -1, rowdef);
this.index = new kelondroCache(table, buffersize / 2, true, false);
}
this.keylen = index.row().width(0) - counterlen;
this.reclen = index.row().width(1);
this.fillChar = fillChar;
@ -90,20 +95,20 @@ public class kelondroDyn {
buffer = new kelondroObjectBuffer(file.toString());
}
public static final kelondroDyn open(File file, long buffersize /* bytes */, long preloadTime, int key, int nodesize, char fillChar) {
return open(file, buffersize, preloadTime, key, nodesize, fillChar, new kelondroNaturalOrder(true));
public static final kelondroDyn open(File file, long buffersize /* bytes */, long preloadTime, int key, int nodesize, char fillChar, boolean usetree) {
return open(file, buffersize, preloadTime, key, nodesize, fillChar, new kelondroNaturalOrder(true), usetree);
}
public static final kelondroDyn open(File file, long buffersize /* bytes */, long preloadTime, int key,
int nodesize, char fillChar, kelondroOrder objectOrder) {
int nodesize, char fillChar, kelondroOrder objectOrder, boolean usetree) {
// opens new or existing file; in case that any error occur the file is deleted again and it is tried to create the file again
// if that fails, the method returns null
try {
return new kelondroDyn(file, buffersize, preloadTime, key, nodesize, fillChar, objectOrder);
return new kelondroDyn(file, buffersize, preloadTime, key, nodesize, fillChar, objectOrder, usetree);
} catch (IOException e) {
file.delete();
try {
return new kelondroDyn(file, buffersize, preloadTime, key, nodesize, fillChar, objectOrder);
return new kelondroDyn(file, buffersize, preloadTime, key, nodesize, fillChar, objectOrder, usetree);
} catch (IOException ee) {
serverLog.logSevere("kelondroDyn", "cannot open or create file " + file.toString());
e.printStackTrace();
@ -484,7 +489,7 @@ public class kelondroDyn {
if (args.length == 1) {
// open a db and list keys
try {
kelondroDyn kd = new kelondroDyn(new File(args[0]), 0x100000, 0, 4 ,100, '_');
kelondroDyn kd = new kelondroDyn(new File(args[0]), 0x100000, 0, 4 ,100, '_', false);
System.out.println(kd.sizeDyn() + " elements in DB");
Iterator i = kd.dynKeys(true, false);
while (i.hasNext())
@ -501,7 +506,7 @@ public class kelondroDyn {
File f = new File(args[3]);
kelondroDyn kd;
try {
kd = new kelondroDyn(db, 0x100000, 0, 80, 200, '_');
kd = new kelondroDyn(db, 0x100000, 0, 80, 200, '_', false);
if (writeFile)
kd.readFile(key, f);
else

2
source/de/anomic/kelondro/kelondroDynTree.java
Unescape View File

@ -79,7 +79,7 @@ public class kelondroDynTree {
this.cache = new Hashtable();
//this.cycleCache = Long.MIN_VALUE;
this.cycleBuffer = Long.MIN_VALUE;
this.table = new kelondroDyn(file, buffersize, preloadTime, keylength, nodesize, fillChar);
this.table = new kelondroDyn(file, buffersize, preloadTime, keylength, nodesize, fillChar, true);
this.treeRAHandles = new Hashtable();
}

2
source/de/anomic/kelondro/kelondroMapTable.java
Unescape View File

@ -82,7 +82,7 @@ public class kelondroMapTable {
File tablefile = new File(tablesPath, "table." + tablename + ".mdb");
kelondroDyn dyn;
if (!(tablefile.exists())) tablefile.getParentFile().mkdirs();
dyn = new kelondroDyn(tablefile, buffersize, preloadTime, keysize, nodesize, fillChar);
dyn = new kelondroDyn(tablefile, buffersize, preloadTime, keysize, nodesize, fillChar, true);
kelondroMap map = new kelondroMap(dyn, sortfields, accfields);
mTables.put(tablename, map);
}

4
source/de/anomic/plasma/plasmaCrawlProfile.java
Unescape View File

@ -69,7 +69,7 @@ public class plasmaCrawlProfile {
this.bufferkb = bufferkb;
this.preloadTime = preloadTime;
profileTableFile.getParentFile().mkdirs();
kelondroDyn dyn = kelondroDyn.open(profileTableFile, bufferkb * 1024, preloadTime, crawlProfileHandleLength, 2000, '#');
kelondroDyn dyn = kelondroDyn.open(profileTableFile, bufferkb * 1024, preloadTime, crawlProfileHandleLength, 2000, '#', true);
profileTable = new kelondroMap(dyn);
domsCache = new HashMap();
}
@ -95,7 +95,7 @@ public class plasmaCrawlProfile {
if (profileTable != null) try { profileTable.close(); } catch (IOException e) {}
if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database");
profileTableFile.getParentFile().mkdirs();
kelondroDyn dyn = kelondroDyn.open(profileTableFile, bufferkb * 1024, preloadTime, crawlProfileHandleLength, 2000, '#');
kelondroDyn dyn = kelondroDyn.open(profileTableFile, bufferkb * 1024, preloadTime, crawlProfileHandleLength, 2000, '#', true);
profileTable = new kelondroMap(dyn);
}

4
source/de/anomic/plasma/plasmaCrawlRobotsTxt.java
Unescape View File

@ -73,7 +73,7 @@ public class plasmaCrawlRobotsTxt {
this.bufferkb = bufferkb;
this.preloadTime = preloadTime;
robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMap(kelondroDyn.open(robotsTableFile, bufferkb * 1024, preloadTime, 256, 512, '_'));
robotsTable = new kelondroMap(kelondroDyn.open(robotsTableFile, bufferkb * 1024, preloadTime, 256, 512, '_', true));
}
public int cacheNodeChunkSize() {
@ -99,7 +99,7 @@ public class plasmaCrawlRobotsTxt {
} catch (IOException e) {}
if (!(robotsTableFile.delete())) throw new RuntimeException("cannot delete robots.txt database");
robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMap(kelondroDyn.open(robotsTableFile, this.bufferkb, preloadTime, 256, 512, '_'));
robotsTable = new kelondroMap(kelondroDyn.open(robotsTableFile, this.bufferkb, preloadTime, 256, 512, '_', true));
}
public void close() {

2
source/de/anomic/plasma/plasmaHTCache.java
Unescape View File

@ -174,7 +174,7 @@ public final class plasmaHTCache {
// open the response header database
File dbfile = new File(this.cachePath, "responseHeader.db");
try {
this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400, preloadTime, yacySeedDB.commonHashLength, 150, '#'));
this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400, preloadTime, yacySeedDB.commonHashLength, 150, '#', true));
} catch (IOException e) {
this.log.logSevere("the request header database could not be opened: " + e.getMessage());
System.exit(0);

6
source/de/anomic/yacy/yacyCore.java
Unescape View File

@ -200,9 +200,9 @@ public class yacyCore {
log.logConfig("DHT Cache memory = " + memDHT + " KB");
seedDB = new yacySeedDB(
sb,
new File(yacyDBPath, "seed.new.db"),
new File(yacyDBPath, "seed.old.db"),
new File(yacyDBPath, "seed.pot.db"),
new File(yacyDBPath, "seed0.new.db"),
new File(yacyDBPath, "seed0.old.db"),
new File(yacyDBPath, "seed0.pot.db"),
memDHT, memDHT_time);
// create or init news database

4
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -206,11 +206,11 @@ public final class yacySeedDB {
private synchronized kelondroMap openSeedTable(File seedDBFile) {
new File(seedDBFile.getParent()).mkdirs();
try {
return new kelondroMap(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#'), sortFields, accFields);
return new kelondroMap(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false), sortFields, accFields);
} catch (Exception e) {
seedDBFile.delete();
// try again
return new kelondroMap(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#'), sortFields, accFields);
return new kelondroMap(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false), sortFields, accFields);
}
}

2
source/yacy.java
Unescape View File

@ -1320,7 +1320,7 @@ public final class yacy {
String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"};
for (int i=0; i < dbFileNames.length; i++) {
File dbFile = new File(yacyDBPath,dbFileNames[i]);
kelondroMap db = new kelondroMap(new kelondroDyn(dbFile, (1024 * 0x400) / 3, 3000, yacySeedDB.commonHashLength, 480, '#'), yacySeedDB.sortFields, yacySeedDB.accFields);
kelondroMap db = new kelondroMap(new kelondroDyn(dbFile, (1024 * 0x400) / 3, 3000, yacySeedDB.commonHashLength, 480, '#', true), yacySeedDB.sortFields, yacySeedDB.accFields);
kelondroMap.mapIterator it;
it = db.maps(true, false);

Write Preview
Loading…
Cancel
Save