diff --git a/source/de/anomic/data/blogBoard.java b/source/de/anomic/data/blogBoard.java index 0a7489842..5e2c80c68 100644 --- a/source/de/anomic/data/blogBoard.java +++ b/source/de/anomic/data/blogBoard.java @@ -83,7 +83,7 @@ public class blogBoard { public blogBoard(File actpath, int bufferkb, long preloadTime) { new File(actpath.getParent()).mkdir(); if (datbase == null) { - datbase = new kelondroMap(kelondroDyn.open(actpath, bufferkb / 2 * 0x40, preloadTime, keyLength, recordSize, '_')); + datbase = new kelondroMap(kelondroDyn.open(actpath, bufferkb / 2 * 0x40, preloadTime, keyLength, recordSize, '_', true)); } } diff --git a/source/de/anomic/data/bookmarksDB.java b/source/de/anomic/data/bookmarksDB.java index 4032eeb5a..ce50a612d 100644 --- a/source/de/anomic/data/bookmarksDB.java +++ b/source/de/anomic/data/bookmarksDB.java @@ -128,17 +128,17 @@ public class bookmarksDB { tagCache=new HashMap(); bookmarkCache=new HashMap(); bookmarksFile.getParentFile().mkdirs(); - this.bookmarksTable = new kelondroMap(kelondroDyn.open(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_')); + this.bookmarksTable = new kelondroMap(kelondroDyn.open(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true)); // tags tagsFile.getParentFile().mkdirs(); boolean tagsFileExisted = tagsFile.exists(); - this.tagsTable = new kelondroMap(kelondroDyn.open(tagsFile, bufferkb * 1024, preloadTime, 12, 256, '_')); + this.tagsTable = new kelondroMap(kelondroDyn.open(tagsFile, bufferkb * 1024, preloadTime, 12, 256, '_', true)); if (!tagsFileExisted) rebuildTags(); // dates boolean datesExisted = datesFile.exists(); - this.datesTable = new kelondroMap(kelondroDyn.open(datesFile, bufferkb * 1024, preloadTime, 20, 256, '_')); + this.datesTable = new kelondroMap(kelondroDyn.open(datesFile, bufferkb * 1024, preloadTime, 20, 256, '_', true)); if (!datesExisted) rebuildDates(); } diff --git a/source/de/anomic/data/messageBoard.java b/source/de/anomic/data/messageBoard.java index 41c8b17c7..6081feae7 100644 --- a/source/de/anomic/data/messageBoard.java +++ b/source/de/anomic/data/messageBoard.java @@ -70,7 +70,7 @@ public class messageBoard { public messageBoard(File path, int bufferkb, long preloadTime) { new File(path.getParent()).mkdir(); if (database == null) { - database = new kelondroMap(kelondroDyn.open(path, bufferkb * 0x400, preloadTime, categoryLength + dateFormat.length() + 2, recordSize, '_')); + database = new kelondroMap(kelondroDyn.open(path, bufferkb * 0x400, preloadTime, categoryLength + dateFormat.length() + 2, recordSize, '_', true)); } sn = 0; } diff --git a/source/de/anomic/data/userDB.java b/source/de/anomic/data/userDB.java index 2b3e27598..cc2798bfd 100644 --- a/source/de/anomic/data/userDB.java +++ b/source/de/anomic/data/userDB.java @@ -77,7 +77,7 @@ public final class userDB { this.bufferkb = bufferkb; this.preloadTime = preloadTime; userTableFile.getParentFile().mkdirs(); - this.userTable = new kelondroMap(kelondroDyn.open(userTableFile, bufferkb * 1024, preloadTime, 128, 256, '_')); + this.userTable = new kelondroMap(kelondroDyn.open(userTableFile, bufferkb * 1024, preloadTime, 128, 256, '_', true)); } public int dbCacheNodeChunkSize() { @@ -95,7 +95,7 @@ public final class userDB { } catch (IOException e) {} if (!(userTableFile.delete())) throw new RuntimeException("cannot delete user database"); userTableFile.getParentFile().mkdirs(); - userTable = new kelondroMap(kelondroDyn.open(userTableFile, this.bufferkb, preloadTime, 256, 512, '_')); + userTable = new kelondroMap(kelondroDyn.open(userTableFile, this.bufferkb, preloadTime, 256, 512, '_', true)); } public void close() { diff --git a/source/de/anomic/data/wikiBoard.java b/source/de/anomic/data/wikiBoard.java index 2e00a4ce1..ae89206e5 100644 --- a/source/de/anomic/data/wikiBoard.java +++ b/source/de/anomic/data/wikiBoard.java @@ -72,11 +72,11 @@ public class wikiBoard { public wikiBoard(File actpath, File bkppath, int bufferkb, long preloadTime) { new File(actpath.getParent()).mkdirs(); if (datbase == null) { - datbase = new kelondroMap(kelondroDyn.open(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_')); + datbase = new kelondroMap(kelondroDyn.open(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_', true)); } new File(bkppath.getParent()).mkdirs(); if (bkpbase == null) { - bkpbase = new kelondroMap(kelondroDyn.open(bkppath, bufferkb / 2 * 0x400, preloadTime, keyLength + dateFormat.length(), recordSize, '_')); + bkpbase = new kelondroMap(kelondroDyn.open(bkppath, bufferkb / 2 * 0x400, preloadTime, keyLength + dateFormat.length(), recordSize, '_', true)); } } diff --git a/source/de/anomic/kelondro/kelondroDyn.java b/source/de/anomic/kelondro/kelondroDyn.java index b7ddc4b29..61a0fb82b 100644 --- a/source/de/anomic/kelondro/kelondroDyn.java +++ b/source/de/anomic/kelondro/kelondroDyn.java @@ -72,16 +72,21 @@ public class kelondroDyn { private kelondroObjectBuffer buffer; private kelondroRow rowdef; - public kelondroDyn(File file, long buffersize /*bytes*/, long preloadTime, int key, int nodesize, char fillChar) throws IOException { - this(file, buffersize, preloadTime, key, nodesize, fillChar, new kelondroNaturalOrder(true)); + public kelondroDyn(File file, long buffersize /*bytes*/, long preloadTime, int key, int nodesize, char fillChar, boolean usetree) throws IOException { + this(file, buffersize, preloadTime, key, nodesize, fillChar, new kelondroNaturalOrder(true), usetree); } public kelondroDyn(File file, long buffersize /* bytes */, long preloadTime, int key, - int nodesize, char fillChar, kelondroOrder objectOrder) throws IOException { + int nodesize, char fillChar, kelondroOrder objectOrder, boolean usetree) throws IOException { // creates or opens a dynamic tree rowdef = new kelondroRow("byte[] key-" + (key + counterlen) + ", byte[] node-" + nodesize, objectOrder, 0); - kelondroTree tree = new kelondroTree(file, buffersize / 2, preloadTime, rowdef, 1, 8); - this.index = new kelondroCache(tree, buffersize / 2, true, false); + if (usetree) { + kelondroTree tree = new kelondroTree(file, buffersize / 2, preloadTime, rowdef, 1, 8); + this.index = new kelondroCache(tree, buffersize / 2, true, false); + } else { + kelondroFlexTable table = new kelondroFlexTable(file.getParentFile(), file.getName(), buffersize / 2, -1, rowdef); + this.index = new kelondroCache(table, buffersize / 2, true, false); + } this.keylen = index.row().width(0) - counterlen; this.reclen = index.row().width(1); this.fillChar = fillChar; @@ -90,20 +95,20 @@ public class kelondroDyn { buffer = new kelondroObjectBuffer(file.toString()); } - public static final kelondroDyn open(File file, long buffersize /* bytes */, long preloadTime, int key, int nodesize, char fillChar) { - return open(file, buffersize, preloadTime, key, nodesize, fillChar, new kelondroNaturalOrder(true)); + public static final kelondroDyn open(File file, long buffersize /* bytes */, long preloadTime, int key, int nodesize, char fillChar, boolean usetree) { + return open(file, buffersize, preloadTime, key, nodesize, fillChar, new kelondroNaturalOrder(true), usetree); } public static final kelondroDyn open(File file, long buffersize /* bytes */, long preloadTime, int key, - int nodesize, char fillChar, kelondroOrder objectOrder) { + int nodesize, char fillChar, kelondroOrder objectOrder, boolean usetree) { // opens new or existing file; in case that any error occur the file is deleted again and it is tried to create the file again // if that fails, the method returns null try { - return new kelondroDyn(file, buffersize, preloadTime, key, nodesize, fillChar, objectOrder); + return new kelondroDyn(file, buffersize, preloadTime, key, nodesize, fillChar, objectOrder, usetree); } catch (IOException e) { file.delete(); try { - return new kelondroDyn(file, buffersize, preloadTime, key, nodesize, fillChar, objectOrder); + return new kelondroDyn(file, buffersize, preloadTime, key, nodesize, fillChar, objectOrder, usetree); } catch (IOException ee) { serverLog.logSevere("kelondroDyn", "cannot open or create file " + file.toString()); e.printStackTrace(); @@ -484,7 +489,7 @@ public class kelondroDyn { if (args.length == 1) { // open a db and list keys try { - kelondroDyn kd = new kelondroDyn(new File(args[0]), 0x100000, 0, 4 ,100, '_'); + kelondroDyn kd = new kelondroDyn(new File(args[0]), 0x100000, 0, 4 ,100, '_', false); System.out.println(kd.sizeDyn() + " elements in DB"); Iterator i = kd.dynKeys(true, false); while (i.hasNext()) @@ -501,7 +506,7 @@ public class kelondroDyn { File f = new File(args[3]); kelondroDyn kd; try { - kd = new kelondroDyn(db, 0x100000, 0, 80, 200, '_'); + kd = new kelondroDyn(db, 0x100000, 0, 80, 200, '_', false); if (writeFile) kd.readFile(key, f); else diff --git a/source/de/anomic/kelondro/kelondroDynTree.java b/source/de/anomic/kelondro/kelondroDynTree.java index 85f360cbe..fc6a5130e 100644 --- a/source/de/anomic/kelondro/kelondroDynTree.java +++ b/source/de/anomic/kelondro/kelondroDynTree.java @@ -79,7 +79,7 @@ public class kelondroDynTree { this.cache = new Hashtable(); //this.cycleCache = Long.MIN_VALUE; this.cycleBuffer = Long.MIN_VALUE; - this.table = new kelondroDyn(file, buffersize, preloadTime, keylength, nodesize, fillChar); + this.table = new kelondroDyn(file, buffersize, preloadTime, keylength, nodesize, fillChar, true); this.treeRAHandles = new Hashtable(); } diff --git a/source/de/anomic/kelondro/kelondroMapTable.java b/source/de/anomic/kelondro/kelondroMapTable.java index ec21fd72a..146418986 100644 --- a/source/de/anomic/kelondro/kelondroMapTable.java +++ b/source/de/anomic/kelondro/kelondroMapTable.java @@ -82,7 +82,7 @@ public class kelondroMapTable { File tablefile = new File(tablesPath, "table." + tablename + ".mdb"); kelondroDyn dyn; if (!(tablefile.exists())) tablefile.getParentFile().mkdirs(); - dyn = new kelondroDyn(tablefile, buffersize, preloadTime, keysize, nodesize, fillChar); + dyn = new kelondroDyn(tablefile, buffersize, preloadTime, keysize, nodesize, fillChar, true); kelondroMap map = new kelondroMap(dyn, sortfields, accfields); mTables.put(tablename, map); } diff --git a/source/de/anomic/plasma/plasmaCrawlProfile.java b/source/de/anomic/plasma/plasmaCrawlProfile.java index 2a3c300ff..c2d11ce26 100644 --- a/source/de/anomic/plasma/plasmaCrawlProfile.java +++ b/source/de/anomic/plasma/plasmaCrawlProfile.java @@ -69,7 +69,7 @@ public class plasmaCrawlProfile { this.bufferkb = bufferkb; this.preloadTime = preloadTime; profileTableFile.getParentFile().mkdirs(); - kelondroDyn dyn = kelondroDyn.open(profileTableFile, bufferkb * 1024, preloadTime, crawlProfileHandleLength, 2000, '#'); + kelondroDyn dyn = kelondroDyn.open(profileTableFile, bufferkb * 1024, preloadTime, crawlProfileHandleLength, 2000, '#', true); profileTable = new kelondroMap(dyn); domsCache = new HashMap(); } @@ -95,7 +95,7 @@ public class plasmaCrawlProfile { if (profileTable != null) try { profileTable.close(); } catch (IOException e) {} if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database"); profileTableFile.getParentFile().mkdirs(); - kelondroDyn dyn = kelondroDyn.open(profileTableFile, bufferkb * 1024, preloadTime, crawlProfileHandleLength, 2000, '#'); + kelondroDyn dyn = kelondroDyn.open(profileTableFile, bufferkb * 1024, preloadTime, crawlProfileHandleLength, 2000, '#', true); profileTable = new kelondroMap(dyn); } diff --git a/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java b/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java index be86e2289..60d1ae06d 100644 --- a/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java +++ b/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java @@ -73,7 +73,7 @@ public class plasmaCrawlRobotsTxt { this.bufferkb = bufferkb; this.preloadTime = preloadTime; robotsTableFile.getParentFile().mkdirs(); - robotsTable = new kelondroMap(kelondroDyn.open(robotsTableFile, bufferkb * 1024, preloadTime, 256, 512, '_')); + robotsTable = new kelondroMap(kelondroDyn.open(robotsTableFile, bufferkb * 1024, preloadTime, 256, 512, '_', true)); } public int cacheNodeChunkSize() { @@ -99,7 +99,7 @@ public class plasmaCrawlRobotsTxt { } catch (IOException e) {} if (!(robotsTableFile.delete())) throw new RuntimeException("cannot delete robots.txt database"); robotsTableFile.getParentFile().mkdirs(); - robotsTable = new kelondroMap(kelondroDyn.open(robotsTableFile, this.bufferkb, preloadTime, 256, 512, '_')); + robotsTable = new kelondroMap(kelondroDyn.open(robotsTableFile, this.bufferkb, preloadTime, 256, 512, '_', true)); } public void close() { diff --git a/source/de/anomic/plasma/plasmaHTCache.java b/source/de/anomic/plasma/plasmaHTCache.java index 9de1c4440..2282c9451 100644 --- a/source/de/anomic/plasma/plasmaHTCache.java +++ b/source/de/anomic/plasma/plasmaHTCache.java @@ -174,7 +174,7 @@ public final class plasmaHTCache { // open the response header database File dbfile = new File(this.cachePath, "responseHeader.db"); try { - this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400, preloadTime, yacySeedDB.commonHashLength, 150, '#')); + this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400, preloadTime, yacySeedDB.commonHashLength, 150, '#', true)); } catch (IOException e) { this.log.logSevere("the request header database could not be opened: " + e.getMessage()); System.exit(0); diff --git a/source/de/anomic/yacy/yacyCore.java b/source/de/anomic/yacy/yacyCore.java index f63d9eb3f..4fb375977 100644 --- a/source/de/anomic/yacy/yacyCore.java +++ b/source/de/anomic/yacy/yacyCore.java @@ -200,9 +200,9 @@ public class yacyCore { log.logConfig("DHT Cache memory = " + memDHT + " KB"); seedDB = new yacySeedDB( sb, - new File(yacyDBPath, "seed.new.db"), - new File(yacyDBPath, "seed.old.db"), - new File(yacyDBPath, "seed.pot.db"), + new File(yacyDBPath, "seed0.new.db"), + new File(yacyDBPath, "seed0.old.db"), + new File(yacyDBPath, "seed0.pot.db"), memDHT, memDHT_time); // create or init news database diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index a82b8a0fe..a48ee7467 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -206,11 +206,11 @@ public final class yacySeedDB { private synchronized kelondroMap openSeedTable(File seedDBFile) { new File(seedDBFile.getParent()).mkdirs(); try { - return new kelondroMap(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#'), sortFields, accFields); + return new kelondroMap(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false), sortFields, accFields); } catch (Exception e) { seedDBFile.delete(); // try again - return new kelondroMap(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#'), sortFields, accFields); + return new kelondroMap(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false), sortFields, accFields); } } diff --git a/source/yacy.java b/source/yacy.java index e0c22f738..cef18bda0 100644 --- a/source/yacy.java +++ b/source/yacy.java @@ -1320,7 +1320,7 @@ public final class yacy { String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"}; for (int i=0; i < dbFileNames.length; i++) { File dbFile = new File(yacyDBPath,dbFileNames[i]); - kelondroMap db = new kelondroMap(new kelondroDyn(dbFile, (1024 * 0x400) / 3, 3000, yacySeedDB.commonHashLength, 480, '#'), yacySeedDB.sortFields, yacySeedDB.accFields); + kelondroMap db = new kelondroMap(new kelondroDyn(dbFile, (1024 * 0x400) / 3, 3000, yacySeedDB.commonHashLength, 480, '#', true), yacySeedDB.sortFields, yacySeedDB.accFields); kelondroMap.mapIterator it; it = db.maps(true, false);