diff --git a/source/dbtest.java b/source/dbtest.java index 0579b96c5..afc5c0d7a 100644 --- a/source/dbtest.java +++ b/source/dbtest.java @@ -34,7 +34,6 @@ public class dbtest { public final static int keylength = 12; public final static int valuelength = 223; // sum of all data length as defined in plasmaURL - public final static long preload = 1000; // 1 second public static byte[] dummyvalue2 = new byte[valuelength]; static { // fill the dummy value @@ -201,15 +200,15 @@ public class dbtest { } if (dbe.equals("kelondroTree")) { File tablefile = new File(tablename + ".kelondro.db"); - return new kelondroCache(new kelondroTree(tablefile, true, preload, testRow)); + return new kelondroCache(new kelondroTree(tablefile, true, 0, testRow)); } if (dbe.equals("kelondroFlexTable")) { File tablepath = new File(tablename).getParentFile(); - return new kelondroFlexTable(tablepath, new File(tablename).getName(), preload, testRow, 0, true); + return new kelondroFlexTable(tablepath, new File(tablename).getName(), testRow, 0, true); } if (dbe.equals("kelondroSplitTable")) { File tablepath = new File(tablename).getParentFile(); - return new kelondroSplitTable(tablepath, new File(tablename).getName(), preload, testRow, true); + return new kelondroSplitTable(tablepath, new File(tablename).getName(), testRow, true); } if (dbe.equals("kelondroEcoTable")) { return new kelondroEcoTable(new File(tablename), testRow, kelondroEcoTable.tailCacheForceUsage, 1000, 0); diff --git a/source/de/anomic/data/blogBoard.java b/source/de/anomic/data/blogBoard.java index 627349c1f..a1d05eab5 100644 --- a/source/de/anomic/data/blogBoard.java +++ b/source/de/anomic/data/blogBoard.java @@ -80,10 +80,10 @@ public class blogBoard { kelondroMapObjects database = null; - public blogBoard(File actpath, long preloadTime) { + public blogBoard(File actpath) { new File(actpath.getParent()).mkdir(); if (database == null) { - database = new kelondroMapObjects(new kelondroDyn(actpath, true, true, preloadTime, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500); + database = new kelondroMapObjects(new kelondroDyn(actpath, true, true, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500); } } public int size() { diff --git a/source/de/anomic/data/blogBoardComments.java b/source/de/anomic/data/blogBoardComments.java index 063295edd..38cd9bf06 100644 --- a/source/de/anomic/data/blogBoardComments.java +++ b/source/de/anomic/data/blogBoardComments.java @@ -82,10 +82,10 @@ public class blogBoardComments { SimpleFormatter.setTimeZone(TimeZone.getTimeZone("GMT")); } private kelondroMapObjects database = null; - public blogBoardComments(File actpath, long preloadTime) { + public blogBoardComments(File actpath) { new File(actpath.getParent()).mkdir(); if (database == null) { - database = new kelondroMapObjects(new kelondroDyn(actpath, true, true, preloadTime, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, false, false, false), 500); + database = new kelondroMapObjects(new kelondroDyn(actpath, true, true, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, false, false, false), 500); } } public int size() { diff --git a/source/de/anomic/data/bookmarksDB.java b/source/de/anomic/data/bookmarksDB.java index fa9d25ddd..d0866c7ff 100644 --- a/source/de/anomic/data/bookmarksDB.java +++ b/source/de/anomic/data/bookmarksDB.java @@ -111,22 +111,22 @@ public class bookmarksDB { // bookmarksDB's class constructor // ------------------------------------ - public bookmarksDB(File bookmarksFile, File tagsFile, File datesFile, long preloadTime) { + public bookmarksDB(File bookmarksFile, File tagsFile, File datesFile) { // bookmarks tagCache=new HashMap(); bookmarksFile.getParentFile().mkdirs(); //this.bookmarksTable = new kelondroMap(kelondroDyn.open(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true, false)); - this.bookmarksTable = new kelondroObjects(new kelondroDyn(bookmarksFile, true, true, preloadTime, 12, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 1000); + this.bookmarksTable = new kelondroObjects(new kelondroDyn(bookmarksFile, true, true, 12, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 1000); // tags tagsFile.getParentFile().mkdirs(); boolean tagsFileExisted = tagsFile.exists(); - this.tagsTable = new kelondroMapObjects(new kelondroDyn(tagsFile, true, true, preloadTime, 12, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500); + this.tagsTable = new kelondroMapObjects(new kelondroDyn(tagsFile, true, true, 12, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500); if (!tagsFileExisted) rebuildTags(); // dates boolean datesExisted = datesFile.exists(); - this.datesTable = new kelondroMapObjects(new kelondroDyn(datesFile, true, true, preloadTime, 20, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500); + this.datesTable = new kelondroMapObjects(new kelondroDyn(datesFile, true, true, 20, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500); if (!datesExisted) rebuildDates(); } diff --git a/source/de/anomic/data/messageBoard.java b/source/de/anomic/data/messageBoard.java index e24e0d722..8a901a21b 100644 --- a/source/de/anomic/data/messageBoard.java +++ b/source/de/anomic/data/messageBoard.java @@ -69,10 +69,10 @@ public class messageBoard { private kelondroMapObjects database = null; private int sn = 0; - public messageBoard(File path, long preloadTime) { + public messageBoard(File path) { new File(path.getParent()).mkdir(); if (database == null) { - database = new kelondroMapObjects(new kelondroDyn(path, true, true, preloadTime, categoryLength + dateFormat.length() + 2, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500); + database = new kelondroMapObjects(new kelondroDyn(path, true, true, categoryLength + dateFormat.length() + 2, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500); } sn = 0; } diff --git a/source/de/anomic/data/userDB.java b/source/de/anomic/data/userDB.java index 19ed2c315..0e58f7fc0 100644 --- a/source/de/anomic/data/userDB.java +++ b/source/de/anomic/data/userDB.java @@ -70,15 +70,13 @@ public final class userDB { kelondroMapObjects userTable; private final File userTableFile; - private long preloadTime; private HashMap ipUsers = new HashMap(); private HashMap cookieUsers = new HashMap(); - public userDB(File userTableFile, long preloadTime) { + public userDB(File userTableFile) { this.userTableFile = userTableFile; - this.preloadTime = preloadTime; userTableFile.getParentFile().mkdirs(); - this.userTable = new kelondroMapObjects(new kelondroDyn(userTableFile, true, true, preloadTime, 128, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 10); + this.userTable = new kelondroMapObjects(new kelondroDyn(userTableFile, true, true, 128, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 10); } void resetDatabase() { @@ -86,7 +84,7 @@ public final class userDB { if (userTable != null) userTable.close(); if (!(userTableFile.delete())) throw new RuntimeException("cannot delete user database"); userTableFile.getParentFile().mkdirs(); - userTable = new kelondroMapObjects(new kelondroDyn(userTableFile, true, true, preloadTime, 256, 512, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 10); + userTable = new kelondroMapObjects(new kelondroDyn(userTableFile, true, true, 256, 512, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 10); } public void close() { diff --git a/source/de/anomic/data/wikiBoard.java b/source/de/anomic/data/wikiBoard.java index 1f31a0df0..8cb1ce626 100644 --- a/source/de/anomic/data/wikiBoard.java +++ b/source/de/anomic/data/wikiBoard.java @@ -70,14 +70,14 @@ public class wikiBoard { private kelondroMapObjects bkpbase = null; private static HashMap authors = new HashMap(); - public wikiBoard(File actpath, File bkppath, long preloadTime) { + public wikiBoard(File actpath, File bkppath) { new File(actpath.getParent()).mkdirs(); if (datbase == null) { - datbase = new kelondroMapObjects(new kelondroDyn(actpath, true, true, preloadTime, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500); + datbase = new kelondroMapObjects(new kelondroDyn(actpath, true, true, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500); } new File(bkppath.getParent()).mkdirs(); if (bkpbase == null) { - bkpbase = new kelondroMapObjects(new kelondroDyn(bkppath, true, true, preloadTime, keyLength + dateFormat.length(), recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500); + bkpbase = new kelondroMapObjects(new kelondroDyn(bkppath, true, true, keyLength + dateFormat.length(), recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500); } } diff --git a/source/de/anomic/htmlFilter/htmlFilterContentScraper.java b/source/de/anomic/htmlFilter/htmlFilterContentScraper.java index 7698ec366..82dbf7c4d 100644 --- a/source/de/anomic/htmlFilter/htmlFilterContentScraper.java +++ b/source/de/anomic/htmlFilter/htmlFilterContentScraper.java @@ -375,7 +375,7 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen public String getDescription() { String s = metas.get("description"); - if (s == null) s = metas.get("DC.description"); + if (s == null) s = metas.get("dc.description"); if (s == null) return ""; else return s; } @@ -387,21 +387,21 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen public String getAuthor() { String s = metas.get("author"); if (s == null) s = metas.get("copyright"); - if (s == null) s = metas.get("DC.creator"); + if (s == null) s = metas.get("dc.creator"); if (s == null) return ""; return s; } public String[] getContentLanguages() { String s = metas.get("content-language"); - if (s == null) s = metas.get("DC.language"); + if (s == null) s = metas.get("dc.language"); if (s == null) s = ""; return s.split(" |,"); } public String[] getKeywords() { String s = metas.get("keywords"); - if (s == null) s = metas.get("DC.description"); + if (s == null) s = metas.get("dc.description"); if (s == null) s = ""; if (s.length() == 0) { return getTitle().toLowerCase().split(splitrex); diff --git a/source/de/anomic/index/indexCollectionRI.java b/source/de/anomic/index/indexCollectionRI.java index 87c8d2ab0..eadf9cb98 100644 --- a/source/de/anomic/index/indexCollectionRI.java +++ b/source/de/anomic/index/indexCollectionRI.java @@ -45,14 +45,13 @@ public class indexCollectionRI implements indexRI { kelondroCollectionIndex collectionIndex; - public indexCollectionRI(File path, String filenameStub, long preloadTime, int maxpartition, kelondroRow payloadrow) { + public indexCollectionRI(File path, String filenameStub, int maxpartition, kelondroRow payloadrow) { try { collectionIndex = new kelondroCollectionIndex( path, filenameStub, 12 /*keyLength*/, kelondroBase64Order.enhancedCoder, - preloadTime, 4 /*loadfactor*/, maxpartition, payloadrow); diff --git a/source/de/anomic/kelondro/kelondroCollectionIndex.java b/source/de/anomic/kelondro/kelondroCollectionIndex.java index a9a23d09d..989a2b0e0 100644 --- a/source/de/anomic/kelondro/kelondroCollectionIndex.java +++ b/source/de/anomic/kelondro/kelondroCollectionIndex.java @@ -114,7 +114,7 @@ public class kelondroCollectionIndex { } public kelondroCollectionIndex(File path, String filenameStub, int keyLength, kelondroByteOrder indexOrder, - long preloadTime, int loadfactor, int maxpartitions, kelondroRow rowdef) throws IOException { + int loadfactor, int maxpartitions, kelondroRow rowdef) throws IOException { // the buffersize is number of bytes that are only used if the kelondroFlexTable is backed up with a kelondroTree this.path = path; this.filenameStub = filenameStub; @@ -131,7 +131,7 @@ public class kelondroCollectionIndex { // open index and array files this.arrays = new HashMap(); // all entries will be dynamically created with getArray() - index = openIndexFile(path, filenameStub, indexOrder, preloadTime, loadfactor, rowdef, 0); + index = openIndexFile(path, filenameStub, indexOrder, loadfactor, rowdef, 0); openAllArrayFiles(false, indexOrder); } else { // calculate initialSpace @@ -164,7 +164,7 @@ public class kelondroCollectionIndex { } else if (serverMemory.request(necessaryRAM4fullIndex, false)) { index = new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, initialSpace); } else { - index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRowdef, initialSpace, true); + index = new kelondroFlexTable(path, filenameStub + ".index", indexRowdef, initialSpace, true); } // open array files @@ -264,14 +264,14 @@ public class kelondroCollectionIndex { } private kelondroIndex openIndexFile(File path, String filenameStub, kelondroByteOrder indexOrder, - long preloadTime, int loadfactor, kelondroRow rowdef, int initialSpace) throws IOException { + int loadfactor, kelondroRow rowdef, int initialSpace) throws IOException { // open/create index table File f = new File(path, filenameStub + ".index"); kelondroRow indexRowdef = indexRow(keylength, indexOrder); if (f.isDirectory()) { // use a flextable - kelondroIndex theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRowdef, initialSpace, true)); + kelondroIndex theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", indexRowdef, initialSpace, true)); // save/check property file for this array File propfile = propertyFile(path, filenameStub, loadfactor, rowdef.objectsize); @@ -1087,12 +1087,11 @@ public class kelondroCollectionIndex { File path = new File(args[0]); String filenameStub = args[1]; - long preloadTime = 10000; try { // initialize collection index kelondroCollectionIndex collectionIndex = new kelondroCollectionIndex( path, filenameStub, 9 /*keyLength*/, - kelondroNaturalOrder.naturalOrder, preloadTime, + kelondroNaturalOrder.naturalOrder, 4 /*loadfactor*/, 7, rowdef); // fill index with values @@ -1120,7 +1119,7 @@ public class kelondroCollectionIndex { // printout of index collectionIndex.close(); - kelondroFlexTable index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, kelondroCollectionIndex.indexRow(9, kelondroNaturalOrder.naturalOrder), 0, true); + kelondroFlexTable index = new kelondroFlexTable(path, filenameStub + ".index", kelondroCollectionIndex.indexRow(9, kelondroNaturalOrder.naturalOrder), 0, true); index.print(); index.close(); } catch (IOException e) { diff --git a/source/de/anomic/kelondro/kelondroDyn.java b/source/de/anomic/kelondro/kelondroDyn.java index d48dce0b0..ec67f0b29 100644 --- a/source/de/anomic/kelondro/kelondroDyn.java +++ b/source/de/anomic/kelondro/kelondroDyn.java @@ -71,14 +71,14 @@ public class kelondroDyn { private kelondroObjectBuffer buffer; private kelondroRow rowdef; - public kelondroDyn(File file, boolean useNodeCache, boolean useObjectCache, long preloadTime, int key, + public kelondroDyn(File file, boolean useNodeCache, boolean useObjectCache, int key, int nodesize, char fillChar, kelondroByteOrder objectOrder, boolean usetree, boolean writebuffer, boolean resetOnFail) { // creates or opens a dynamic tree rowdef = new kelondroRow("byte[] key-" + (key + counterlen) + ", byte[] node-" + nodesize, objectOrder, 0); kelondroIndex fbi; if (usetree) { try { - fbi = new kelondroTree(file, useNodeCache, preloadTime, rowdef, 1, 8); + fbi = new kelondroTree(file, useNodeCache, 0, rowdef, 1, 8); } catch (IOException e) { e.printStackTrace(); if (resetOnFail) { @@ -97,7 +97,7 @@ public class kelondroDyn { } else { if (file.exists()) { if (file.isDirectory()) { - fbi = new kelondroFlexTable(file.getParentFile(), file.getName(), 10000, rowdef, 0, resetOnFail); + fbi = new kelondroFlexTable(file.getParentFile(), file.getName(), rowdef, 0, resetOnFail); } else { fbi = new kelondroEcoTable(file, rowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize, 0); } @@ -482,7 +482,7 @@ public class kelondroDyn { if (args.length == 1) { // open a db and list keys try { - kelondroDyn kd = new kelondroDyn(new File(args[0]), true, true, 0, 4 ,100, '_', kelondroNaturalOrder.naturalOrder, false, false, true); + kelondroDyn kd = new kelondroDyn(new File(args[0]), true, true, 4 ,100, '_', kelondroNaturalOrder.naturalOrder, false, false, true); System.out.println(kd.sizeDyn() + " elements in DB"); Iterator i = kd.dynKeys(true, false); while (i.hasNext()) @@ -499,7 +499,7 @@ public class kelondroDyn { File f = new File(args[3]); kelondroDyn kd; try { - kd = new kelondroDyn(db, true, true, 0, 80, 200, '_', kelondroNaturalOrder.naturalOrder, false, false, true); + kd = new kelondroDyn(db, true, true, 80, 200, '_', kelondroNaturalOrder.naturalOrder, false, false, true); if (writeFile) kd.readFile(key, f); else diff --git a/source/de/anomic/kelondro/kelondroDynTree.java b/source/de/anomic/kelondro/kelondroDynTree.java index a2012fa25..962978619 100644 --- a/source/de/anomic/kelondro/kelondroDynTree.java +++ b/source/de/anomic/kelondro/kelondroDynTree.java @@ -62,23 +62,21 @@ public class kelondroDynTree { private int maxcountBuffer = 1000, maxsizeBuffer = 100; //private long maxageCache = 60000, cycletimeCache = 10000; private long maxageBuffer = 60000, cycletimeBuffer = 10000; - private long preloadTime = 0; // data structures for the cache and buffer private HashMap buffer; private HashMap cache; private long cycleBuffer; - public kelondroDynTree(File file, long preloadTime, int keylength, int nodesize, kelondroRow rowdef, char fillChar, boolean resetOnFail) { + public kelondroDynTree(File file, int keylength, int nodesize, kelondroRow rowdef, char fillChar, boolean resetOnFail) { // creates or opens a DynTree this.file = file; - this.preloadTime = preloadTime; this.rowdef = rowdef; this.buffer = new HashMap(); this.cache = new HashMap(); //this.cycleCache = Long.MIN_VALUE; this.cycleBuffer = Long.MIN_VALUE; - this.table = new kelondroDyn(file, true, true, preloadTime, keylength, nodesize, fillChar, rowdef.objectOrder, true, false, resetOnFail); + this.table = new kelondroDyn(file, true, true, keylength, nodesize, fillChar, rowdef.objectOrder, true, false, resetOnFail); this.treeRAHandles = new HashMap(); } @@ -106,7 +104,7 @@ public class kelondroDynTree { kelondroRA ra = table.getRA(key); // works always, even with no-existing entry treeRAHandles.put(key, ra); try { - return new kelondroTree(ra, this.file.getCanonicalPath() + "#" + key, true, preloadTime, rowdef, false); + return new kelondroTree(ra, this.file.getCanonicalPath() + "#" + key, true, 0, rowdef, false); } catch (RuntimeException e) { throw new IOException(e.getMessage()); } @@ -116,7 +114,7 @@ public class kelondroDynTree { if (table.existsDyn(key)) { kelondroRA ra = table.getRA(key); treeRAHandles.put(key, ra); - return new kelondroTree(ra, this.file.getCanonicalPath() + "#" + key, true, preloadTime); + return new kelondroTree(ra, this.file.getCanonicalPath() + "#" + key, true, 0); } return null; } @@ -297,10 +295,10 @@ public class kelondroDynTree { System.out.println("start"); File file = new File("D:\\bin\\testDyn.db"); if (file.exists()) { - kelondroDynTree dt = new kelondroDynTree(file, 0, 16, 512, new kelondroRow("byte[] a-10, byte[] b-20, byte[] c-30", kelondroNaturalOrder.naturalOrder, 0), '_', true); + kelondroDynTree dt = new kelondroDynTree(file, 16, 512, new kelondroRow("byte[] a-10, byte[] b-20, byte[] c-30", kelondroNaturalOrder.naturalOrder, 0), '_', true); System.out.println("opened: table keylength=" + dt.table.row().width(0) + ", sectorsize=" + dt.table.row().width(1) + ", " + dt.table.sizeDyn() + " entries."); } else { - kelondroDynTree dt = new kelondroDynTree(file, 0, 16, 512, new kelondroRow("byte[] a-10, byte[] b-20, byte[] c-30", kelondroNaturalOrder.naturalOrder, 0), '_', true); + kelondroDynTree dt = new kelondroDynTree(file, 16, 512, new kelondroRow("byte[] a-10, byte[] b-20, byte[] c-30", kelondroNaturalOrder.naturalOrder, 0), '_', true); String name; kelondroTree t; kelondroRow.Entry line; diff --git a/source/de/anomic/kelondro/kelondroFlexTable.java b/source/de/anomic/kelondro/kelondroFlexTable.java index 12d6644b4..feedbe49b 100644 --- a/source/de/anomic/kelondro/kelondroFlexTable.java +++ b/source/de/anomic/kelondro/kelondroFlexTable.java @@ -47,7 +47,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr protected kelondroBytesIntMap index; private boolean RAMIndex; - public kelondroFlexTable(File path, String tablename, long preloadTime, kelondroRow rowdef, int minimumSpace, boolean resetOnFail) { + public kelondroFlexTable(File path, String tablename, kelondroRow rowdef, int minimumSpace, boolean resetOnFail) { // the buffersize applies to a possible load of the ram-index // the minimumSpace is a initial allocation space for the index; names the number of index slots // if the ram is not sufficient, a tree file is generated @@ -94,13 +94,13 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr if (indexfile.exists()) { // use existing index file System.out.println("*** Using File index " + indexfile); - ki = new kelondroCache(kelondroTree.open(indexfile, true, preloadTime, treeIndexRow(rowdef.width(0), rowdef.objectOrder), 2, 80)); + ki = new kelondroCache(kelondroTree.open(indexfile, true, 0, treeIndexRow(rowdef.width(0), rowdef.objectOrder), 2, 80)); RAMIndex = false; } else { // generate new index file System.out.println("*** Generating File index for " + size() + " entries from " + indexfile); System.out.println("*** Cause: too less RAM (" + serverMemory.available() + " Bytes) configured. Assign at least " + (neededRAM / 1024 / 1024) + " MB more RAM to enable a RAM index."); - ki = initializeTreeIndex(indexfile, preloadTime, rowdef.objectOrder); + ki = initializeTreeIndex(indexfile, 0, rowdef.objectOrder); System.out.println(" -done-"); System.out.println(ki.size() + " entries indexed from " + super.col[0].size() + " keys."); @@ -490,7 +490,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr String name = args[1]; kelondroRow row = new kelondroRow("Cardinal key-4 {b256}, byte[] x-64", kelondroNaturalOrder.naturalOrder, 0); try { - kelondroFlexTable t = new kelondroFlexTable(f, name, 0, row, 0, true); + kelondroFlexTable t = new kelondroFlexTable(f, name, row, 0, true); kelondroRow.Entry entry = row.newEntry(); entry.setCol(0, System.currentTimeMillis()); entry.setCol(1, "dummy".getBytes()); diff --git a/source/de/anomic/kelondro/kelondroMapTable.java b/source/de/anomic/kelondro/kelondroMapTable.java index 89895cdf7..d959ccae9 100644 --- a/source/de/anomic/kelondro/kelondroMapTable.java +++ b/source/de/anomic/kelondro/kelondroMapTable.java @@ -69,22 +69,16 @@ public class kelondroMapTable { declareMaps(tablename, keysize, objectOrder, nodesize, cacheslots, null, null, null, fillChar, resetOnFail); } - public void declareMaps( - String tablename, int keysize, kelondroByteOrder objectOrder, int nodesize, int cacheslots, - String[] sortfields, String[] longaccfields, String[] doubleaccfields, char fillChar, boolean resetOnFail) { - declareMaps(tablename, keysize, objectOrder, nodesize, cacheslots, sortfields, longaccfields, doubleaccfields, fillChar, 0, resetOnFail); - } - public void declareMaps( String tablename, int keysize, kelondroByteOrder objectOrder, int nodesize, int cacheslots, String[] sortfields, String[] longaccfields, String[] doubleaccfields, char fillChar, - long preloadTime, boolean resetOnFail) { + boolean resetOnFail) { if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared twice."); if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared already in other context."); File tablefile = new File(tablesPath, "table." + tablename + ".mdb"); kelondroDyn dyn; if (!(tablefile.exists())) tablefile.getParentFile().mkdirs(); - dyn = new kelondroDyn(tablefile, true, true, preloadTime, keysize, nodesize, fillChar, objectOrder, true, false, resetOnFail); + dyn = new kelondroDyn(tablefile, true, true, keysize, nodesize, fillChar, objectOrder, true, false, resetOnFail); kelondroMapObjects map = new kelondroMapObjects(dyn, cacheslots, sortfields, longaccfields, doubleaccfields, null, null); mTables.put(tablename, map); } diff --git a/source/de/anomic/kelondro/kelondroSplitTable.java b/source/de/anomic/kelondro/kelondroSplitTable.java index 36be1c277..d044fc181 100644 --- a/source/de/anomic/kelondro/kelondroSplitTable.java +++ b/source/de/anomic/kelondro/kelondroSplitTable.java @@ -54,15 +54,15 @@ public class kelondroSplitTable implements kelondroIndex { private String tablename; private kelondroOrder entryOrder; - public kelondroSplitTable(File path, String tablename, long preloadTime, kelondroRow rowdef, boolean resetOnFail) { + public kelondroSplitTable(File path, String tablename, kelondroRow rowdef, boolean resetOnFail) { this.path = path; this.tablename = tablename; this.rowdef = rowdef; this.entryOrder = new kelondroRow.EntryComparator(rowdef.objectOrder); - init(preloadTime, resetOnFail); + init(resetOnFail); } - public void init(long preloadTime, boolean resetOnFail) { + public void init(boolean resetOnFail) { // initialized tables map this.tables = new HashMap(); @@ -117,7 +117,7 @@ public class kelondroSplitTable implements kelondroIndex { f = new File(path, maxf); if (f.isDirectory()) { // this is a kelonodroFlex table - table = new kelondroCache(new kelondroFlexTable(path, maxf, preloadTime, rowdef, 0, resetOnFail)); + table = new kelondroCache(new kelondroFlexTable(path, maxf, rowdef, 0, resetOnFail)); } else { table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize, 0); } @@ -134,7 +134,7 @@ public class kelondroSplitTable implements kelondroIndex { if (f.isDirectory()) kelondroFlexTable.delete(path, l[i]); else f.delete(); } } - init(-1, true); + init(true); } public String filename() { @@ -226,7 +226,7 @@ public class kelondroSplitTable implements kelondroIndex { if (f.exists()) { if (f.isDirectory()) { // open a flex table - table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef, 0, true); + table = new kelondroFlexTable(path, tablename + "." + suffix, rowdef, 0, true); } else { // open a eco table table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0); @@ -238,7 +238,7 @@ public class kelondroSplitTable implements kelondroIndex { table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0); } else { // use the flex table - table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef, 0, true); + table = new kelondroFlexTable(path, tablename + "." + suffix, rowdef, 0, true); } } tables.put(suffix, table); @@ -276,7 +276,7 @@ public class kelondroSplitTable implements kelondroIndex { table = new kelondroEcoTable(new File(path, tablename + "." + suffix), rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0); } else { // use the flex table - table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef, 0, true); + table = new kelondroFlexTable(path, tablename + "." + suffix, rowdef, 0, true); } tables.put(suffix, table); } diff --git a/source/de/anomic/plasma/dbImport/plasmaCrawlNURLImporter.java b/source/de/anomic/plasma/dbImport/plasmaCrawlNURLImporter.java index 856f6a70b..dc3188ad3 100644 --- a/source/de/anomic/plasma/dbImport/plasmaCrawlNURLImporter.java +++ b/source/de/anomic/plasma/dbImport/plasmaCrawlNURLImporter.java @@ -106,7 +106,7 @@ public class plasmaCrawlNURLImporter extends AbstractImporter implements dbImpor // init profile DB this.log.logInfo("Initializing the source profileDB"); - this.importProfileDB = new plasmaCrawlProfile(profileDbFile, 300); + this.importProfileDB = new plasmaCrawlProfile(profileDbFile); } @SuppressWarnings("unchecked") diff --git a/source/de/anomic/plasma/dbImport/plasmaDbImporter.java b/source/de/anomic/plasma/dbImport/plasmaDbImporter.java index 735535a78..5eeefd70f 100644 --- a/source/de/anomic/plasma/dbImport/plasmaDbImporter.java +++ b/source/de/anomic/plasma/dbImport/plasmaDbImporter.java @@ -85,8 +85,6 @@ public class plasmaDbImporter extends AbstractImporter implements dbImporter { this.cacheSize = Integer.valueOf((String)initParams.get("cacheSize")).intValue(); if (this.cacheSize < 2*1024*1024) this.cacheSize = 8*1024*1024; - this.preloadTime = Long.valueOf((String)initParams.get("preloadTime")).longValue(); - // configure import DB String errorMsg = null; if (!this.importPrimaryPath.exists()) errorMsg = "Primary Import directory does not exist."; @@ -107,7 +105,7 @@ public class plasmaDbImporter extends AbstractImporter implements dbImporter { } this.log.logFine("Initializing source word index db."); - this.importWordIndex = new plasmaWordIndex(this.importPrimaryPath, this.importSecondaryPath, preloadTime / 2, this.log); + this.importWordIndex = new plasmaWordIndex(this.importPrimaryPath, this.importSecondaryPath, this.log); this.importStartSize = this.importWordIndex.size(); } diff --git a/source/de/anomic/plasma/plasmaCrawlLURL.java b/source/de/anomic/plasma/plasmaCrawlLURL.java index 134456d6d..4b405a510 100644 --- a/source/de/anomic/plasma/plasmaCrawlLURL.java +++ b/source/de/anomic/plasma/plasmaCrawlLURL.java @@ -97,10 +97,10 @@ public final class plasmaCrawlLURL { // the class object private kelondroIndex urlIndexFile; - public plasmaCrawlLURL(File indexPath, long preloadTime) { + public plasmaCrawlLURL(File indexPath) { super(); - urlIndexFile = new kelondroSplitTable(new File(indexPath, "PUBLIC/TEXT"), "urls", preloadTime, indexURLEntry.rowdef, false); + urlIndexFile = new kelondroSplitTable(new File(indexPath, "PUBLIC/TEXT"), "urls", indexURLEntry.rowdef, false); // init result stacks externResultStack = new LinkedList(); @@ -676,7 +676,7 @@ public final class plasmaCrawlLURL { } catch (MalformedURLException e) {} if (args[0].equals("-l")) try { // arg 1 is path to URLCache - final plasmaCrawlLURL urls = new plasmaCrawlLURL(new File(args[2]), 0); + final plasmaCrawlLURL urls = new plasmaCrawlLURL(new File(args[2])); final Iterator enu = urls.entries(true, null); while (enu.hasNext()) { System.out.println(enu.next().toString()); diff --git a/source/de/anomic/plasma/plasmaCrawlProfile.java b/source/de/anomic/plasma/plasmaCrawlProfile.java index 1bfd12a96..95d63cd78 100644 --- a/source/de/anomic/plasma/plasmaCrawlProfile.java +++ b/source/de/anomic/plasma/plasmaCrawlProfile.java @@ -64,13 +64,11 @@ public class plasmaCrawlProfile { private kelondroMapObjects profileTable; private File profileTableFile; - private long preloadTime; - public plasmaCrawlProfile(File file, long preloadTime) { + public plasmaCrawlProfile(File file) { this.profileTableFile = file; - this.preloadTime = preloadTime; profileTableFile.getParentFile().mkdirs(); - kelondroDyn dyn = new kelondroDyn(profileTableFile, true, true, preloadTime, yacySeedDB.commonHashLength, 2000, '#', kelondroNaturalOrder.naturalOrder, false, false, true); + kelondroDyn dyn = new kelondroDyn(profileTableFile, true, true, yacySeedDB.commonHashLength, 2000, '#', kelondroNaturalOrder.naturalOrder, false, false, true); profileTable = new kelondroMapObjects(dyn, 500); } @@ -79,7 +77,7 @@ public class plasmaCrawlProfile { if (profileTable != null) profileTable.close(); if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database"); profileTableFile.getParentFile().mkdirs(); - kelondroDyn dyn = new kelondroDyn(profileTableFile, true, true, preloadTime, yacySeedDB.commonHashLength, 2000, '#', kelondroNaturalOrder.naturalOrder, false, false, true); + kelondroDyn dyn = new kelondroDyn(profileTableFile, true, true, yacySeedDB.commonHashLength, 2000, '#', kelondroNaturalOrder.naturalOrder, false, false, true); profileTable = new kelondroMapObjects(dyn, 500); } diff --git a/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java b/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java index dc067d6d1..60e761ae0 100644 --- a/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java +++ b/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java @@ -65,13 +65,11 @@ public class plasmaCrawlRobotsTxt { kelondroMapObjects robotsTable; private final File robotsTableFile; - private long preloadTime; - public plasmaCrawlRobotsTxt(File robotsTableFile, long preloadTime) { + public plasmaCrawlRobotsTxt(File robotsTableFile) { this.robotsTableFile = robotsTableFile; - this.preloadTime = preloadTime; robotsTableFile.getParentFile().mkdirs(); - robotsTable = new kelondroMapObjects(new kelondroDyn(robotsTableFile, true, true, preloadTime, 256, 512, '_', kelondroNaturalOrder.naturalOrder, false, false, true), 100); + robotsTable = new kelondroMapObjects(new kelondroDyn(robotsTableFile, true, true, 256, 512, '_', kelondroNaturalOrder.naturalOrder, false, false, true), 100); } private void resetDatabase() { @@ -79,7 +77,7 @@ public class plasmaCrawlRobotsTxt { if (robotsTable != null) robotsTable.close(); if (!(robotsTableFile.delete())) throw new RuntimeException("cannot delete robots.txt database"); robotsTableFile.getParentFile().mkdirs(); - robotsTable = new kelondroMapObjects(new kelondroDyn(robotsTableFile, true, true, preloadTime, 256, 512, '_', kelondroNaturalOrder.naturalOrder, false, false, true), 100); + robotsTable = new kelondroMapObjects(new kelondroDyn(robotsTableFile, true, true, 256, 512, '_', kelondroNaturalOrder.naturalOrder, false, false, true), 100); } public void close() { diff --git a/source/de/anomic/plasma/plasmaCrawlStacker.java b/source/de/anomic/plasma/plasmaCrawlStacker.java index b8056c7f8..d54827867 100644 --- a/source/de/anomic/plasma/plasmaCrawlStacker.java +++ b/source/de/anomic/plasma/plasmaCrawlStacker.java @@ -85,7 +85,6 @@ public final class plasmaCrawlStacker extends Thread { private final LinkedList urlEntryHashCache; private kelondroIndex urlEntryCache; private File cacheStacksPath; - private long preloadTime; private int dbtype; private boolean prequeue; private long dnsHit, dnsMiss; @@ -95,7 +94,7 @@ public final class plasmaCrawlStacker extends Thread { // objects for the prefetch task private ArrayList dnsfetchHosts = new ArrayList(); - public plasmaCrawlStacker(plasmaSwitchboard sb, File dbPath, long preloadTime, int dbtype, boolean prequeue) { + public plasmaCrawlStacker(plasmaSwitchboard sb, File dbPath, int dbtype, boolean prequeue) { this.sb = sb; this.prequeue = prequeue; this.dnsHit = 0; @@ -107,7 +106,6 @@ public final class plasmaCrawlStacker extends Thread { // create a stack for newly entered entries this.cacheStacksPath = dbPath; - this.preloadTime = preloadTime; this.dbtype = dbtype; openDB(); @@ -321,7 +319,7 @@ public final class plasmaCrawlStacker extends Thread { if (this.dbtype == QUEUE_DB_TYPE_TREE) { File cacheFile = new File(cacheStacksPath, stackfile); cacheFile.getParentFile().mkdirs(); - this.urlEntryCache = new kelondroCache(kelondroTree.open(cacheFile, true, preloadTime, plasmaCrawlEntry.rowdef)); + this.urlEntryCache = new kelondroCache(kelondroTree.open(cacheFile, true, 0, plasmaCrawlEntry.rowdef)); } } diff --git a/source/de/anomic/plasma/plasmaHTCache.java b/source/de/anomic/plasma/plasmaHTCache.java index e46be8e6f..b4230f6a0 100644 --- a/source/de/anomic/plasma/plasmaHTCache.java +++ b/source/de/anomic/plasma/plasmaHTCache.java @@ -229,7 +229,7 @@ public final class plasmaHTCache { return doctype; } - public static void init(File htCachePath, long CacheSizeMax, long preloadTime, String layout, boolean migration) { + public static void init(File htCachePath, long CacheSizeMax, String layout, boolean migration) { cachePath = htCachePath; cacheLayout = layout; @@ -287,7 +287,7 @@ public final class plasmaHTCache { } // open the response header database - openResponseHeaderDB(preloadTime); + openResponseHeaderDB(); // start the cache startup thread // this will collect information about the current cache size and elements @@ -302,13 +302,13 @@ public final class plasmaHTCache { if (responseHeaderDB != null) responseHeaderDB.close(); File dbfile = new File(cachePath, DB_NAME); if (dbfile.exists()) dbfile.delete(); - openResponseHeaderDB(0); + openResponseHeaderDB(); } - private static void openResponseHeaderDB(long preloadTime) { + private static void openResponseHeaderDB() { // open the response header database File dbfile = new File(cachePath, DB_NAME); - responseHeaderDB = new kelondroMapObjects(new kelondroDyn(dbfile, true, true, preloadTime, yacySeedDB.commonHashLength, 150, '#', kelondroBase64Order.enhancedCoder, false, false, true), 500); + responseHeaderDB = new kelondroMapObjects(new kelondroDyn(dbfile, true, true, yacySeedDB.commonHashLength, 150, '#', kelondroBase64Order.enhancedCoder, false, false, true), 500); } private static void deleteOldHTCache(File directory) { diff --git a/source/de/anomic/plasma/plasmaRankingCRProcess.java b/source/de/anomic/plasma/plasmaRankingCRProcess.java index 23c78e740..9ccc220cf 100644 --- a/source/de/anomic/plasma/plasmaRankingCRProcess.java +++ b/source/de/anomic/plasma/plasmaRankingCRProcess.java @@ -260,8 +260,8 @@ public class plasmaRankingCRProcess { kelondroCollectionIndex newseq = null; if (newdb) { File path = to_file.getParentFile(); // path to storage place - newacc = new kelondroFlexTable(path, CRG_accname, -1, CRG_accrow, 0, false); - newseq = new kelondroCollectionIndex(path, CRG_seqname, 12, kelondroBase64Order.enhancedCoder, -1, 2, 9, CRG_colrow); + newacc = new kelondroFlexTable(path, CRG_accname, CRG_accrow, 0, false); + newseq = new kelondroCollectionIndex(path, CRG_seqname, 12, kelondroBase64Order.enhancedCoder, 2, 9, CRG_colrow); } else { if (!(to_file.exists())) { acc = new kelondroAttrSeq("Global Ranking Accumulator File", @@ -390,8 +390,8 @@ public class plasmaRankingCRProcess { public static int genrcix(File cr_path_in, File rci_path_out) throws IOException { //kelondroFlexTable acc = new kelondroFlexTable(cr_path_in, CRG_accname, kelondroBase64Order.enhancedCoder, 128 * 1024 * 1024, -1, CRG_accrow, true); - kelondroCollectionIndex seq = new kelondroCollectionIndex(cr_path_in, CRG_seqname, 12, kelondroBase64Order.enhancedCoder, -1, 2, 9, CRG_colrow); - kelondroCollectionIndex rci = new kelondroCollectionIndex(rci_path_out, RCI_colname, 6, kelondroBase64Order.enhancedCoder, -1, 2, 9, RCI_coli); + kelondroCollectionIndex seq = new kelondroCollectionIndex(cr_path_in, CRG_seqname, 12, kelondroBase64Order.enhancedCoder, 2, 9, CRG_colrow); + kelondroCollectionIndex rci = new kelondroCollectionIndex(rci_path_out, RCI_colname, 6, kelondroBase64Order.enhancedCoder, 2, 9, RCI_coli); // loop over all referees int count = 0; diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index df6e67d3a..5110fd93c 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -947,12 +947,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser this.acceptGlobalURLs = "global.any".indexOf(getConfig("network.unit.domain", "global")) >= 0; this.acceptLocalURLs = "local.any".indexOf(getConfig("network.unit.domain", "global")) >= 0; - // start yacy core - log.logConfig("Starting YaCy Protocol Core"); - this.yc = new yacyCore(this); - serverInstantThread.oneTimeJob(yacyCore.peerActions, "loadSeedLists", yacyCore.log, 0); - long startedSeedListAquisition = System.currentTimeMillis(); - // load values from configs this.plasmaPath = getConfigPath(DBPATH, DBPATH_DEFAULT); this.log.logConfig("Plasma DB Path: " + this.plasmaPath.toString()); @@ -970,6 +964,16 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser this.workPath = getConfigPath(WORK_PATH, WORK_PATH_DEFAULT); this.log.logConfig("Work Path: " + this.workPath.toString()); + // start indexing management + log.logConfig("Starting Indexing Management"); + wordIndex = new plasmaWordIndex(indexPrimaryPath, indexSecondaryPath, log); + + // start yacy core + log.logConfig("Starting YaCy Protocol Core"); + this.yc = new yacyCore(this); + serverInstantThread.oneTimeJob(yacyCore.peerActions, "loadSeedLists", yacyCore.log, 0); + long startedSeedListAquisition = System.currentTimeMillis(); + // set up local robots.txt this.robotstxtConfig = httpdRobotsTxtConfig.init(this); @@ -1040,41 +1044,17 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser if (YBRPath.exists()) { plasmaSearchRankingProcess.loadYBR(YBRPath, 15); } - - // read memory amount - long ramLURL_time = getConfigLong(RAM_CACHE_LURL_TIME, 1000); - long ramNURL_time = getConfigLong(RAM_CACHE_NURL_TIME, 1000); - long ramEURL_time = getConfigLong(RAM_CACHE_EURL_TIME, 1000); - long ramRWI_time = getConfigLong(RAM_CACHE_RWI_TIME, 1000); - long ramHTTP_time = getConfigLong(RAM_CACHE_HTTP_TIME, 1000); - long ramMessage_time = getConfigLong(RAM_CACHE_MESSAGE_TIME, 1000); - long ramRobots_time = getConfigLong(RAM_CACHE_ROBOTS_TIME, 1000); - long ramProfiles_time= getConfigLong(RAM_CACHE_PROFILES_TIME, 1000); - long ramPreNURL_time = getConfigLong(RAM_CACHE_PRE_NURL_TIME, 1000); - long ramWiki_time = getConfigLong(RAM_CACHE_WIKI_TIME, 1000); - long ramBlog_time = getConfigLong(RAM_CACHE_BLOG_TIME, 1000); - this.log.logConfig("LURL preloadTime = " + ramLURL_time); - this.log.logConfig("NURL preloadTime = " + ramNURL_time); - this.log.logConfig("EURL preloadTime = " + ramEURL_time); - this.log.logConfig("RWI preloadTime = " + ramRWI_time); - this.log.logConfig("HTTP preloadTime = " + ramHTTP_time); - this.log.logConfig("Message preloadTime = " + ramMessage_time); - this.log.logConfig("Wiki preloadTime = " + ramWiki_time); - this.log.logConfig("Blog preloadTime = " + ramBlog_time); - this.log.logConfig("Robots preloadTime = " + ramRobots_time); - this.log.logConfig("Profiles preloadTime = " + ramProfiles_time); - this.log.logConfig("PreNURL preloadTime = " + ramPreNURL_time); // make crawl profiles database and default profiles this.log.logConfig("Initializing Crawl Profiles"); File profilesActiveFile = new File(this.plasmaPath, DBFILE_ACTIVE_CRAWL_PROFILES); - this.profilesActiveCrawls = new plasmaCrawlProfile(profilesActiveFile, ramProfiles_time); + this.profilesActiveCrawls = new plasmaCrawlProfile(profilesActiveFile); initActiveCrawlProfiles(); log.logConfig("Loaded active crawl profiles from file " + profilesActiveFile.getName() + ", " + this.profilesActiveCrawls.size() + " entries" + ", " + ppRamString(profilesActiveFile.length()/1024)); File profilesPassiveFile = new File(this.plasmaPath, DBFILE_PASSIVE_CRAWL_PROFILES); - this.profilesPassiveCrawls = new plasmaCrawlProfile(profilesPassiveFile, ramProfiles_time); + this.profilesPassiveCrawls = new plasmaCrawlProfile(profilesPassiveFile); log.logConfig("Loaded passive crawl profiles from file " + profilesPassiveFile.getName() + ", " + this.profilesPassiveCrawls.size() + " entries" + ", " + ppRamString(profilesPassiveFile.length()/1024)); @@ -1082,7 +1062,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // loading the robots.txt db this.log.logConfig("Initializing robots.txt DB"); File robotsDBFile = new File(this.plasmaPath, DBFILE_CRAWL_ROBOTS); - robots = new plasmaCrawlRobotsTxt(robotsDBFile, ramRobots_time); + robots = new plasmaCrawlRobotsTxt(robotsDBFile); this.log.logConfig("Loaded robots.txt DB from file " + robotsDBFile.getName() + ", " + robots.size() + " entries" + ", " + ppRamString(robotsDBFile.length()/1024)); @@ -1096,7 +1076,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser long maxCacheSize = 1024 * 1024 * Long.parseLong(getConfig(PROXY_CACHE_SIZE, "2")); // this is megabyte String cacheLayout = getConfig(PROXY_CACHE_LAYOUT, PROXY_CACHE_LAYOUT_TREE); boolean cacheMigration = getConfigBool(PROXY_CACHE_MIGRATION, true); - plasmaHTCache.init(htCachePath, maxCacheSize, ramHTTP_time, cacheLayout, cacheMigration); + plasmaHTCache.init(htCachePath, maxCacheSize, cacheLayout, cacheMigration); // create the release download directory releasePath = getConfigPath(RELEASE_PATH, RELEASE_PATH_DEFAULT); @@ -1104,18 +1084,18 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser this.log.logInfo("RELEASE Path = " + releasePath.getAbsolutePath()); // starting message board - initMessages(ramMessage_time); + initMessages(); // starting wiki - initWiki(ramWiki_time); + initWiki(); //starting blog - initBlog(ramBlog_time); + initBlog(); // Init User DB this.log.logConfig("Loading User DB"); File userDbFile = new File(getRootPath(), DBFILE_USER); - this.userDB = new userDB(userDbFile, 2000); + this.userDB = new userDB(userDbFile); this.log.logConfig("Loaded User DB from file " + userDbFile.getName() + ", " + this.userDB.size() + " entries" + ", " + ppRamString(userDbFile.length()/1024)); @@ -1123,10 +1103,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser //Init bookmarks DB initBookmarks(); - // start indexing management - log.logConfig("Starting Indexing Management"); - wordIndex = new plasmaWordIndex(indexPrimaryPath, indexSecondaryPath, ramRWI_time, log); - // set a high maximum cache size to current size; this is adopted later automatically int wordCacheMaxCount = Math.max((int) getConfigLong(WORDCACHE_INIT_COUNT, 30000), (int) getConfigLong(WORDCACHE_MAX_COUNT, 20000)); @@ -1285,7 +1261,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser } // initializing the stackCrawlThread - this.crawlStacker = new plasmaCrawlStacker(this, this.plasmaPath, ramPreNURL_time, (int) getConfigLong("tableTypeForPreNURL", 0), (((int) getConfigLong("tableTypeForPreNURL", 0) == 0) && (getConfigLong(CRAWLSTACK_BUSYSLEEP, 0) <= 100))); + this.crawlStacker = new plasmaCrawlStacker(this, this.plasmaPath, (int) getConfigLong("tableTypeForPreNURL", 0), (((int) getConfigLong("tableTypeForPreNURL", 0) == 0) && (getConfigLong(CRAWLSTACK_BUSYSLEEP, 0) <= 100))); //this.sbStackCrawlThread = new plasmaStackCrawlThread(this,this.plasmaPath,ramPreNURL); //this.sbStackCrawlThread.start(); @@ -1359,34 +1335,34 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser log.logConfig("Finished Switchboard Initialization"); } - public void initMessages(long ramMessage_time) { + public void initMessages() { this.log.logConfig("Starting Message Board"); File messageDbFile = new File(workPath, DBFILE_MESSAGE); - this.messageDB = new messageBoard(messageDbFile, ramMessage_time); + this.messageDB = new messageBoard(messageDbFile); this.log.logConfig("Loaded Message Board DB from file " + messageDbFile.getName() + ", " + this.messageDB.size() + " entries" + ", " + ppRamString(messageDbFile.length()/1024)); } - public void initWiki(long ramWiki_time) { + public void initWiki() { this.log.logConfig("Starting Wiki Board"); File wikiDbFile = new File(workPath, DBFILE_WIKI); - this.wikiDB = new wikiBoard(wikiDbFile, new File(workPath, DBFILE_WIKI_BKP), ramWiki_time); + this.wikiDB = new wikiBoard(wikiDbFile, new File(workPath, DBFILE_WIKI_BKP)); this.log.logConfig("Loaded Wiki Board DB from file " + wikiDbFile.getName() + ", " + this.wikiDB.size() + " entries" + ", " + ppRamString(wikiDbFile.length()/1024)); } - public void initBlog(long ramBlog_time) { + public void initBlog() { this.log.logConfig("Starting Blog"); File blogDbFile = new File(workPath, DBFILE_BLOG); - this.blogDB = new blogBoard(blogDbFile, ramBlog_time); + this.blogDB = new blogBoard(blogDbFile); this.log.logConfig("Loaded Blog DB from file " + blogDbFile.getName() + ", " + this.blogDB.size() + " entries" + ", " + ppRamString(blogDbFile.length()/1024)); File blogCommentDbFile = new File(workPath, DBFILE_BLOGCOMMENTS); - this.blogCommentDB = new blogBoardComments(blogCommentDbFile, ramBlog_time); + this.blogCommentDB = new blogBoardComments(blogCommentDbFile); this.log.logConfig("Loaded Blog-Comment DB from file " + blogCommentDbFile.getName() + ", " + this.blogCommentDB.size() + " entries" + ", " + ppRamString(blogCommentDbFile.length()/1024)); @@ -1397,7 +1373,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser File bookmarksFile = new File(workPath, DBFILE_BOOKMARKS); File tagsFile = new File(workPath, DBFILE_BOOKMARKS_TAGS); File datesFile = new File(workPath, DBFILE_BOOKMARKS_DATES); - this.bookmarksDB = new bookmarksDB(bookmarksFile, tagsFile, datesFile, 2000); + this.bookmarksDB = new bookmarksDB(bookmarksFile, tagsFile, datesFile); this.log.logConfig("Loaded Bookmarks DB from files "+ bookmarksFile.getName()+ ", "+tagsFile.getName()); this.log.logConfig(this.bookmarksDB.tagsSize()+" Tag, "+this.bookmarksDB.bookmarksSize()+" Bookmarks"); } @@ -1575,8 +1551,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser private void resetProfiles() { final File pdb = new File(plasmaPath, DBFILE_ACTIVE_CRAWL_PROFILES); if (pdb.exists()) pdb.delete(); - long ramProfiles_time = getConfigLong(RAM_CACHE_PROFILES_TIME, 1000); - profilesActiveCrawls = new plasmaCrawlProfile(pdb, ramProfiles_time); + profilesActiveCrawls = new plasmaCrawlProfile(pdb); initActiveCrawlProfiles(); } diff --git a/source/de/anomic/plasma/plasmaWordConnotation.java b/source/de/anomic/plasma/plasmaWordConnotation.java index 0f12a7d40..fac2ae6e8 100644 --- a/source/de/anomic/plasma/plasmaWordConnotation.java +++ b/source/de/anomic/plasma/plasmaWordConnotation.java @@ -59,8 +59,8 @@ public class plasmaWordConnotation { private static final int nodesize = 4048; private kelondroDynTree refDB; - public plasmaWordConnotation(File refDBfile, long preloadTime, char fillChar) { - refDB = new kelondroDynTree(refDBfile, preloadTime, wordlength, nodesize, new kelondroRow("byte[] word-" + wordlength + ", Cardinal count-" + countlength, kelondroNaturalOrder.naturalOrder, 0), fillChar, true); + public plasmaWordConnotation(File refDBfile, char fillChar) { + refDB = new kelondroDynTree(refDBfile, wordlength, nodesize, new kelondroRow("byte[] word-" + wordlength + ", Cardinal count-" + countlength, kelondroNaturalOrder.naturalOrder, 0), fillChar, true); } private void addSingleRef(String word, String reference) throws IOException { diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index 3f5fcf62f..a27b52752 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -74,7 +74,7 @@ public final class plasmaWordIndex implements indexRI { private int flushsize; public final plasmaCrawlLURL loadedURL; - public plasmaWordIndex(File indexPrimaryRoot, File indexSecondaryRoot, long preloadTime, serverLog log) { + public plasmaWordIndex(File indexPrimaryRoot, File indexSecondaryRoot, serverLog log) { File textindexcache = new File(indexPrimaryRoot, "PUBLIC/TEXT/RICACHE"); if (!(textindexcache.exists())) textindexcache.mkdirs(); this.dhtOutCache = new indexRAMRI(textindexcache, indexRWIRowEntry.urlEntryRow, wCacheMaxChunk, wCacheMaxAge, "dump1.array", log); @@ -83,10 +83,10 @@ public final class plasmaWordIndex implements indexRI { // create collections storage path File textindexcollections = new File(indexPrimaryRoot, "PUBLIC/TEXT/RICOLLECTION"); if (!(textindexcollections.exists())) textindexcollections.mkdirs(); - this.collections = new indexCollectionRI(textindexcollections, "collection", preloadTime, maxCollectionPartition, indexRWIRowEntry.urlEntryRow); + this.collections = new indexCollectionRI(textindexcollections, "collection", maxCollectionPartition, indexRWIRowEntry.urlEntryRow); // create LURL-db - loadedURL = new plasmaCrawlLURL(indexSecondaryRoot, preloadTime); + loadedURL = new plasmaCrawlLURL(indexSecondaryRoot); // performance settings busyCacheFlush = false; diff --git a/source/de/anomic/yacy/yacyCore.java b/source/de/anomic/yacy/yacyCore.java index 51ba2d253..d4a72fb7a 100644 --- a/source/de/anomic/yacy/yacyCore.java +++ b/source/de/anomic/yacy/yacyCore.java @@ -129,13 +129,11 @@ public class yacyCore { if (!yacyDBPath.exists()) { yacyDBPath.mkdir(); } // create or init seed cache - long memDHT_time = Long.parseLong(switchboard.getConfig("ramCacheDHT_time", "1000")); seedDB = new yacySeedDB( sb, new File(yacyDBPath, "seed2.new.db"), new File(yacyDBPath, "seed2.old.db"), - new File(yacyDBPath, "seed2.pot.db"), - memDHT_time); + new File(yacyDBPath, "seed2.pot.db")); // create or init news database newsPool = new yacyNewsPool(yacyDBPath); diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index 59d231219..2820f14df 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -100,7 +100,6 @@ public final class yacySeedDB { protected File seedActiveDBFile, seedPassiveDBFile, seedPotentialDBFile; protected kelondroMapObjects seedActiveDB, seedPassiveDB, seedPotentialDB; - private long preloadTime; private final plasmaSwitchboard sb; private yacySeed mySeed; // my own seed @@ -111,15 +110,13 @@ public final class yacySeedDB { public yacySeedDB(plasmaSwitchboard sb, File seedActiveDBFile, File seedPassiveDBFile, - File seedPotentialDBFile, - long preloadTime) { + File seedPotentialDBFile) { this.seedActiveDBFile = seedActiveDBFile; this.seedPassiveDBFile = seedPassiveDBFile; this.seedPotentialDBFile = seedPotentialDBFile; this.mySeed = null; // my own seed this.sb = sb; - this.preloadTime = preloadTime; // set up seed database seedActiveDB = openSeedTable(seedActiveDBFile); @@ -224,11 +221,11 @@ public final class yacySeedDB { initializeHandlerMethod = null; } try { - return new kelondroMapObjects(new kelondroDyn(seedDBFile, true, true, preloadTime / 3, commonHashLength, 480, '#', kelondroBase64Order.enhancedCoder, usetree, false, true), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this); + return new kelondroMapObjects(new kelondroDyn(seedDBFile, true, true, commonHashLength, 480, '#', kelondroBase64Order.enhancedCoder, usetree, false, true), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this); } catch (Exception e) { // try again kelondroDyn.delete(seedDBFile, usetree); - return new kelondroMapObjects(new kelondroDyn(seedDBFile, true, true, preloadTime / 3, commonHashLength, 480, '#', kelondroBase64Order.enhancedCoder, usetree, false, true), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this); + return new kelondroMapObjects(new kelondroDyn(seedDBFile, true, true, commonHashLength, 480, '#', kelondroBase64Order.enhancedCoder, usetree, false, true), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this); } } diff --git a/source/migration.java b/source/migration.java index 48e3b3333..f424fd6d2 100644 --- a/source/migration.java +++ b/source/migration.java @@ -163,7 +163,7 @@ public class migration { file.delete(); } catch (IOException e) {} } - sb.initWiki(sb.getConfigLong("ramCacheWiki_time", 1000)); + sb.initWiki(); } @@ -176,7 +176,7 @@ public class migration { serverFileUtils.copy(file, file2); file.delete(); } catch (IOException e) {} - sb.initMessages(sb.getConfigLong("ramCacheMessage_time", 1000)); + sb.initMessages(); } } diff --git a/source/yacy.java b/source/yacy.java index 9297b7d81..83eb2f41b 100644 --- a/source/yacy.java +++ b/source/yacy.java @@ -598,16 +598,16 @@ public final class yacy { log.logInfo("STARTING URL CLEANUP"); // db containing all currently loades urls - plasmaCrawlLURL currentUrlDB = new plasmaCrawlLURL(indexSecondaryRoot, 10000); + plasmaCrawlLURL currentUrlDB = new plasmaCrawlLURL(indexSecondaryRoot); // db used to hold all neede urls - plasmaCrawlLURL minimizedUrlDB = new plasmaCrawlLURL(indexRoot2, 10000); + plasmaCrawlLURL minimizedUrlDB = new plasmaCrawlLURL(indexRoot2); Runtime rt = Runtime.getRuntime(); int cacheMem = (int)(rt.maxMemory() - rt.totalMemory()); if (cacheMem < 2048000) throw new OutOfMemoryError("Not enough memory available to start clean up."); - plasmaWordIndex wordIndex = new plasmaWordIndex(indexPrimaryRoot, indexSecondaryRoot, 10000, log); + plasmaWordIndex wordIndex = new plasmaWordIndex(indexPrimaryRoot, indexSecondaryRoot, log); Iterator indexContainerIterator = wordIndex.wordContainers("AAAAAAAAAAAA", false, false); long urlCounter = 0, wordCounter = 0; @@ -781,7 +781,7 @@ public final class yacy { File root = homePath; File indexroot = new File(root, "DATA/INDEX"); try {serverLog.configureLogging(homePath, new File(homePath, "DATA/LOG/yacy.logging"));} catch (Exception e) {} - plasmaCrawlLURL currentUrlDB = new plasmaCrawlLURL(indexroot, 10000); + plasmaCrawlLURL currentUrlDB = new plasmaCrawlLURL(indexroot); currentUrlDB.urldbcleanup(); currentUrlDB.close(); } @@ -798,7 +798,7 @@ public final class yacy { try { Iterator indexContainerIterator = null; if (resource.equals("all")) { - WordIndex = new plasmaWordIndex(indexPrimaryRoot, indexSecondaryRoot, 3000, log); + WordIndex = new plasmaWordIndex(indexPrimaryRoot, indexSecondaryRoot, log); indexContainerIterator = WordIndex.wordContainers(wordChunkStartHash, false, false); } int counter = 0; @@ -858,7 +858,7 @@ public final class yacy { String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"}; for (int i=0; i < dbFileNames.length; i++) { File dbFile = new File(yacyDBPath,dbFileNames[i]); - kelondroMapObjects db = new kelondroMapObjects(new kelondroDyn(dbFile, true, true, 3000, yacySeedDB.commonHashLength, 480, '#', kelondroBase64Order.enhancedCoder, true, false, true), 500, yacySeedDB.sortFields, yacySeedDB.longaccFields, yacySeedDB.doubleaccFields, null, null); + kelondroMapObjects db = new kelondroMapObjects(new kelondroDyn(dbFile, true, true, yacySeedDB.commonHashLength, 480, '#', kelondroBase64Order.enhancedCoder, true, false, true), 500, yacySeedDB.sortFields, yacySeedDB.longaccFields, yacySeedDB.doubleaccFields, null, null); kelondroMapObjects.mapIterator it; it = db.maps(true, false); diff --git a/yacy.init b/yacy.init index 13b81de55..6b6a146aa 100644 --- a/yacy.init +++ b/yacy.init @@ -167,7 +167,7 @@ htDocsPath = DATA/HTDOCS # The complete path to this file is created by combination with the rootPath # you can set a list of defaults, separated by comma # the first one is priorized -defaultFiles = ConfigBasic.html,index.html,default.html,search.html,console.html,control.html,welcome.html,wiki.html,forum.html,blog.html,email.html,content.html,monitor.html,share.html,dir.html,readme.txt +defaultFiles = ConfigBasic.html,index.html,index.htm,default.html,search.html,console.html,control.html,welcome.html,wiki.html,forum.html,blog.html,email.html,content.html,monitor.html,share.html,dir.html,readme.txt # locale-options: YaCy supports localization. # Web pages for special languages are located in the htLocalePath @@ -595,44 +595,6 @@ cleanup.deletionPublishedNews = true # a cluster value of '2' would be appropriate 80_indexing_cluster=1 -# ram cache init timeout for database files - -# ram cache for collection index -ramCacheRWI_time = 30000 - -# ram cache for responseHeader.db -ramCacheHTTP_time = 1000 - -# ram cache for urlHash.db -ramCacheLURL_time = 10000 - -# ram cache for stack crawl thread db -ramCachePreNURL_time = 3000 - -# ram cache for urlNotice.db -ramCacheNURL_time = 2000 - -# ram cache for urlErr.db -ramCacheEURL_time = 1000 - -# ram cache for seedDBs -ramCacheDHT_time = 1000 - -# ram cache for message.db -ramCacheMessage_time = 500 - -# ram cache for wiki.db -ramCacheWiki_time = 500 - -# ram cache for blog.db -ramCacheBlog_time = 500 - -# ram cache for robotsTxt.db -ramCacheRobots_time = 0 - -# ram cache for crawlProfile.db -ramCacheProfiles_time= 500 - # default memory settings for startup of yacy # is valid in unix/shell and windows environments but # not for first startup of YaCy