diff --git a/source/dbtest.java b/source/dbtest.java index d5326527c..74dc6cc95 100644 --- a/source/dbtest.java +++ b/source/dbtest.java @@ -167,7 +167,7 @@ public class dbtest { if (tablefile.exists()) { table = new kelondroTree(tablefile, buffer); } else { - table = new kelondroTree(tablefile, buffer, new int[]{keylength, valuelength, valuelength}); + table = new kelondroTree(tablefile, buffer, new int[]{keylength, valuelength, valuelength}, true); } } diff --git a/source/de/anomic/data/messageBoard.java b/source/de/anomic/data/messageBoard.java index 925907b57..09bc2af74 100644 --- a/source/de/anomic/data/messageBoard.java +++ b/source/de/anomic/data/messageBoard.java @@ -67,15 +67,19 @@ public class messageBoard { private kelondroMap database = null; private int sn = 0; - public messageBoard(File path, int bufferkb) throws IOException { - new File(path.getParent()).mkdir(); - if (database == null) { - if (path.exists()) - database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400)); - else - database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, categoryLength + dateFormat.length() + 2, recordSize)); - } - sn = 0; + public messageBoard(File path, int bufferkb) { + new File(path.getParent()).mkdir(); + if (database == null) { + if (path.exists()) try { + database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400)); + } catch (IOException e) { + path.delete(); + database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, categoryLength + dateFormat.length() + 2, recordSize, true)); + } else { + database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, categoryLength + dateFormat.length() + 2, recordSize, true)); + } + } + sn = 0; } public int size() { diff --git a/source/de/anomic/data/userDB.java b/source/de/anomic/data/userDB.java index a1d5304e2..ec3e5569d 100644 --- a/source/de/anomic/data/userDB.java +++ b/source/de/anomic/data/userDB.java @@ -58,7 +58,6 @@ import java.lang.NumberFormatException; import de.anomic.kelondro.kelondroDyn; import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroMap; -import de.anomic.server.logging.serverLog; import de.anomic.server.serverCodings; public final class userDB { @@ -72,7 +71,7 @@ public final class userDB { private final serverCodings codings = new serverCodings(true); private HashMap ipUsers = new HashMap(); - public userDB(File userTableFile, int bufferkb) throws IOException { + public userDB(File userTableFile, int bufferkb) { this.userTableFile = userTableFile; this.bufferkb = bufferkb; if (userTableFile.exists()) { @@ -81,11 +80,15 @@ public final class userDB { } catch (kelondroException e) { userTableFile.delete(); userTableFile.getParentFile().mkdirs(); - this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, 128, 256)); + this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, 128, 256, true)); + } catch (IOException e) { + userTableFile.delete(); + userTableFile.getParentFile().mkdirs(); + this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, 128, 256, true)); } } else { userTableFile.getParentFile().mkdirs(); - this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, 128, 256)); + this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, 128, 256, true)); } } @@ -103,12 +106,8 @@ public final class userDB { userTable.close(); } catch (IOException e) {} if (!(userTableFile.delete())) throw new RuntimeException("cannot delete user database"); - try { - userTableFile.getParentFile().mkdirs(); - userTable = new kelondroMap(new kelondroDyn(userTableFile, this.bufferkb, 256, 512)); - } catch (IOException e){ - serverLog.logSevere("PLASMA", "user.resetDatabase", e); - } + userTableFile.getParentFile().mkdirs(); + userTable = new kelondroMap(new kelondroDyn(userTableFile, this.bufferkb, 256, 512, true)); } public void close() { diff --git a/source/de/anomic/data/wikiBoard.java b/source/de/anomic/data/wikiBoard.java index ce63d75fe..da2e348c4 100644 --- a/source/de/anomic/data/wikiBoard.java +++ b/source/de/anomic/data/wikiBoard.java @@ -69,21 +69,27 @@ public class wikiBoard { private kelondroMap bkpbase = null; private HashMap authors = new HashMap(); - public wikiBoard(File actpath, File bkppath, int bufferkb) throws IOException { + public wikiBoard(File actpath, File bkppath, int bufferkb) { new File(actpath.getParent()).mkdir(); - if (datbase == null) { - if (actpath.exists()) - datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb/2 * 0x40)); - else - datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb/2 * 0x400, keyLength, recordSize)); - } - new File(bkppath.getParent()).mkdir(); - if (bkpbase == null) { - if (bkppath.exists()) - bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb/2 * 0x400)); - else - bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb/2 * 0x400, keyLength + dateFormat.length(), recordSize)); - } + if (datbase == null) { + if (actpath.exists()) try { + datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x40)); + } catch (IOException e) { + datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, keyLength, recordSize, true)); + } else { + datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, keyLength, recordSize, true)); + } + } + new File(bkppath.getParent()).mkdir(); + if (bkpbase == null) { + if (bkppath.exists()) try { + bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400)); + } catch (IOException e) { + bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400, keyLength + dateFormat.length(), recordSize, true)); + } else { + bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400, keyLength + dateFormat.length(), recordSize, true)); + } + } } public int sizeOfTwo() { diff --git a/source/de/anomic/kelondro/kelondroArray.java b/source/de/anomic/kelondro/kelondroArray.java index 174db6daf..b31616b45 100644 --- a/source/de/anomic/kelondro/kelondroArray.java +++ b/source/de/anomic/kelondro/kelondroArray.java @@ -55,10 +55,16 @@ public class kelondroArray extends kelondroRecords { private static short thisOHBytes = 0; // our record definition does not need extra bytes private static short thisOHHandles = 0; // and two handles overhead for a double-chained list - public kelondroArray(File file, int[] columns, int intprops) throws IOException { - // this creates a new array - super(file, 0, thisOHBytes, thisOHHandles, columns, intprops, columns.length /*txtProps*/, 80 /*txtPropWidth*/); - for (int i = 0; i < intprops; i++) setHandle(i, new Handle(0)); + public kelondroArray(File file, int[] columns, int intprops, boolean exitOnFail) { + // this creates a new array + super(file, 0, thisOHBytes, thisOHHandles, columns, intprops, columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail); + for (int i = 0; i < intprops; i++) try { + setHandle(i, new Handle(0)); + } catch (IOException e) { + super.logFailure("cannot set handle " + i + " / " + e.getMessage()); + if (exitOnFail) System.exit(-1); + throw new RuntimeException("cannot set handle " + i + " / " + e.getMessage()); + } } public kelondroArray(File file) throws IOException{ @@ -129,7 +135,7 @@ public class kelondroArray extends kelondroRecords { // create File f = new File(args[1]); if (f.exists()) f.delete(); - kelondroArray fm = new kelondroArray(f, new int[]{Integer.parseInt(args[2])}, 2); + kelondroArray fm = new kelondroArray(f, new int[]{Integer.parseInt(args[2])}, 2, true); fm.close(); } else if ((args.length == 2) && (args[0].equals("-v"))) { @@ -157,7 +163,7 @@ public class kelondroArray extends kelondroRecords { if ((args.length == 1) && (args[0].equals("-test"))) { File testfile = new File("test.array"); if (testfile.exists()) testfile.delete(); - kelondroArray fm = new kelondroArray(testfile, new int[]{30, 50}, 9); + kelondroArray fm = new kelondroArray(testfile, new int[]{30, 50}, 9, true); for (int i = 0; i < 100; i++) { fm.set(i, new byte[][]{("name" + i).getBytes(), ("value" + i).getBytes()}); } diff --git a/source/de/anomic/kelondro/kelondroDyn.java b/source/de/anomic/kelondro/kelondroDyn.java index fdd7012e3..9e6673d7f 100644 --- a/source/de/anomic/kelondro/kelondroDyn.java +++ b/source/de/anomic/kelondro/kelondroDyn.java @@ -70,9 +70,9 @@ public class kelondroDyn extends kelondroTree { private int reclen; private int segmentCount; - public kelondroDyn(File file, long buffersize /*bytes*/, int key, int nodesize) throws IOException { + public kelondroDyn(File file, long buffersize /*bytes*/, int key, int nodesize, boolean exitOnFail) { // creates a new dynamic tree - super(file, buffersize, new int[] {key + counterlen, nodesize}, 1, 8); + super(file, buffersize, new int[] {key + counterlen, nodesize}, 1, 8, exitOnFail); this.keylen = columnSize(0) - counterlen; this.reclen = columnSize(1); this.segmentCacheKey = null; @@ -82,7 +82,7 @@ public class kelondroDyn extends kelondroTree { writeSegmentCount(); } - public kelondroDyn(File file, long buffersize) throws IOException{ + public kelondroDyn(File file, long buffersize) throws IOException { // this opens a file with an existing dynamic tree super(file, buffersize); this.keylen = columnSize(0) - counterlen; @@ -199,7 +199,7 @@ public class kelondroDyn extends kelondroTree { } // read from db - byte[][] r = get(key); + final byte[][] r = get(key); if (r == null) return null; // update cache @@ -442,7 +442,7 @@ public class kelondroDyn extends kelondroTree { File f = new File(args[3]); kelondroDyn kd; try { - if (db.exists()) kd = new kelondroDyn(db, 0x100000); else kd = new kelondroDyn(db, 0x100000, 80, 200); + if (db.exists()) kd = new kelondroDyn(db, 0x100000); else kd = new kelondroDyn(db, 0x100000, 80, 200, true); if (writeFile) kd.readFile(key, f); else kd.writeFile(key, f); } catch (IOException e) { System.out.println("ERROR: " + e.toString()); @@ -463,7 +463,7 @@ public class kelondroDyn extends kelondroTree { int steps = 0; while (true) { if (testFile.exists()) testFile.delete(); - tt = new kelondroDyn(testFile, 0, 4 ,100); + tt = new kelondroDyn(testFile, 0, 4 ,100, true); steps = ((int) System.currentTimeMillis() % 7) * (((int) System.currentTimeMillis() + 17) % 11); t = s; d = ""; diff --git a/source/de/anomic/kelondro/kelondroDynTree.java b/source/de/anomic/kelondro/kelondroDynTree.java index bdd822bfb..8e4d6af61 100644 --- a/source/de/anomic/kelondro/kelondroDynTree.java +++ b/source/de/anomic/kelondro/kelondroDynTree.java @@ -69,7 +69,7 @@ public class kelondroDynTree { private Hashtable buffer, cache; private long cycleBuffer; - public kelondroDynTree(File file, long buffersize, int keylength, int nodesize, int[] columns) throws IOException { + public kelondroDynTree(File file, long buffersize, int keylength, int nodesize, int[] columns, boolean exitOnFail) { // creates a new DynTree this.file = file; this.columns = columns; @@ -77,8 +77,8 @@ public class kelondroDynTree { this.cache = new Hashtable(); //this.cycleCache = Long.MIN_VALUE; this.cycleBuffer = Long.MIN_VALUE; - if (file.exists()) throw new IOException("DynTree " + file.toString() + " already exists"); - this.table = new kelondroDyn(file, buffersize, keylength, nodesize); + if (file.exists()) file.delete(); + this.table = new kelondroDyn(file, buffersize, keylength, nodesize, exitOnFail); this.treeRAHandles = new Hashtable(); } @@ -134,7 +134,11 @@ public class kelondroDynTree { if (table.existsDyn(key)) throw new IOException("table " + key + " already exists."); kelondroRA ra = table.getRA(key); // works always, even with no-existing entry treeRAHandles.put(key, ra); - return new kelondroTree(ra, buffersize, columns); + try { + return new kelondroTree(ra, buffersize, columns, false); + } catch (RuntimeException e) { + throw new IOException(e.getMessage()); + } } protected kelondroTree getTree(String key) throws IOException { @@ -348,7 +352,7 @@ public class kelondroDynTree { kelondroDynTree dt = new kelondroDynTree(file, 0x100000L); System.out.println("opened: table keylength=" + dt.table.columnSize(0) + ", sectorsize=" + dt.table.columnSize(1) + ", " + dt.table.size() + " entries."); } else { - kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, 16, 512, new int[] {10,20,30}); + kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, 16, 512, new int[] {10,20,30}, true); String name; kelondroTree t; byte[][] line = new byte[][] {"".getBytes(), "abc".getBytes(), "def".getBytes()}; diff --git a/source/de/anomic/kelondro/kelondroFScoreCluster.java b/source/de/anomic/kelondro/kelondroFScoreCluster.java index 55d668b4a..7f798172d 100644 --- a/source/de/anomic/kelondro/kelondroFScoreCluster.java +++ b/source/de/anomic/kelondro/kelondroFScoreCluster.java @@ -63,17 +63,26 @@ public class kelondroFScoreCluster { private kelondroTree refcountDB; private kelondroTree countrefDB; - public kelondroFScoreCluster(File refcountDBfile, File countrefDBfile) throws IOException { - if ((refcountDBfile.exists()) && (countrefDBfile.exists())) { + public kelondroFScoreCluster(File refcountDBfile, File countrefDBfile, boolean exitOnFail) { + if ((refcountDBfile.exists()) && (countrefDBfile.exists())) try { refcountDB = new kelondroTree(refcountDBfile, 0x100000L); refcountDB.setText(0, serverCodings.enhancedCoder.encodeBase64Long(0, countlength).getBytes()); // counter of all occurrences countrefDB = new kelondroTree(countrefDBfile, 0x100000L); countrefDB.setText(0, serverCodings.enhancedCoder.encodeBase64Long(0, countlength).getBytes()); + } catch (IOException e) { + refcountDBfile.delete(); + countrefDBfile.delete(); + refcountDB = new kelondroTree(refcountDBfile, 0x100000L, new int[] {wordlength, countlength}, 1, countlength, exitOnFail); + countrefDB = new kelondroTree(countrefDBfile, 0x100000L, new int[] {countlength + wordlength, 4}, 1, countlength, exitOnFail); } else if ((!(refcountDBfile.exists())) && (!(countrefDBfile.exists()))) { - refcountDB = new kelondroTree(refcountDBfile, 0x100000L, new int[] {wordlength, countlength}, 1, countlength); - countrefDB = new kelondroTree(countrefDBfile, 0x100000L, new int[] {countlength + wordlength, 4}, 1, countlength); + refcountDB = new kelondroTree(refcountDBfile, 0x100000L, new int[] {wordlength, countlength}, 1, countlength, exitOnFail); + countrefDB = new kelondroTree(countrefDBfile, 0x100000L, new int[] {countlength + wordlength, 4}, 1, countlength, exitOnFail); } else { - throw new IOException("both word/count db files must exists"); + if (exitOnFail) { + System.exit(-1); + } else { + throw new RuntimeException("both word/count db files must exists"); + } } } diff --git a/source/de/anomic/kelondro/kelondroHashtable.java b/source/de/anomic/kelondro/kelondroHashtable.java index 12aee822d..5441011ef 100644 --- a/source/de/anomic/kelondro/kelondroHashtable.java +++ b/source/de/anomic/kelondro/kelondroHashtable.java @@ -144,7 +144,7 @@ public class kelondroHashtable { private static final byte[] dummyKey = serverCodings.enhancedCoder.encodeBase64Long(0, 5).getBytes(); - public kelondroHashtable(File file, int[] columns, int offset, int maxsize, int maxrehash) throws IOException { + public kelondroHashtable(File file, int[] columns, int offset, int maxsize, int maxrehash, boolean exitOnFail) { // this creates a new hashtable // the key element is not part of the columns array // this is unlike the kelondroTree, where the key is part of a row @@ -154,17 +154,23 @@ public class kelondroHashtable { // this number is needed to omit grow of the table in case of re-hashing // the maxsize is re-computed to a virtual folding height and will result in a tablesize // less than the given maxsize. The actual maxsize can be retrieved by maxsize() - this.hashArray = new kelondroArray(file, extCol(columns), 6); + this.hashArray = new kelondroArray(file, extCol(columns), 6, exitOnFail); this.offset = offset; this.maxk = kelondroMSetTools.log2a(maxsize); // equal to |log2(maxsize)| + 1 if (this.maxk >= kelondroMSetTools.log2a(maxsize + power2(offset + 1) + 1) - 1) this.maxk--; this.maxrehash = maxrehash; - hashArray.seti(0, this.offset); - hashArray.seti(1, this.maxk); - hashArray.seti(2, this.maxrehash); dummyRow = new byte[hashArray.columns()][]; dummyRow[0] = dummyKey; - for (int i = 0; i < hashArray.columns(); i++) dummyRow[i] = new byte[0]; + for (int i = 0; i < hashArray.columns(); i++) + try { + hashArray.seti(0, this.offset); + hashArray.seti(1, this.maxk); + hashArray.seti(2, this.maxrehash); + } catch (IOException e) { + hashArray.logFailure("cannot set properties / " + e.getMessage()); + if (exitOnFail) System.exit(-1); + throw new RuntimeException("cannot set properties / " + e.getMessage()); + } } public kelondroHashtable(File file) throws IOException{ diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java index a2c0a0f7a..bac77c337 100644 --- a/source/de/anomic/kelondro/kelondroRecords.java +++ b/source/de/anomic/kelondro/kelondroRecords.java @@ -194,7 +194,8 @@ public class kelondroRecords { public kelondroRecords(File file, long buffersize /* bytes */, short ohbytec, short ohhandlec, - int[] columns, int FHandles, int txtProps, int txtPropWidth) throws IOException { + int[] columns, int FHandles, int txtProps, int txtPropWidth, + boolean exitOnFail) { // creates a new file // file: the file that shall be created // oha : overhead size array of four bytes: oha[0]=# of bytes, oha[1]=# of shorts, oha[2]=# of ints, oha[3]=# of longs, @@ -202,20 +203,32 @@ public class kelondroRecords { // FHandles: number of integer properties // txtProps: number of text properties - assert (!file.exists()): "file " + file + " already exist"; - this.filename = file.getCanonicalPath(); - kelondroRA raf = new kelondroFileRA(this.filename); - //kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100); - // kelondroRA raf = new kelondroNIOFileRA(this.filename, false, 10000); - init(raf, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth); + assert (!file.exists()) : "file " + file + " already exist"; + try { + this.filename = file.getCanonicalPath(); + kelondroRA raf = new kelondroFileRA(this.filename); + // kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100); + // kelondroRA raf = new kelondroNIOFileRA(this.filename, false, 10000); + init(raf, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth); + } catch (IOException e) { + logFailure("cannot create / " + e.getMessage()); + if (exitOnFail) + System.exit(-1); + } initCache(buffersize); } public kelondroRecords(kelondroRA ra, long buffersize /* bytes */, short ohbytec, short ohhandlec, - int[] columns, int FHandles, int txtProps, int txtPropWidth) throws IOException { + int[] columns, int FHandles, int txtProps, int txtPropWidth, + boolean exitOnFail) { this.filename = null; - init(ra, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth); + try { + init(ra, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth); + } catch (IOException e) { + logFailure("cannot create / " + e.getMessage()); + if (exitOnFail) System.exit(-1); + } initCache(buffersize); } @@ -295,6 +308,13 @@ public class kelondroRecords { this.theLogger.warning("KELONDRO WARNING for file " + this.filename + ": " + message); } + public void logFailure(String message) { + if (this.theLogger == null) + System.err.println("KELONDRO FAILURE for file " + this.filename + ": " + message); + else + this.theLogger.severe("KELONDRO FAILURE for file " + this.filename + ": " + message); + } + public void clear() throws IOException { // Removes all mappings from this map // throw new UnsupportedOperationException("clear not supported"); diff --git a/source/de/anomic/kelondro/kelondroStack.java b/source/de/anomic/kelondro/kelondroStack.java index 136bcdbd0..1778311c3 100644 --- a/source/de/anomic/kelondro/kelondroStack.java +++ b/source/de/anomic/kelondro/kelondroStack.java @@ -67,15 +67,21 @@ public final class kelondroStack extends kelondroRecords { private static int root = 0; // pointer for FHandles-array: pointer to root node private static int toor = 1; // pointer for FHandles-array: pointer to root node - public kelondroStack(File file, long buffersize, int key, int value) throws IOException { - this(file, buffersize, new int[] {key, value}); + public kelondroStack(File file, long buffersize, int key, int value, boolean exitOnFail) { + this(file, buffersize, new int[] { key, value }, exitOnFail); } - public kelondroStack(File file, long buffersize, int[] columns) throws IOException { - // this creates a new tree - super(file, buffersize, thisOHBytes, thisOHHandles, columns, thisFHandles, columns.length /*txtProps*/, 80 /*txtPropWidth*/); - setHandle(root, null); // define the root value - setHandle(toor, null); // define the toor value + public kelondroStack(File file, long buffersize, int[] columns, boolean exitOnFail) { + // this creates a new tree + super(file, buffersize, thisOHBytes, thisOHHandles, columns, thisFHandles, columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail); + try { + setHandle(root, null); // define the root value + setHandle(toor, null); // define the toor value + } catch (IOException e) { + super.logFailure("cannot set root/toor handles / " + e.getMessage()); + if (exitOnFail) System.exit(-1); + throw new RuntimeException("cannot set root/toor handles / " + e.getMessage()); + } } public kelondroStack(File file, long buffersize) throws IOException{ @@ -90,7 +96,7 @@ public final class kelondroStack extends kelondroRecords { setHandle(toor, null); // reset the toor value } - public static kelondroStack reset(kelondroStack stack) throws IOException { + public static kelondroStack reset(kelondroStack stack) { // memorize settings to this file File f = new File(stack.filename); long bz = stack.XcacheSize * stack.cacheChunkSize(true); @@ -101,7 +107,7 @@ public final class kelondroStack extends kelondroRecords { if (f.exists()) f.delete(); // re-open a database with same settings as before - return new kelondroStack(f, bz, cols); + return new kelondroStack(f, bz, cols, true); } public class Counter implements Iterator { @@ -410,7 +416,7 @@ public final class kelondroStack extends kelondroRecords { int[] lens = new int[2]; lens[0] = Integer.parseInt(args[1]); lens[1] = Integer.parseInt(args[2]); - kelondroStack fm = new kelondroStack(f, 0x100000, lens); + kelondroStack fm = new kelondroStack(f, 0x100000, lens, true); fm.close(); } else if (args[0].equals("-p")) { kelondroStack fm = new kelondroStack(new File(args[3]), 0x100000); diff --git a/source/de/anomic/kelondro/kelondroTables.java b/source/de/anomic/kelondro/kelondroTables.java index 5a82659a4..f8d3ce836 100644 --- a/source/de/anomic/kelondro/kelondroTables.java +++ b/source/de/anomic/kelondro/kelondroTables.java @@ -61,39 +61,45 @@ public class kelondroTables { if (!(tablesPath.exists())) tablesPath.mkdirs(); } - public void declareMaps(String tablename, int keysize, int nodesize) throws IOException { - declareMaps(tablename, keysize, nodesize, null, null); + public void declareMaps(String tablename, int keysize, int nodesize, boolean exitOnFail) { + declareMaps(tablename, keysize, nodesize, null, null, exitOnFail); } - public void declareMaps(String tablename, int keysize, int nodesize, String[] sortfields, String[] accfields) throws IOException { - declareMaps(tablename, keysize, nodesize, sortfields, accfields, 0x800); + public void declareMaps(String tablename, int keysize, int nodesize, String[] sortfields, String[] accfields, boolean exitOnFail) { + declareMaps(tablename, keysize, nodesize, sortfields, accfields, 0x800, exitOnFail); } - public void declareMaps(String tablename, int keysize, int nodesize, String[] sortfields, String[] accfields, long buffersize /*bytes*/) throws IOException { + public void declareMaps(String tablename, int keysize, int nodesize, String[] sortfields, String[] accfields, long buffersize /*bytes*/, boolean exitOnFail) { if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared twice."); if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared already in other context."); File tablefile = new File(tablesPath, "table." + tablename + ".mdb"); kelondroDyn dyn; - if (tablefile.exists()) { + if (tablefile.exists()) try { dyn = new kelondroDyn(tablefile, buffersize); + } catch (IOException e) { + tablefile.getParentFile().mkdirs(); + dyn = new kelondroDyn(tablefile, buffersize, keysize, nodesize, exitOnFail); } else { tablefile.getParentFile().mkdirs(); - dyn = new kelondroDyn(tablefile, buffersize, keysize, nodesize); + dyn = new kelondroDyn(tablefile, buffersize, keysize, nodesize, exitOnFail); } kelondroMap map = new kelondroMap(dyn, sortfields, accfields); mTables.put(tablename, map); } - public void declareTree(String tablename, int[] columns, long buffersize /*bytes*/) throws IOException { + public void declareTree(String tablename, int[] columns, long buffersize /*bytes*/, boolean exitOnFail) { if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared already in other context."); if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared twice."); File tablefile = new File(tablesPath, "table." + tablename + ".tdb"); kelondroTree Tree; - if (tablefile.exists()) { + if (tablefile.exists()) try { Tree = new kelondroTree(tablefile, buffersize); + } catch (IOException e) { + tablefile.getParentFile().mkdirs(); + Tree = new kelondroTree(tablefile, buffersize, columns, exitOnFail); } else { tablefile.getParentFile().mkdirs(); - Tree = new kelondroTree(tablefile, buffersize, columns); + Tree = new kelondroTree(tablefile, buffersize, columns, exitOnFail); } tTables.put(tablename, Tree); } diff --git a/source/de/anomic/kelondro/kelondroTree.java b/source/de/anomic/kelondro/kelondroTree.java index 9fe900190..c11e34e06 100644 --- a/source/de/anomic/kelondro/kelondroTree.java +++ b/source/de/anomic/kelondro/kelondroTree.java @@ -78,35 +78,47 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr private Search writeSearchObj = new Search(); //private kelondroLock writeLock = new kelondroLock(); - public kelondroTree(File file, long buffersize, int key, int value) throws IOException { - this(file, buffersize, new int[] { key, value }, 1, 8); + public kelondroTree(File file, long buffersize, int key, int value, boolean exitOnFail) { + this(file, buffersize, new int[] { key, value }, 1, 8, exitOnFail); } - public kelondroTree(kelondroRA ra, long buffersize, int key, int value) throws IOException { - this(ra, buffersize, new int[] { key, value }, 1, 8); + public kelondroTree(kelondroRA ra, long buffersize, int key, int value, boolean exitOnFail) { + this(ra, buffersize, new int[] { key, value }, 1, 8, exitOnFail); } - public kelondroTree(File file, long buffersize, int[] columns) throws IOException { + public kelondroTree(File file, long buffersize, int[] columns, boolean exitOnFail) { // this creates a new tree file - this(file, buffersize, columns, columns.length /* txtProps */, 80 /* txtPropWidth */); + this(file, buffersize, columns, columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail); } - public kelondroTree(File file, long buffersize, int[] columns, int txtProps, int txtPropsWidth) throws IOException { + public kelondroTree(File file, long buffersize, int[] columns, int txtProps, int txtPropsWidth, boolean exitOnFail) { // this creates a new tree file - super(file, buffersize, thisOHBytes, thisOHHandles, columns, thisFHandles, txtProps, txtPropsWidth); - setHandle(root, null); // define the root value + super(file, buffersize, thisOHBytes, thisOHHandles, columns, thisFHandles, txtProps, txtPropsWidth, exitOnFail); + try { + setHandle(root, null); // define the root value + } catch (IOException e) { + super.logFailure("cannot set root handle / " + e.getMessage()); + if (exitOnFail) System.exit(-1); + throw new RuntimeException("cannot set root handle / " + e.getMessage()); + } } - public kelondroTree(kelondroRA ra, long buffersize, int[] columns) throws IOException { + public kelondroTree(kelondroRA ra, long buffersize, int[] columns, boolean exitOnFail) { // this creates a new tree within a kelondroRA - this(ra, buffersize, columns, columns.length /* txtProps */, 80 /* txtPropWidth */); + this(ra, buffersize, columns, columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail); } - public kelondroTree(kelondroRA ra, long buffersize, int[] columns, int txtProps, int txtPropsWidth) throws IOException { + public kelondroTree(kelondroRA ra, long buffersize, int[] columns, int txtProps, int txtPropsWidth, boolean exitOnFail) { // this creates a new tree within a kelondroRA super(ra, buffersize, thisOHBytes, thisOHHandles, columns, - thisFHandles, txtProps, txtPropsWidth); - setHandle(root, null); // define the root value + thisFHandles, txtProps, txtPropsWidth, exitOnFail); + try { + setHandle(root, null); // define the root value + } catch (IOException e) { + super.logFailure("cannot set root handle / " + e.getMessage()); + if (exitOnFail) System.exit(-1); + throw new RuntimeException("cannot set root handle / " + e.getMessage()); + } } public kelondroTree(File file, long buffersize) throws IOException { @@ -1185,7 +1197,7 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr // test script File testFile = new File("test.db"); while (testFile.exists()) testFile.delete(); - kelondroTree fm = new kelondroTree(testFile, 0x100000, 4, 4); + kelondroTree fm = new kelondroTree(testFile, 0x100000, 4, 4, true); byte[] dummy = "".getBytes(); fm.put("abc0".getBytes(), dummy); fm.put("bcd0".getBytes(), dummy); fm.put("def0".getBytes(), dummy); fm.put("bab0".getBytes(), dummy); @@ -1263,7 +1275,7 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr int[] lens = new int[2]; lens[0] = Integer.parseInt(args[1]); lens[1] = Integer.parseInt(args[2]); - kelondroTree fm = new kelondroTree(f, 0x100000, lens); + kelondroTree fm = new kelondroTree(f, 0x100000, lens, true); fm.close(); } else if (args[0].equals("-u")) { kelondroTree fm = new kelondroTree(new File(args[3]), 0x100000); @@ -1366,7 +1378,7 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr int steps = 0; while (true) { if (testFile.exists()) testFile.delete(); - tt = new kelondroTree(testFile, 200, 4 ,4); + tt = new kelondroTree(testFile, 200, 4 ,4, true); steps = 10 + ((int) System.currentTimeMillis() % 7) * (((int) System.currentTimeMillis() + 17) % 11); t = s; d = ""; @@ -1432,7 +1444,7 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr File f = new File("test.db"); if (f.exists()) f.delete(); try { - kelondroTree tt = new kelondroTree(f, 0, 4, 4); + kelondroTree tt = new kelondroTree(f, 0, 4, 4, true); byte[] b; b = testWord('B'); tt.put(b, b); //tt.print(); b = testWord('C'); tt.put(b, b); //tt.print(); @@ -1464,7 +1476,7 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr public static kelondroTree testTree(File f, String testentities) throws IOException { if (f.exists()) f.delete(); - kelondroTree tt = new kelondroTree(f, 0, 4, 4); + kelondroTree tt = new kelondroTree(f, 0, 4, 4, true); byte[] b; for (int i = 0; i < testentities.length(); i++) { b = testWord(testentities.charAt(i)); diff --git a/source/de/anomic/plasma/plasmaCrawlBalancer.java b/source/de/anomic/plasma/plasmaCrawlBalancer.java index 463bb9e13..095dfee55 100644 --- a/source/de/anomic/plasma/plasmaCrawlBalancer.java +++ b/source/de/anomic/plasma/plasmaCrawlBalancer.java @@ -56,15 +56,16 @@ public class plasmaCrawlBalancer { private kelondroStack stack; private HashMap domainStacks; - public plasmaCrawlBalancer(File stackFile, long buffersize) throws IOException { - if (stackFile.exists()) + public plasmaCrawlBalancer(File stackFile, long buffersize) { + if (stackFile.exists()) { try { stack = new kelondroStack(stackFile, buffersize); } catch (IOException e) { - stack = new kelondroStack(stackFile, buffersize, new int[] {plasmaURL.urlHashLength}); + stack = new kelondroStack(stackFile, buffersize, new int[] {plasmaURL.urlHashLength}, true); } - else - stack = new kelondroStack(stackFile, buffersize, new int[] {plasmaURL.urlHashLength}); + } else { + stack = new kelondroStack(stackFile, buffersize, new int[] {plasmaURL.urlHashLength}, true); + } domainStacks = new HashMap(); } @@ -74,7 +75,7 @@ public class plasmaCrawlBalancer { stack = null; } - public void reset() throws IOException { + public void reset() { stack = kelondroStack.reset(stack); domainStacks = new HashMap(); } diff --git a/source/de/anomic/plasma/plasmaCrawlEURL.java b/source/de/anomic/plasma/plasmaCrawlEURL.java index f9fb5c78d..0bd012612 100644 --- a/source/de/anomic/plasma/plasmaCrawlEURL.java +++ b/source/de/anomic/plasma/plasmaCrawlEURL.java @@ -60,7 +60,7 @@ public class plasmaCrawlEURL extends plasmaURL { private LinkedList rejectedStack = new LinkedList(); // strings: url - public plasmaCrawlEURL(File cachePath, int bufferkb) throws IOException { + public plasmaCrawlEURL(File cachePath, int bufferkb) { super(); int[] ce = { urlHashLength, // the url's hash @@ -75,14 +75,17 @@ public class plasmaCrawlEURL extends plasmaURL { urlErrorLength, // string describing load failure urlFlagLength // extra space }; - if (cachePath.exists()) { - // open existing cache - urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400); - } else { - // create new cache - cachePath.getParentFile().mkdirs(); - urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, ce); - } + if (cachePath.exists()) try { + // open existing cache + urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400); + } catch (IOException e) { + cachePath.delete(); + urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, ce, true); + } else { + // create new cache + cachePath.getParentFile().mkdirs(); + urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, ce, true); + } } public synchronized Entry newEntry(URL url, String referrer, String initiator, String executor, diff --git a/source/de/anomic/plasma/plasmaCrawlLURL.java b/source/de/anomic/plasma/plasmaCrawlLURL.java index 870571581..b434ec579 100644 --- a/source/de/anomic/plasma/plasmaCrawlLURL.java +++ b/source/de/anomic/plasma/plasmaCrawlLURL.java @@ -87,7 +87,7 @@ public final class plasmaCrawlLURL extends plasmaURL { //public static Set damagedURLS = Collections.synchronizedSet(new HashSet()); - public plasmaCrawlLURL(File cachePath, int bufferkb) throws IOException { + public plasmaCrawlLURL(File cachePath, int bufferkb) { super(); int[] ce = { urlHashLength, @@ -108,11 +108,16 @@ public final class plasmaCrawlLURL extends plasmaURL { for (int i = 0; i < ce.length; i++) { segmentsize += ce[i]; } if (cachePath.exists()) { // open existing cache - urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400); + try { + urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400); + } catch (IOException e) { + cachePath.getParentFile().mkdirs(); + urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, ce, true); + } } else { // create new cache cachePath.getParentFile().mkdirs(); - urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, ce); + urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, ce, true); } // init result stacks diff --git a/source/de/anomic/plasma/plasmaCrawlNURL.java b/source/de/anomic/plasma/plasmaCrawlNURL.java index 5122934c7..b40604238 100644 --- a/source/de/anomic/plasma/plasmaCrawlNURL.java +++ b/source/de/anomic/plasma/plasmaCrawlNURL.java @@ -99,7 +99,7 @@ public class plasmaCrawlNURL extends plasmaURL { private File cacheStacksPath; private int bufferkb; - public plasmaCrawlNURL(File cacheStacksPath, int bufferkb) throws IOException { + public plasmaCrawlNURL(File cacheStacksPath, int bufferkb) { super(); this.cacheStacksPath = cacheStacksPath; this.bufferkb = bufferkb; @@ -123,23 +123,23 @@ public class plasmaCrawlNURL extends plasmaURL { if (imageStackFile.exists()) try { imageStack = new kelondroStack(imageStackFile, 0); } catch (IOException e) { - imageStack = new kelondroStack(imageStackFile, 0, new int[] {plasmaURL.urlHashLength}); + imageStack = new kelondroStack(imageStackFile, 0, new int[] {plasmaURL.urlHashLength}, true); } else { - imageStack = new kelondroStack(imageStackFile, 0, new int[] {plasmaURL.urlHashLength}); + imageStack = new kelondroStack(imageStackFile, 0, new int[] {plasmaURL.urlHashLength}, true); } if (movieStackFile.exists()) try { movieStack = new kelondroStack(movieStackFile, 0); } catch (IOException e) { - movieStack = new kelondroStack(movieStackFile, 0, new int[] {plasmaURL.urlHashLength}); + movieStack = new kelondroStack(movieStackFile, 0, new int[] {plasmaURL.urlHashLength}, true); } else { - movieStack = new kelondroStack(movieStackFile, 0, new int[] {plasmaURL.urlHashLength}); + movieStack = new kelondroStack(movieStackFile, 0, new int[] {plasmaURL.urlHashLength}, true); } if (musicStackFile.exists()) try { musicStack = new kelondroStack(musicStackFile, 0); } catch (IOException e) { - musicStack = new kelondroStack(musicStackFile, 0, new int[] {plasmaURL.urlHashLength}); + musicStack = new kelondroStack(musicStackFile, 0, new int[] {plasmaURL.urlHashLength}, true); } else { - musicStack = new kelondroStack(musicStackFile, 0, new int[] {plasmaURL.urlHashLength}); + musicStack = new kelondroStack(musicStackFile, 0, new int[] {plasmaURL.urlHashLength}, true); } // init stack Index @@ -147,19 +147,22 @@ public class plasmaCrawlNURL extends plasmaURL { new initStackIndex().start(); } - private void openHashCache() throws IOException { + private void openHashCache() { File cacheFile = new File(cacheStacksPath, "urlNotice1.db"); - if (cacheFile.exists()) { + if (cacheFile.exists()) try { // open existing cache urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400); + } catch (IOException e) { + cacheFile.delete(); + urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, ce, true); } else { // create new cache cacheFile.getParentFile().mkdirs(); - urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, ce); + urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, ce, true); } } - private void resetHashCache() throws IOException { + private void resetHashCache() { if (urlHashCache != null) { try {urlHashCache.close();} catch (IOException e) {} urlHashCache = null; @@ -186,14 +189,47 @@ public class plasmaCrawlNURL extends plasmaURL { public void run() { Iterator i; try { - try { i = coreStack.iterator(); while (i.hasNext()) stackIndex.add(new String((byte[]) i.next()));} catch (Exception e) {coreStack.reset();} - try { i = limitStack.iterator(); while (i.hasNext()) stackIndex.add(new String((byte[]) i.next()));} catch (Exception e) {limitStack.reset();} - try { i = overhangStack.iterator(); while (i.hasNext()) stackIndex.add(new String((byte[]) i.next()));} catch (Exception e) {overhangStack.reset();} - try { i = remoteStack.iterator(); while (i.hasNext()) stackIndex.add(new String((byte[]) i.next()));} catch (Exception e) {remoteStack.reset();} - try { i = imageStack.iterator(); while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey()));} catch (Exception e) {imageStack = kelondroStack.reset(imageStack);} - try { i = movieStack.iterator(); while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey()));} catch (Exception e) {movieStack = kelondroStack.reset(movieStack);} - try { i = musicStack.iterator(); while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey()));} catch (Exception e) {musicStack = kelondroStack.reset(musicStack);} - } catch (IOException e) {} + i = coreStack.iterator(); + while (i.hasNext()) stackIndex.add(new String((byte[]) i.next())); + } catch (Exception e) { + coreStack.reset(); + } + try { + i = limitStack.iterator(); + while (i.hasNext()) stackIndex.add(new String((byte[]) i.next())); + } catch (Exception e) { + limitStack.reset(); + } + try { + i = overhangStack.iterator(); + while (i.hasNext()) stackIndex.add(new String((byte[]) i.next())); + } catch (Exception e) { + overhangStack.reset(); + } + try { + i = remoteStack.iterator(); + while (i.hasNext()) stackIndex.add(new String((byte[]) i.next())); + } catch (Exception e) { + remoteStack.reset(); + } + try { + i = imageStack.iterator(); + while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey())); + } catch (Exception e) { + imageStack = kelondroStack.reset(imageStack); + } + try { + i = movieStack.iterator(); + while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey())); + } catch (Exception e) { + movieStack = kelondroStack.reset(movieStack); + } + try { + i = musicStack.iterator(); + while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey())); + } catch (Exception e) { + musicStack = kelondroStack.reset(musicStack); + } } } @@ -500,21 +536,11 @@ public class plasmaCrawlNURL extends plasmaURL { } catch (IOException e) { serverLog.logSevere("PLASMA", "INTERNAL ERROR AT plasmaNURL:store:" + e.toString() + ", resetting NURL-DB"); e.printStackTrace(); - try { - resetHashCache(); - } catch (IOException ee) { - serverLog.logSevere("PLASMA", "INTERNAL ERROR AT plasmaNURL:store:" + e.toString() + ", reset of NURL-DB failed"); - System.exit(0); // this is REALLY severe - } + resetHashCache(); } catch (kelondroException e) { serverLog.logSevere("PLASMA", "plasmaCrawlNURL.store failed: " + e.toString() + ", resetting NURL-DB"); e.printStackTrace(); - try { - resetHashCache(); - } catch (IOException ee) { - serverLog.logSevere("PLASMA", "INTERNAL ERROR AT plasmaNURL:store:" + e.toString() + ", reset of NURL-DB failed"); - System.exit(0); // this is REALLY severe - } + resetHashCache(); } } diff --git a/source/de/anomic/plasma/plasmaCrawlProfile.java b/source/de/anomic/plasma/plasmaCrawlProfile.java index accc66d02..e87fe8db1 100644 --- a/source/de/anomic/plasma/plasmaCrawlProfile.java +++ b/source/de/anomic/plasma/plasmaCrawlProfile.java @@ -52,7 +52,6 @@ import de.anomic.kelondro.kelondroDyn; import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroMap; import de.anomic.server.serverCodings; -import de.anomic.server.logging.serverLog; public class plasmaCrawlProfile { @@ -60,14 +59,19 @@ public class plasmaCrawlProfile { private File profileTableFile; private int bufferkb; - public plasmaCrawlProfile(File profileTableFile, int bufferkb) throws IOException { - this.profileTableFile = profileTableFile; - if (profileTableFile.exists()) { - profileTable = new kelondroMap(new kelondroDyn(profileTableFile, bufferkb * 1024)); + public plasmaCrawlProfile(File file, int bufferkb) { + this.profileTableFile = file; + kelondroDyn dyn = null; + if (profileTableFile.exists()) try { + dyn = new kelondroDyn(file, bufferkb * 1024); + } catch (IOException e) { + profileTableFile.delete(); + dyn = new kelondroDyn(file, bufferkb * 1024, plasmaURL.urlCrawlProfileHandleLength, 2000, true); } else { profileTableFile.getParentFile().mkdirs(); - profileTable = new kelondroMap(new kelondroDyn(profileTableFile, bufferkb * 1024, plasmaURL.urlCrawlProfileHandleLength, 2000)); + dyn = new kelondroDyn(file, bufferkb * 1024, plasmaURL.urlCrawlProfileHandleLength, 2000, true); } + profileTable = new kelondroMap(dyn); } public int[] dbCacheChunkSize() { @@ -80,16 +84,10 @@ public class plasmaCrawlProfile { private void resetDatabase() { // deletes the profile database and creates a new one - if (profileTable != null) try { - profileTable.close(); - } catch (IOException e) {} + if (profileTable != null) try { profileTable.close(); } catch (IOException e) {} if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database"); - try { - profileTableFile.getParentFile().mkdirs(); - profileTable = new kelondroMap(new kelondroDyn(profileTableFile, bufferkb * 1024, plasmaURL.urlCrawlProfileHandleLength, 2000)); - } catch (IOException e){ - serverLog.logSevere("PLASMA", "plasmaCrawlProfile.resetDatabase", e); - } + profileTableFile.getParentFile().mkdirs(); + profileTable = new kelondroMap(new kelondroDyn(profileTableFile, bufferkb * 1024, plasmaURL.urlCrawlProfileHandleLength, 2000, true)); } public void close() { @@ -104,11 +102,11 @@ public class plasmaCrawlProfile { public Iterator profiles(boolean up) { // enumerates profile entries - try { - return new profileIterator(up); - } catch (IOException e) { - return new HashSet().iterator(); - } + try { + return new profileIterator(up); + } catch (IOException e) { + return new HashSet().iterator(); + } } public class profileIterator implements Iterator { @@ -156,7 +154,7 @@ public class plasmaCrawlProfile { boolean crawlingQ, boolean storeHTCache, boolean storeTXCache, boolean localIndexing, boolean remoteIndexing, - boolean xsstopw, boolean xdstopw, boolean xpstopw) throws IOException { + boolean xsstopw, boolean xdstopw, boolean xpstopw) { entry ne = new entry(name, startURL, generalFilter, specificFilter, generalDepth, specificDepth, @@ -166,7 +164,20 @@ public class plasmaCrawlProfile { profileTable.set(ne.handle(), ne.map()); } catch (kelondroException e) { resetDatabase(); - profileTable.set(ne.handle(), ne.map()); + try { + profileTable.set(ne.handle(), ne.map()); + } catch (IOException ee) { + e.printStackTrace(); + System.exit(0); + } + } catch (IOException e) { + resetDatabase(); + try { + profileTable.set(ne.handle(), ne.map()); + } catch (IOException ee) { + e.printStackTrace(); + System.exit(0); + } } return ne; } diff --git a/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java b/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java index a35fc3f9c..233ef1fd4 100644 --- a/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java +++ b/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java @@ -57,14 +57,13 @@ import java.util.Map; import de.anomic.kelondro.kelondroDyn; import de.anomic.kelondro.kelondroMap; import de.anomic.kelondro.kelondroException; -import de.anomic.server.logging.serverLog; public class plasmaCrawlRobotsTxt { kelondroMap robotsTable; private final File robotsTableFile; private int bufferkb; - public plasmaCrawlRobotsTxt(File robotsTableFile, int bufferkb) throws IOException { + public plasmaCrawlRobotsTxt(File robotsTableFile, int bufferkb) { this.robotsTableFile = robotsTableFile; this.bufferkb = bufferkb; if (robotsTableFile.exists()) { @@ -72,12 +71,14 @@ public class plasmaCrawlRobotsTxt { robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024)); } catch (kelondroException e) { robotsTableFile.delete(); - robotsTableFile.getParentFile().mkdirs(); - robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, 256, 512)); + robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, 256, 512, true)); + } catch (IOException e) { + robotsTableFile.delete(); + robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, 256, 512, true)); } } else { robotsTableFile.getParentFile().mkdirs(); - robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, 256, 512)); + robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, 256, 512, true)); } } @@ -95,12 +96,8 @@ public class plasmaCrawlRobotsTxt { robotsTable.close(); } catch (IOException e) {} if (!(robotsTableFile.delete())) throw new RuntimeException("cannot delete robots.txt database"); - try { - robotsTableFile.getParentFile().mkdirs(); - robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, this.bufferkb, 256, 512)); - } catch (IOException e){ - serverLog.logSevere("PLASMA", "robotsTxt.resetDatabase", e); - } + robotsTableFile.getParentFile().mkdirs(); + robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, this.bufferkb, 256, 512, true)); } public void close() { diff --git a/source/de/anomic/plasma/plasmaCrawlStacker.java b/source/de/anomic/plasma/plasmaCrawlStacker.java index 759d9de8d..b320d41be 100644 --- a/source/de/anomic/plasma/plasmaCrawlStacker.java +++ b/source/de/anomic/plasma/plasmaCrawlStacker.java @@ -78,7 +78,7 @@ public final class plasmaCrawlStacker { //private boolean stopped = false; private stackCrawlQueue queue; - public plasmaCrawlStacker(plasmaSwitchboard sb, File dbPath, int dbCacheSize) throws IOException { + public plasmaCrawlStacker(plasmaSwitchboard sb, File dbPath, int dbCacheSize) { this.sb = sb; this.queue = new stackCrawlQueue(dbPath,dbCacheSize); @@ -500,7 +500,7 @@ public final class plasmaCrawlStacker { private final LinkedList urlEntryHashCache; private kelondroTree urlEntryCache; - public stackCrawlQueue(File cacheStacksPath, int bufferkb) throws IOException { + public stackCrawlQueue(File cacheStacksPath, int bufferkb) { // init the read semaphore this.readSync = new serverSemaphore (0); @@ -516,7 +516,12 @@ public final class plasmaCrawlStacker { File cacheFile = new File(cacheStacksPath, "urlPreNotice.db"); if (cacheFile.exists()) { // open existing cache - this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400); + try { + this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400); + } catch (IOException e) { + cacheFile.delete(); + this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, plasmaCrawlNURL.ce, true); + } try { // loop through the list and fill the messageList with url hashs Iterator iter = this.urlEntryCache.nodeIterator(true,false); @@ -538,12 +543,12 @@ public final class plasmaCrawlStacker { // deleting old db and creating a new db try {this.urlEntryCache.close();}catch(Exception ex){} cacheFile.delete(); - this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, plasmaCrawlNURL.ce); + this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, plasmaCrawlNURL.ce, true); } } else { // create new cache cacheFile.getParentFile().mkdirs(); - this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, plasmaCrawlNURL.ce); + this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, plasmaCrawlNURL.ce, true); } } diff --git a/source/de/anomic/plasma/plasmaDbImporter.java b/source/de/anomic/plasma/plasmaDbImporter.java index 8c952d802..9545906cb 100644 --- a/source/de/anomic/plasma/plasmaDbImporter.java +++ b/source/de/anomic/plasma/plasmaDbImporter.java @@ -1,7 +1,6 @@ package de.anomic.plasma; import java.io.File; -import java.io.IOException; import java.util.Iterator; import java.util.Vector; @@ -142,7 +141,7 @@ public class plasmaDbImporter extends Thread { return this.importWordIndex.size(); } - public plasmaDbImporter(plasmaWordIndex theHomeIndexDB, plasmaCrawlLURL theHomeUrlDB, String theImportPath) throws IOException { + public plasmaDbImporter(plasmaWordIndex theHomeIndexDB, plasmaCrawlLURL theHomeUrlDB, String theImportPath) { super(runningJobs,"DB-Import_" + theImportPath); this.log = new serverLog("DB-IMPORT"); diff --git a/source/de/anomic/plasma/plasmaHTCache.java b/source/de/anomic/plasma/plasmaHTCache.java index 113c17be1..381624b9a 100644 --- a/source/de/anomic/plasma/plasmaHTCache.java +++ b/source/de/anomic/plasma/plasmaHTCache.java @@ -114,7 +114,7 @@ public final class plasmaHTCache { if (dbfile.exists()) this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400)); else - this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400, plasmaURL.urlHashLength, 150)); + this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400, plasmaURL.urlHashLength, 150, false)); } catch (IOException e) { this.log.logSevere("the request header database could not be opened: " + e.getMessage()); System.exit(0); diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 6fbb6b4b6..1efca8f92 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -118,6 +118,7 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; import java.util.logging.Level; + import de.anomic.data.messageBoard; import de.anomic.data.wikiBoard; import de.anomic.data.userDB; @@ -220,7 +221,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser private boolean crawlingIsPaused = false; private static plasmaSwitchboard sb; - public plasmaSwitchboard(String rootPath, String initPath, String configPath) throws IOException { + public plasmaSwitchboard(String rootPath, String initPath, String configPath) { super(rootPath, initPath, configPath); // set loglevel and log @@ -370,16 +371,19 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // going through the sbQueue Entries and registering all content files as in use int count = 0; - ArrayList sbQueueEntries = this.sbQueue.list(); - for (int i=0; i 0) || (cacheLoader.size() > 0) || (urlPool.noticeURL.stackSize() > 0)) return false; final Iterator iter = profiles.profiles(true); diff --git a/source/de/anomic/plasma/plasmaSwitchboardQueue.java b/source/de/anomic/plasma/plasmaSwitchboardQueue.java index ff8b5cda8..aaf5ca73f 100644 --- a/source/de/anomic/plasma/plasmaSwitchboardQueue.java +++ b/source/de/anomic/plasma/plasmaSwitchboardQueue.java @@ -65,14 +65,26 @@ public class plasmaSwitchboardQueue { private plasmaHTCache htCache; private plasmaCrawlLURL lurls; - public plasmaSwitchboardQueue(plasmaHTCache htCache, plasmaCrawlLURL lurls, File sbQueueStackPath, plasmaCrawlProfile profiles) throws IOException { + public plasmaSwitchboardQueue(plasmaHTCache htCache, plasmaCrawlLURL lurls, File sbQueueStackPath, plasmaCrawlProfile profiles) { this.profiles = profiles; this.htCache = htCache; this.lurls = lurls; - if (sbQueueStackPath.exists()) + if (sbQueueStackPath.exists()) try { sbQueueStack = new kelondroStack(sbQueueStackPath, 0); - else + } catch (IOException e) { + sbQueueStackPath.delete(); + sbQueueStack = new kelondroStack(sbQueueStackPath, 0, new int[] { + plasmaURL.urlStringLength, + plasmaURL.urlHashLength, + 11, + 1, + yacySeedDB.commonHashLength, + plasmaURL.urlCrawlDepthLength, + plasmaURL.urlCrawlProfileHandleLength, + plasmaURL.urlDescrLength + }, true); + } else { sbQueueStack = new kelondroStack(sbQueueStackPath, 0, new int[] { plasmaURL.urlStringLength, plasmaURL.urlHashLength, @@ -82,7 +94,8 @@ public class plasmaSwitchboardQueue { plasmaURL.urlCrawlDepthLength, plasmaURL.urlCrawlProfileHandleLength, plasmaURL.urlDescrLength - }); + }, true); + } } public int size() { diff --git a/source/de/anomic/plasma/plasmaURLPool.java b/source/de/anomic/plasma/plasmaURLPool.java index 3b31888b5..5885f892c 100644 --- a/source/de/anomic/plasma/plasmaURLPool.java +++ b/source/de/anomic/plasma/plasmaURLPool.java @@ -55,7 +55,7 @@ public class plasmaURLPool { public final plasmaCrawlNURL noticeURL; public final plasmaCrawlEURL errorURL; - public plasmaURLPool(File plasmaPath, int ramLURL, int ramNURL, int ramEURL) throws IOException { + public plasmaURLPool(File plasmaPath, int ramLURL, int ramNURL, int ramEURL) { loadedURL = new plasmaCrawlLURL(new File(plasmaPath, "urlHash.db"), ramLURL); noticeURL = new plasmaCrawlNURL(plasmaPath, ramNURL); errorURL = new plasmaCrawlEURL(new File(plasmaPath, "urlErr0.db"), ramEURL); diff --git a/source/de/anomic/plasma/plasmaWordConnotation.java b/source/de/anomic/plasma/plasmaWordConnotation.java index 9d87d569f..69ebc3a8c 100644 --- a/source/de/anomic/plasma/plasmaWordConnotation.java +++ b/source/de/anomic/plasma/plasmaWordConnotation.java @@ -58,12 +58,14 @@ public class plasmaWordConnotation { private static final int nodesize = 4048; private kelondroDynTree refDB; - public plasmaWordConnotation(File refDBfile, int bufferkb) throws IOException { - if (refDBfile.exists()) + public plasmaWordConnotation(File refDBfile, int bufferkb) { + if (refDBfile.exists()) try { refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400); - else - refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, wordlength, nodesize, new int[] {wordlength, countlength}); - + } catch (IOException e) { + refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, wordlength, nodesize, new int[] {wordlength, countlength}, true); + } else { + refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, wordlength, nodesize, new int[] {wordlength, countlength}, true); + } } private void addSingleRef(String word, String reference) throws IOException { diff --git a/source/de/anomic/plasma/plasmaWordIndexAssortment.java b/source/de/anomic/plasma/plasmaWordIndexAssortment.java index a2794ee0f..1dc6b5bb4 100644 --- a/source/de/anomic/plasma/plasmaWordIndexAssortment.java +++ b/source/de/anomic/plasma/plasmaWordIndexAssortment.java @@ -120,12 +120,8 @@ public final class plasmaWordIndexAssortment { assortmentFile.delete(); // make space for new one } // create new assortment tree file - try { - assortments = new kelondroTree(assortmentFile, bufferSize, bufferStructure(assortmentLength)); - if (log != null) log.logConfig("Created new Assortment Database, width " + assortmentLength + ", " + bufferkb + "kb buffer"); - } catch (IOException e){ - serverLog.logSevere("PLASMA", "unable to create assortment database: " + e.getMessage(), e); - } + assortments = new kelondroTree(assortmentFile, bufferSize, bufferStructure(assortmentLength), true); + if (log != null) log.logConfig("Created new Assortment Database, width " + assortmentLength + ", " + bufferkb + "kb buffer"); } public void store(String wordHash, plasmaWordIndexEntryContainer newContainer) { @@ -194,11 +190,7 @@ public final class plasmaWordIndexAssortment { assortments.close(); } catch (IOException e) {} if (!(assortmentFile.delete())) throw new RuntimeException("cannot delete assortment database"); - try { - assortments = new kelondroTree(assortmentFile, bufferSize, bufferStructure(assortmentLength)); - } catch (IOException e){ - log.logSevere("unable to re-create assortment database: " + e.getMessage(), e); - } + assortments = new kelondroTree(assortmentFile, bufferSize, bufferStructure(assortmentLength), true); } public Iterator hashes(String startWordHash, boolean up, boolean rot) { diff --git a/source/de/anomic/plasma/plasmaWordIndexCache.java b/source/de/anomic/plasma/plasmaWordIndexCache.java index 46410f864..8e71df42d 100644 --- a/source/de/anomic/plasma/plasmaWordIndexCache.java +++ b/source/de/anomic/plasma/plasmaWordIndexCache.java @@ -134,8 +134,7 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface { File indexDumpFile = new File(databaseRoot, indexArrayFileName); if (indexDumpFile.exists()) indexDumpFile.delete(); kelondroArray dumpArray = null; - try { - dumpArray = new kelondroArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0); + dumpArray = new kelondroArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0, false); long startTime = System.currentTimeMillis(); long messageTime = System.currentTimeMillis() + 5000; long wordsPerSecond = 0, wordcount = 0, urlcount = 0; @@ -182,9 +181,6 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface { dumpArray.close(); dumpArray = null; log.logConfig("dumped " + urlcount + " word/URL relations in " + ((System.currentTimeMillis() - startTime) / 1000) + " seconds"); - } finally { - if (dumpArray != null) try {dumpArray.close();}catch(Exception e){} - } } private long restore() throws IOException { diff --git a/source/de/anomic/plasma/plasmaWordIndexClassicDB.java b/source/de/anomic/plasma/plasmaWordIndexClassicDB.java index 7a930257c..008f5dbd3 100644 --- a/source/de/anomic/plasma/plasmaWordIndexClassicDB.java +++ b/source/de/anomic/plasma/plasmaWordIndexClassicDB.java @@ -182,12 +182,7 @@ public class plasmaWordIndexClassicDB implements plasmaWordIndexInterface { } public plasmaWordIndexEntity getIndex(String wordHash, boolean deleteIfEmpty, long maxTime) { - try { - return new plasmaWordIndexEntity(databaseRoot, wordHash, deleteIfEmpty); - } catch (IOException e) { - log.logSevere("plasmaWordIndexClassic.getIndex: " + e.getMessage()); - return null; - } + return new plasmaWordIndexEntity(databaseRoot, wordHash, deleteIfEmpty); } public long getUpdateTime(String wordHash) { diff --git a/source/de/anomic/plasma/plasmaWordIndexEntity.java b/source/de/anomic/plasma/plasmaWordIndexEntity.java index 925b4b62b..e8f5c5b03 100644 --- a/source/de/anomic/plasma/plasmaWordIndexEntity.java +++ b/source/de/anomic/plasma/plasmaWordIndexEntity.java @@ -62,7 +62,7 @@ public final class plasmaWordIndexEntity { private File theLocation; private boolean delete; - public plasmaWordIndexEntity(File databaseRoot, String wordHash, boolean deleteIfEmpty) throws IOException { + public plasmaWordIndexEntity(File databaseRoot, String wordHash, boolean deleteIfEmpty) { theWordHash = wordHash; theIndex = indexFile(databaseRoot, wordHash); theTmpMap = null; @@ -82,20 +82,23 @@ public final class plasmaWordIndexEntity { return success; } - private kelondroTree indexFile(File databaseRoot, String wordHash) throws IOException { - if (wordHash.length() < 12) throw new IOException("word hash wrong: '" + wordHash + "'"); + private kelondroTree indexFile(File databaseRoot, String wordHash) { + if (wordHash.length() < 12) throw new RuntimeException("word hash wrong: '" + wordHash + "'"); theLocation = wordHash2path(databaseRoot, wordHash); File fp = theLocation.getParentFile(); if (fp != null) fp.mkdirs(); kelondroTree kt; long cacheSize = theLocation.length(); if (cacheSize > 1048576) cacheSize = 1048576; - if (theLocation.exists()) { + if (theLocation.exists()) try { // open existing index file kt = new kelondroTree(theLocation, cacheSize); + } catch (IOException e) { + theLocation.delete(); + kt = new kelondroTree(theLocation, cacheSize, plasmaURL.urlHashLength, plasmaWordIndexEntry.attrSpaceShort, false); } else { // create new index file - kt = new kelondroTree(theLocation, cacheSize, plasmaURL.urlHashLength, plasmaWordIndexEntry.attrSpaceShort); + kt = new kelondroTree(theLocation, cacheSize, plasmaURL.urlHashLength, plasmaWordIndexEntry.attrSpaceShort, false); } return kt; // everyone who get this should close it when finished! } diff --git a/source/de/anomic/urlRedirector/urlRedirectord.java b/source/de/anomic/urlRedirector/urlRedirectord.java index 08653f061..e974f4d74 100644 --- a/source/de/anomic/urlRedirector/urlRedirectord.java +++ b/source/de/anomic/urlRedirector/urlRedirectord.java @@ -34,7 +34,6 @@ public class urlRedirectord implements serverHandler { } if (profile == null) { - try { profile = switchboard.profiles.newEntry( // name "URL Redirector", @@ -63,9 +62,6 @@ public class urlRedirectord implements serverHandler { // xpstopw true ); - } catch (IOException e) { - this.theLogger.logSevere("Unable to create a crawling profile for the URL-Redirector",e); - } } } diff --git a/source/de/anomic/yacy/yacyCore.java b/source/de/anomic/yacy/yacyCore.java index f82bcbcaf..b637fe318 100644 --- a/source/de/anomic/yacy/yacyCore.java +++ b/source/de/anomic/yacy/yacyCore.java @@ -147,7 +147,7 @@ public class yacyCore { return (int) ((System.currentTimeMillis() - startupTime) / 1000); } - public yacyCore(plasmaSwitchboard sb) throws IOException { + public yacyCore(plasmaSwitchboard sb) { long time = System.currentTimeMillis(); this.switchboard = sb; diff --git a/source/de/anomic/yacy/yacyNewsDB.java b/source/de/anomic/yacy/yacyNewsDB.java index 9578fa4df..f85d3cf61 100644 --- a/source/de/anomic/yacy/yacyNewsDB.java +++ b/source/de/anomic/yacy/yacyNewsDB.java @@ -63,27 +63,30 @@ public class yacyNewsDB { - yacyCore.universalDateShortPattern.length() - 2; - public yacyNewsDB(File path, int bufferkb) throws IOException { + public yacyNewsDB(File path, int bufferkb) { this.path = path; this.bufferkb = bufferkb; - if (path.exists()) + if (path.exists()) try { news = new kelondroTree(path, bufferkb * 0x400); - else - news = createDB(path, bufferkb); + } catch (IOException e) { + news = createDB(path, bufferkb); + } else { + news = createDB(path, bufferkb); + } } - private static kelondroTree createDB(File path, int bufferkb) throws IOException { + private static kelondroTree createDB(File path, int bufferkb) { return new kelondroTree(path, bufferkb * 0x400, new int[] { yacyNewsRecord.idLength(), // id = created + originator yacyNewsRecord.categoryStringLength, // category yacyCore.universalDateShortPattern.length(), // received 2, attributesMaxLength - }); + }, true); } - private void resetDB() throws IOException { + private void resetDB() { try {close();} catch (Exception e) {} if (path.exists()) path.delete(); news = createDB(path, bufferkb); diff --git a/source/de/anomic/yacy/yacyNewsPool.java b/source/de/anomic/yacy/yacyNewsPool.java index 7baa5072d..afcd51517 100644 --- a/source/de/anomic/yacy/yacyNewsPool.java +++ b/source/de/anomic/yacy/yacyNewsPool.java @@ -93,7 +93,7 @@ public class yacyNewsPool { private int maxDistribution; - public yacyNewsPool(File yacyDBPath, int bufferkb) throws IOException { + public yacyNewsPool(File yacyDBPath, int bufferkb) { newsDB = new yacyNewsDB(new File(yacyDBPath, "news1.db"), bufferkb); outgoingNews = new yacyNewsQueue(new File(yacyDBPath, "newsOut1.stack"), newsDB); publishedNews = new yacyNewsQueue(new File(yacyDBPath, "newsPublished1.stack"), newsDB); @@ -243,7 +243,7 @@ public class yacyNewsPool { return null; } - public void clear(int dbKey) throws IOException { + public void clear(int dbKey) { // this is called if a queue element shall be moved to another queue or off the queue // it depends on the dbKey how the record is handled switch (dbKey) { diff --git a/source/de/anomic/yacy/yacyNewsQueue.java b/source/de/anomic/yacy/yacyNewsQueue.java index 4dc01dff1..3d76479ea 100644 --- a/source/de/anomic/yacy/yacyNewsQueue.java +++ b/source/de/anomic/yacy/yacyNewsQueue.java @@ -54,35 +54,37 @@ public class yacyNewsQueue { private kelondroStack queueStack; private yacyNewsDB newsDB; - public yacyNewsQueue(File path, yacyNewsDB newsDB) throws IOException { + public yacyNewsQueue(File path, yacyNewsDB newsDB) { this.path = path; this.newsDB = newsDB; - if (path.exists()) { - try { - queueStack = new kelondroStack(path, 0); - } catch (kelondroException e) { - path.delete(); - queueStack = createStack(path); - } - } else + if (path.exists()) try { + queueStack = new kelondroStack(path, 0); + } catch (kelondroException e) { + path.delete(); + queueStack = createStack(path); + } catch (IOException e) { + path.delete(); queueStack = createStack(path); + } else { + queueStack = createStack(path); + } } - private static kelondroStack createStack(File path) throws IOException { + private static kelondroStack createStack(File path) { return new kelondroStack(path, 0, new int[] { yacyNewsRecord.idLength(), // id = created + originator yacyCore.universalDateShortPattern.length() // last touched - }); + }, true); } - private void resetDB() throws IOException { + private void resetDB() { try {close();} catch (Exception e) {} if (path.exists()) path.delete(); queueStack = createStack(path); } - public void clear() throws IOException { + public void clear() { resetDB(); } diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index 9efb4feae..2df99f4cd 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -100,7 +100,7 @@ public final class yacySeedDB { File seedActiveDBFile, File seedPassiveDBFile, File seedPotentialDBFile, - int bufferkb) throws IOException { + int bufferkb) { this.seedDBBufferKB = bufferkb; this.seedActiveDBFile = seedActiveDBFile; @@ -116,15 +116,27 @@ public final class yacySeedDB { // create or init own seed myOwnSeedFile = sb.getOwnSeedFile(); - if (myOwnSeedFile.length() > 0) { + if (myOwnSeedFile.length() > 0) try { // load existing identity mySeed = yacySeed.load(myOwnSeedFile); + } catch (IOException e) { + // create new identity + mySeed = yacySeed.genLocalSeed(sb); + try { + mySeed.save(myOwnSeedFile); + } catch (IOException ee) { + ee.printStackTrace(); + System.exit(-1); + } } else { // create new identity mySeed = yacySeed.genLocalSeed(sb); - // save of for later use - mySeed.save(myOwnSeedFile); // in a file - //writeMap(mySeed.hash, mySeed.dna, "new"); // in a database + try { + mySeed.save(myOwnSeedFile); + } catch (IOException ee) { + ee.printStackTrace(); + System.exit(-1); + } } if (sb.getConfig("portForwardingEnabled","false").equalsIgnoreCase("true")) { @@ -172,7 +184,7 @@ public final class yacySeedDB { return new int[]{ac[0] + pa[0] + po[0], ac[1] + pa[1] + po[1], ac[2] + pa[2] + po[2], ac[3] + pa[3] + po[3]}; } - private synchronized kelondroMap openSeedTable(File seedDBFile) throws IOException { + private synchronized kelondroMap openSeedTable(File seedDBFile) { if (seedDBFile.exists()) try { // open existing seed database return new kelondroMap(new kelondroDyn(seedDBFile, (seedDBBufferKB * 0x400) / 3), sortFields, accFields); @@ -185,7 +197,7 @@ public final class yacySeedDB { } // create new seed database new File(seedDBFile.getParent()).mkdir(); - return new kelondroMap(new kelondroDyn(seedDBFile, (seedDBBufferKB * 0x400) / 3, commonHashLength, 480), sortFields, accFields); + return new kelondroMap(new kelondroDyn(seedDBFile, (seedDBBufferKB * 0x400) / 3, commonHashLength, 480, true), sortFields, accFields); } private synchronized kelondroMap resetSeedTable(kelondroMap seedDB, File seedDBFile) {