boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

better startup failure handling

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1205 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 20 years ago
parent f27f9ecf15
commit 0c762daf4b
36 changed files with 431 additions and 288 deletions
Show Stats Download Patch File Download Diff File

2
source/dbtest.java
Unescape View File

@ -167,7 +167,7 @@ public class dbtest {
if (tablefile.exists()) {
table = new kelondroTree(tablefile, buffer);
} else {
table = new kelondroTree(tablefile, buffer, new int[]{keylength, valuelength, valuelength});
table = new kelondroTree(tablefile, buffer, new int[]{keylength, valuelength, valuelength}, true);
}
}

22
source/de/anomic/data/messageBoard.java
Unescape View File

@ -67,15 +67,19 @@ public class messageBoard {
private kelondroMap database = null;
private int sn = 0;
public messageBoard(File path, int bufferkb) throws IOException {
new File(path.getParent()).mkdir();
if (database == null) {
if (path.exists())
database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400));
else
database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, categoryLength + dateFormat.length() + 2, recordSize));
}
sn = 0;
public messageBoard(File path, int bufferkb) {
new File(path.getParent()).mkdir();
if (database == null) {
if (path.exists()) try {
database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400));
} catch (IOException e) {
path.delete();
database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, categoryLength + dateFormat.length() + 2, recordSize, true));
} else {
database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, categoryLength + dateFormat.length() + 2, recordSize, true));
}
}
sn = 0;
}
public int size() {

19
source/de/anomic/data/userDB.java
Unescape View File

@ -58,7 +58,6 @@ import java.lang.NumberFormatException;
import de.anomic.kelondro.kelondroDyn;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMap;
import de.anomic.server.logging.serverLog;
import de.anomic.server.serverCodings;
public final class userDB {
@ -72,7 +71,7 @@ public final class userDB {
private final serverCodings codings = new serverCodings(true);
private HashMap ipUsers = new HashMap();
public userDB(File userTableFile, int bufferkb) throws IOException {
public userDB(File userTableFile, int bufferkb) {
this.userTableFile = userTableFile;
this.bufferkb = bufferkb;
if (userTableFile.exists()) {
@ -81,11 +80,15 @@ public final class userDB {
} catch (kelondroException e) {
userTableFile.delete();
userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, 128, 256));
this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, 128, 256, true));
} catch (IOException e) {
userTableFile.delete();
userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, 128, 256, true));
}
} else {
userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, 128, 256));
this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, 128, 256, true));
}
}
@ -103,12 +106,8 @@ public final class userDB {
userTable.close();
} catch (IOException e) {}
if (!(userTableFile.delete())) throw new RuntimeException("cannot delete user database");
try {
userTableFile.getParentFile().mkdirs();
userTable = new kelondroMap(new kelondroDyn(userTableFile, this.bufferkb, 256, 512));
} catch (IOException e){
serverLog.logSevere("PLASMA", "user.resetDatabase", e);
}
userTableFile.getParentFile().mkdirs();
userTable = new kelondroMap(new kelondroDyn(userTableFile, this.bufferkb, 256, 512, true));
}
public void close() {

34
source/de/anomic/data/wikiBoard.java
Unescape View File

@ -69,21 +69,27 @@ public class wikiBoard {
private kelondroMap bkpbase = null;
private HashMap authors = new HashMap();
public wikiBoard(File actpath, File bkppath, int bufferkb) throws IOException {
public wikiBoard(File actpath, File bkppath, int bufferkb) {
new File(actpath.getParent()).mkdir();
if (datbase == null) {
if (actpath.exists())
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb/2 * 0x40));
else
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb/2 * 0x400, keyLength, recordSize));
}
new File(bkppath.getParent()).mkdir();
if (bkpbase == null) {
if (bkppath.exists())
bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb/2 * 0x400));
else
bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb/2 * 0x400, keyLength + dateFormat.length(), recordSize));
}
if (datbase == null) {
if (actpath.exists()) try {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x40));
} catch (IOException e) {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, keyLength, recordSize, true));
} else {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, keyLength, recordSize, true));
}
}
new File(bkppath.getParent()).mkdir();
if (bkpbase == null) {
if (bkppath.exists()) try {
bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400));
} catch (IOException e) {
bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400, keyLength + dateFormat.length(), recordSize, true));
} else {
bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400, keyLength + dateFormat.length(), recordSize, true));
}
}
}
public int sizeOfTwo() {

18
source/de/anomic/kelondro/kelondroArray.java
Unescape View File

@ -55,10 +55,16 @@ public class kelondroArray extends kelondroRecords {
private static short thisOHBytes = 0; // our record definition does not need extra bytes
private static short thisOHHandles = 0; // and two handles overhead for a double-chained list
public kelondroArray(File file, int[] columns, int intprops) throws IOException {
// this creates a new array
super(file, 0, thisOHBytes, thisOHHandles, columns, intprops, columns.length /*txtProps*/, 80 /*txtPropWidth*/);
for (int i = 0; i < intprops; i++) setHandle(i, new Handle(0));
public kelondroArray(File file, int[] columns, int intprops, boolean exitOnFail) {
// this creates a new array
super(file, 0, thisOHBytes, thisOHHandles, columns, intprops, columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
for (int i = 0; i < intprops; i++) try {
setHandle(i, new Handle(0));
} catch (IOException e) {
super.logFailure("cannot set handle " + i + " / " + e.getMessage());
if (exitOnFail) System.exit(-1);
throw new RuntimeException("cannot set handle " + i + " / " + e.getMessage());
}
}
public kelondroArray(File file) throws IOException{
@ -129,7 +135,7 @@ public class kelondroArray extends kelondroRecords {
// create <filename> <valuelen>
File f = new File(args[1]);
if (f.exists()) f.delete();
kelondroArray fm = new kelondroArray(f, new int[]{Integer.parseInt(args[2])}, 2);
kelondroArray fm = new kelondroArray(f, new int[]{Integer.parseInt(args[2])}, 2, true);
fm.close();
} else
if ((args.length == 2) && (args[0].equals("-v"))) {
@ -157,7 +163,7 @@ public class kelondroArray extends kelondroRecords {
if ((args.length == 1) && (args[0].equals("-test"))) {
File testfile = new File("test.array");
if (testfile.exists()) testfile.delete();
kelondroArray fm = new kelondroArray(testfile, new int[]{30, 50}, 9);
kelondroArray fm = new kelondroArray(testfile, new int[]{30, 50}, 9, true);
for (int i = 0; i < 100; i++) {
fm.set(i, new byte[][]{("name" + i).getBytes(), ("value" + i).getBytes()});
}

12
source/de/anomic/kelondro/kelondroDyn.java
Unescape View File

@ -70,9 +70,9 @@ public class kelondroDyn extends kelondroTree {
private int reclen;
private int segmentCount;
public kelondroDyn(File file, long buffersize /*bytes*/, int key, int nodesize) throws IOException {
public kelondroDyn(File file, long buffersize /*bytes*/, int key, int nodesize, boolean exitOnFail) {
// creates a new dynamic tree
super(file, buffersize, new int[] {key + counterlen, nodesize}, 1, 8);
super(file, buffersize, new int[] {key + counterlen, nodesize}, 1, 8, exitOnFail);
this.keylen = columnSize(0) - counterlen;
this.reclen = columnSize(1);
this.segmentCacheKey = null;
@ -82,7 +82,7 @@ public class kelondroDyn extends kelondroTree {
writeSegmentCount();
}
public kelondroDyn(File file, long buffersize) throws IOException{
public kelondroDyn(File file, long buffersize) throws IOException {
// this opens a file with an existing dynamic tree
super(file, buffersize);
this.keylen = columnSize(0) - counterlen;
@ -199,7 +199,7 @@ public class kelondroDyn extends kelondroTree {
}
// read from db
byte[][] r = get(key);
final byte[][] r = get(key);
if (r == null) return null;
// update cache
@ -442,7 +442,7 @@ public class kelondroDyn extends kelondroTree {
File f = new File(args[3]);
kelondroDyn kd;
try {
if (db.exists()) kd = new kelondroDyn(db, 0x100000); else kd = new kelondroDyn(db, 0x100000, 80, 200);
if (db.exists()) kd = new kelondroDyn(db, 0x100000); else kd = new kelondroDyn(db, 0x100000, 80, 200, true);
if (writeFile) kd.readFile(key, f); else kd.writeFile(key, f);
} catch (IOException e) {
System.out.println("ERROR: " + e.toString());
@ -463,7 +463,7 @@ public class kelondroDyn extends kelondroTree {
int steps = 0;
while (true) {
if (testFile.exists()) testFile.delete();
tt = new kelondroDyn(testFile, 0, 4 ,100);
tt = new kelondroDyn(testFile, 0, 4 ,100, true);
steps = ((int) System.currentTimeMillis() % 7) * (((int) System.currentTimeMillis() + 17) % 11);
t = s;
d = "";

14
source/de/anomic/kelondro/kelondroDynTree.java
Unescape View File

@ -69,7 +69,7 @@ public class kelondroDynTree {
private Hashtable buffer, cache;
private long cycleBuffer;
public kelondroDynTree(File file, long buffersize, int keylength, int nodesize, int[] columns) throws IOException {
public kelondroDynTree(File file, long buffersize, int keylength, int nodesize, int[] columns, boolean exitOnFail) {
// creates a new DynTree
this.file = file;
this.columns = columns;
@ -77,8 +77,8 @@ public class kelondroDynTree {
this.cache = new Hashtable();
//this.cycleCache = Long.MIN_VALUE;
this.cycleBuffer = Long.MIN_VALUE;
if (file.exists()) throw new IOException("DynTree " + file.toString() + " already exists");
this.table = new kelondroDyn(file, buffersize, keylength, nodesize);
if (file.exists()) file.delete();
this.table = new kelondroDyn(file, buffersize, keylength, nodesize, exitOnFail);
this.treeRAHandles = new Hashtable();
}
@ -134,7 +134,11 @@ public class kelondroDynTree {
if (table.existsDyn(key)) throw new IOException("table " + key + " already exists.");
kelondroRA ra = table.getRA(key); // works always, even with no-existing entry
treeRAHandles.put(key, ra);
return new kelondroTree(ra, buffersize, columns);
try {
return new kelondroTree(ra, buffersize, columns, false);
} catch (RuntimeException e) {
throw new IOException(e.getMessage());
}
}
protected kelondroTree getTree(String key) throws IOException {
@ -348,7 +352,7 @@ public class kelondroDynTree {
kelondroDynTree dt = new kelondroDynTree(file, 0x100000L);
System.out.println("opened: table keylength=" + dt.table.columnSize(0) + ", sectorsize=" + dt.table.columnSize(1) + ", " + dt.table.size() + " entries.");
} else {
kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, 16, 512, new int[] {10,20,30});
kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, 16, 512, new int[] {10,20,30}, true);
String name;
kelondroTree t;
byte[][] line = new byte[][] {"".getBytes(), "abc".getBytes(), "def".getBytes()};

19
source/de/anomic/kelondro/kelondroFScoreCluster.java
Unescape View File

@ -63,17 +63,26 @@ public class kelondroFScoreCluster {
private kelondroTree refcountDB;
private kelondroTree countrefDB;
public kelondroFScoreCluster(File refcountDBfile, File countrefDBfile) throws IOException {
if ((refcountDBfile.exists()) && (countrefDBfile.exists())) {
public kelondroFScoreCluster(File refcountDBfile, File countrefDBfile, boolean exitOnFail) {
if ((refcountDBfile.exists()) && (countrefDBfile.exists())) try {
refcountDB = new kelondroTree(refcountDBfile, 0x100000L);
refcountDB.setText(0, serverCodings.enhancedCoder.encodeBase64Long(0, countlength).getBytes()); // counter of all occurrences
countrefDB = new kelondroTree(countrefDBfile, 0x100000L);
countrefDB.setText(0, serverCodings.enhancedCoder.encodeBase64Long(0, countlength).getBytes());
} catch (IOException e) {
refcountDBfile.delete();
countrefDBfile.delete();
refcountDB = new kelondroTree(refcountDBfile, 0x100000L, new int[] {wordlength, countlength}, 1, countlength, exitOnFail);
countrefDB = new kelondroTree(countrefDBfile, 0x100000L, new int[] {countlength + wordlength, 4}, 1, countlength, exitOnFail);
} else if ((!(refcountDBfile.exists())) && (!(countrefDBfile.exists()))) {
refcountDB = new kelondroTree(refcountDBfile, 0x100000L, new int[] {wordlength, countlength}, 1, countlength);
countrefDB = new kelondroTree(countrefDBfile, 0x100000L, new int[] {countlength + wordlength, 4}, 1, countlength);
refcountDB = new kelondroTree(refcountDBfile, 0x100000L, new int[] {wordlength, countlength}, 1, countlength, exitOnFail);
countrefDB = new kelondroTree(countrefDBfile, 0x100000L, new int[] {countlength + wordlength, 4}, 1, countlength, exitOnFail);
} else {
throw new IOException("both word/count db files must exists");
if (exitOnFail) {
System.exit(-1);
} else {
throw new RuntimeException("both word/count db files must exists");
}
}
}

18
source/de/anomic/kelondro/kelondroHashtable.java
Unescape View File

@ -144,7 +144,7 @@ public class kelondroHashtable {
private static final byte[] dummyKey = serverCodings.enhancedCoder.encodeBase64Long(0, 5).getBytes();
public kelondroHashtable(File file, int[] columns, int offset, int maxsize, int maxrehash) throws IOException {
public kelondroHashtable(File file, int[] columns, int offset, int maxsize, int maxrehash, boolean exitOnFail) {
// this creates a new hashtable
// the key element is not part of the columns array
// this is unlike the kelondroTree, where the key is part of a row
@ -154,17 +154,23 @@ public class kelondroHashtable {
// this number is needed to omit grow of the table in case of re-hashing
// the maxsize is re-computed to a virtual folding height and will result in a tablesize
// less than the given maxsize. The actual maxsize can be retrieved by maxsize()
this.hashArray = new kelondroArray(file, extCol(columns), 6);
this.hashArray = new kelondroArray(file, extCol(columns), 6, exitOnFail);
this.offset = offset;
this.maxk = kelondroMSetTools.log2a(maxsize); // equal to |log2(maxsize)| + 1
if (this.maxk >= kelondroMSetTools.log2a(maxsize + power2(offset + 1) + 1) - 1) this.maxk--;
this.maxrehash = maxrehash;
hashArray.seti(0, this.offset);
hashArray.seti(1, this.maxk);
hashArray.seti(2, this.maxrehash);
dummyRow = new byte[hashArray.columns()][];
dummyRow[0] = dummyKey;
for (int i = 0; i < hashArray.columns(); i++) dummyRow[i] = new byte[0];
for (int i = 0; i < hashArray.columns(); i++)
try {
hashArray.seti(0, this.offset);
hashArray.seti(1, this.maxk);
hashArray.seti(2, this.maxrehash);
} catch (IOException e) {
hashArray.logFailure("cannot set properties / " + e.getMessage());
if (exitOnFail) System.exit(-1);
throw new RuntimeException("cannot set properties / " + e.getMessage());
}
}
public kelondroHashtable(File file) throws IOException{

38
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -194,7 +194,8 @@ public class kelondroRecords {
public kelondroRecords(File file, long buffersize /* bytes */,
short ohbytec, short ohhandlec,
int[] columns, int FHandles, int txtProps, int txtPropWidth) throws IOException {
int[] columns, int FHandles, int txtProps, int txtPropWidth,
boolean exitOnFail) {
// creates a new file
// file: the file that shall be created
// oha : overhead size array of four bytes: oha[0]=# of bytes, oha[1]=# of shorts, oha[2]=# of ints, oha[3]=# of longs,
@ -202,20 +203,32 @@ public class kelondroRecords {
// FHandles: number of integer properties
// txtProps: number of text properties
assert (!file.exists()): "file " + file + " already exist";
this.filename = file.getCanonicalPath();
kelondroRA raf = new kelondroFileRA(this.filename);
//kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100);
// kelondroRA raf = new kelondroNIOFileRA(this.filename, false, 10000);
init(raf, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth);
assert (!file.exists()) : "file " + file + " already exist";
try {
this.filename = file.getCanonicalPath();
kelondroRA raf = new kelondroFileRA(this.filename);
// kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100);
// kelondroRA raf = new kelondroNIOFileRA(this.filename, false, 10000);
init(raf, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth);
} catch (IOException e) {
logFailure("cannot create / " + e.getMessage());
if (exitOnFail)
System.exit(-1);
}
initCache(buffersize);
}
public kelondroRecords(kelondroRA ra, long buffersize /* bytes */,
short ohbytec, short ohhandlec,
int[] columns, int FHandles, int txtProps, int txtPropWidth) throws IOException {
int[] columns, int FHandles, int txtProps, int txtPropWidth,
boolean exitOnFail) {
this.filename = null;
init(ra, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth);
try {
init(ra, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth);
} catch (IOException e) {
logFailure("cannot create / " + e.getMessage());
if (exitOnFail) System.exit(-1);
}
initCache(buffersize);
}
@ -295,6 +308,13 @@ public class kelondroRecords {
this.theLogger.warning("KELONDRO WARNING for file " + this.filename + ": " + message);
}
public void logFailure(String message) {
if (this.theLogger == null)
System.err.println("KELONDRO FAILURE for file " + this.filename + ": " + message);
else
this.theLogger.severe("KELONDRO FAILURE for file " + this.filename + ": " + message);
}
public void clear() throws IOException {
// Removes all mappings from this map
// throw new UnsupportedOperationException("clear not supported");

26
source/de/anomic/kelondro/kelondroStack.java
Unescape View File

@ -67,15 +67,21 @@ public final class kelondroStack extends kelondroRecords {
private static int root = 0; // pointer for FHandles-array: pointer to root node
private static int toor = 1; // pointer for FHandles-array: pointer to root node
public kelondroStack(File file, long buffersize, int key, int value) throws IOException {
this(file, buffersize, new int[] {key, value});
public kelondroStack(File file, long buffersize, int key, int value, boolean exitOnFail) {
this(file, buffersize, new int[] { key, value }, exitOnFail);
}
public kelondroStack(File file, long buffersize, int[] columns) throws IOException {
// this creates a new tree
super(file, buffersize, thisOHBytes, thisOHHandles, columns, thisFHandles, columns.length /*txtProps*/, 80 /*txtPropWidth*/);
setHandle(root, null); // define the root value
setHandle(toor, null); // define the toor value
public kelondroStack(File file, long buffersize, int[] columns, boolean exitOnFail) {
// this creates a new tree
super(file, buffersize, thisOHBytes, thisOHHandles, columns, thisFHandles, columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
try {
setHandle(root, null); // define the root value
setHandle(toor, null); // define the toor value
} catch (IOException e) {
super.logFailure("cannot set root/toor handles / " + e.getMessage());
if (exitOnFail) System.exit(-1);
throw new RuntimeException("cannot set root/toor handles / " + e.getMessage());
}
}
public kelondroStack(File file, long buffersize) throws IOException{
@ -90,7 +96,7 @@ public final class kelondroStack extends kelondroRecords {
setHandle(toor, null); // reset the toor value
}
public static kelondroStack reset(kelondroStack stack) throws IOException {
public static kelondroStack reset(kelondroStack stack) {
// memorize settings to this file
File f = new File(stack.filename);
long bz = stack.XcacheSize * stack.cacheChunkSize(true);
@ -101,7 +107,7 @@ public final class kelondroStack extends kelondroRecords {
if (f.exists()) f.delete();
// re-open a database with same settings as before
return new kelondroStack(f, bz, cols);
return new kelondroStack(f, bz, cols, true);
}
public class Counter implements Iterator {
@ -410,7 +416,7 @@ public final class kelondroStack extends kelondroRecords {
int[] lens = new int[2];
lens[0] = Integer.parseInt(args[1]);
lens[1] = Integer.parseInt(args[2]);
kelondroStack fm = new kelondroStack(f, 0x100000, lens);
kelondroStack fm = new kelondroStack(f, 0x100000, lens, true);
fm.close();
} else if (args[0].equals("-p")) {
kelondroStack fm = new kelondroStack(new File(args[3]), 0x100000);

26
source/de/anomic/kelondro/kelondroTables.java
Unescape View File

@ -61,39 +61,45 @@ public class kelondroTables {
if (!(tablesPath.exists())) tablesPath.mkdirs();
}
public void declareMaps(String tablename, int keysize, int nodesize) throws IOException {
declareMaps(tablename, keysize, nodesize, null, null);
public void declareMaps(String tablename, int keysize, int nodesize, boolean exitOnFail) {
declareMaps(tablename, keysize, nodesize, null, null, exitOnFail);
}
public void declareMaps(String tablename, int keysize, int nodesize, String[] sortfields, String[] accfields) throws IOException {
declareMaps(tablename, keysize, nodesize, sortfields, accfields, 0x800);
public void declareMaps(String tablename, int keysize, int nodesize, String[] sortfields, String[] accfields, boolean exitOnFail) {
declareMaps(tablename, keysize, nodesize, sortfields, accfields, 0x800, exitOnFail);
}
public void declareMaps(String tablename, int keysize, int nodesize, String[] sortfields, String[] accfields, long buffersize /*bytes*/) throws IOException {
public void declareMaps(String tablename, int keysize, int nodesize, String[] sortfields, String[] accfields, long buffersize /*bytes*/, boolean exitOnFail) {
if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared twice.");
if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared already in other context.");
File tablefile = new File(tablesPath, "table." + tablename + ".mdb");
kelondroDyn dyn;
if (tablefile.exists()) {
if (tablefile.exists()) try {
dyn = new kelondroDyn(tablefile, buffersize);
} catch (IOException e) {
tablefile.getParentFile().mkdirs();
dyn = new kelondroDyn(tablefile, buffersize, keysize, nodesize, exitOnFail);
} else {
tablefile.getParentFile().mkdirs();
dyn = new kelondroDyn(tablefile, buffersize, keysize, nodesize);
dyn = new kelondroDyn(tablefile, buffersize, keysize, nodesize, exitOnFail);
}
kelondroMap map = new kelondroMap(dyn, sortfields, accfields);
mTables.put(tablename, map);
}
public void declareTree(String tablename, int[] columns, long buffersize /*bytes*/) throws IOException {
public void declareTree(String tablename, int[] columns, long buffersize /*bytes*/, boolean exitOnFail) {
if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared already in other context.");
if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared twice.");
File tablefile = new File(tablesPath, "table." + tablename + ".tdb");
kelondroTree Tree;
if (tablefile.exists()) {
if (tablefile.exists()) try {
Tree = new kelondroTree(tablefile, buffersize);
} catch (IOException e) {
tablefile.getParentFile().mkdirs();
Tree = new kelondroTree(tablefile, buffersize, columns, exitOnFail);
} else {
tablefile.getParentFile().mkdirs();
Tree = new kelondroTree(tablefile, buffersize, columns);
Tree = new kelondroTree(tablefile, buffersize, columns, exitOnFail);
}
tTables.put(tablename, Tree);
}

50
source/de/anomic/kelondro/kelondroTree.java
Unescape View File

@ -78,35 +78,47 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr
private Search writeSearchObj = new Search();
//private kelondroLock writeLock = new kelondroLock();
public kelondroTree(File file, long buffersize, int key, int value) throws IOException {
this(file, buffersize, new int[] { key, value }, 1, 8);
public kelondroTree(File file, long buffersize, int key, int value, boolean exitOnFail) {
this(file, buffersize, new int[] { key, value }, 1, 8, exitOnFail);
}
public kelondroTree(kelondroRA ra, long buffersize, int key, int value) throws IOException {
this(ra, buffersize, new int[] { key, value }, 1, 8);
public kelondroTree(kelondroRA ra, long buffersize, int key, int value, boolean exitOnFail) {
this(ra, buffersize, new int[] { key, value }, 1, 8, exitOnFail);
}
public kelondroTree(File file, long buffersize, int[] columns) throws IOException {
public kelondroTree(File file, long buffersize, int[] columns, boolean exitOnFail) {
// this creates a new tree file
this(file, buffersize, columns, columns.length /* txtProps */, 80 /* txtPropWidth */);
this(file, buffersize, columns, columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
}
public kelondroTree(File file, long buffersize, int[] columns, int txtProps, int txtPropsWidth) throws IOException {
public kelondroTree(File file, long buffersize, int[] columns, int txtProps, int txtPropsWidth, boolean exitOnFail) {
// this creates a new tree file
super(file, buffersize, thisOHBytes, thisOHHandles, columns, thisFHandles, txtProps, txtPropsWidth);
setHandle(root, null); // define the root value
super(file, buffersize, thisOHBytes, thisOHHandles, columns, thisFHandles, txtProps, txtPropsWidth, exitOnFail);
try {
setHandle(root, null); // define the root value
} catch (IOException e) {
super.logFailure("cannot set root handle / " + e.getMessage());
if (exitOnFail) System.exit(-1);
throw new RuntimeException("cannot set root handle / " + e.getMessage());
}
}
public kelondroTree(kelondroRA ra, long buffersize, int[] columns) throws IOException {
public kelondroTree(kelondroRA ra, long buffersize, int[] columns, boolean exitOnFail) {
// this creates a new tree within a kelondroRA
this(ra, buffersize, columns, columns.length /* txtProps */, 80 /* txtPropWidth */);
this(ra, buffersize, columns, columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
}
public kelondroTree(kelondroRA ra, long buffersize, int[] columns, int txtProps, int txtPropsWidth) throws IOException {
public kelondroTree(kelondroRA ra, long buffersize, int[] columns, int txtProps, int txtPropsWidth, boolean exitOnFail) {
// this creates a new tree within a kelondroRA
super(ra, buffersize, thisOHBytes, thisOHHandles, columns,
thisFHandles, txtProps, txtPropsWidth);
setHandle(root, null); // define the root value
thisFHandles, txtProps, txtPropsWidth, exitOnFail);
try {
setHandle(root, null); // define the root value
} catch (IOException e) {
super.logFailure("cannot set root handle / " + e.getMessage());
if (exitOnFail) System.exit(-1);
throw new RuntimeException("cannot set root handle / " + e.getMessage());
}
}
public kelondroTree(File file, long buffersize) throws IOException {
@ -1185,7 +1197,7 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr
// test script
File testFile = new File("test.db");
while (testFile.exists()) testFile.delete();
kelondroTree fm = new kelondroTree(testFile, 0x100000, 4, 4);
kelondroTree fm = new kelondroTree(testFile, 0x100000, 4, 4, true);
byte[] dummy = "".getBytes();
fm.put("abc0".getBytes(), dummy); fm.put("bcd0".getBytes(), dummy);
fm.put("def0".getBytes(), dummy); fm.put("bab0".getBytes(), dummy);
@ -1263,7 +1275,7 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr
int[] lens = new int[2];
lens[0] = Integer.parseInt(args[1]);
lens[1] = Integer.parseInt(args[2]);
kelondroTree fm = new kelondroTree(f, 0x100000, lens);
kelondroTree fm = new kelondroTree(f, 0x100000, lens, true);
fm.close();
} else if (args[0].equals("-u")) {
kelondroTree fm = new kelondroTree(new File(args[3]), 0x100000);
@ -1366,7 +1378,7 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr
int steps = 0;
while (true) {
if (testFile.exists()) testFile.delete();
tt = new kelondroTree(testFile, 200, 4 ,4);
tt = new kelondroTree(testFile, 200, 4 ,4, true);
steps = 10 + ((int) System.currentTimeMillis() % 7) * (((int) System.currentTimeMillis() + 17) % 11);
t = s;
d = "";
@ -1432,7 +1444,7 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr
File f = new File("test.db");
if (f.exists()) f.delete();
try {
kelondroTree tt = new kelondroTree(f, 0, 4, 4);
kelondroTree tt = new kelondroTree(f, 0, 4, 4, true);
byte[] b;
b = testWord('B'); tt.put(b, b); //tt.print();
b = testWord('C'); tt.put(b, b); //tt.print();
@ -1464,7 +1476,7 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr
public static kelondroTree testTree(File f, String testentities) throws IOException {
if (f.exists()) f.delete();
kelondroTree tt = new kelondroTree(f, 0, 4, 4);
kelondroTree tt = new kelondroTree(f, 0, 4, 4, true);
byte[] b;
for (int i = 0; i < testentities.length(); i++) {
b = testWord(testentities.charAt(i));

13
source/de/anomic/plasma/plasmaCrawlBalancer.java
Unescape View File

@ -56,15 +56,16 @@ public class plasmaCrawlBalancer {
private kelondroStack stack;
private HashMap domainStacks;
public plasmaCrawlBalancer(File stackFile, long buffersize) throws IOException {
if (stackFile.exists())
public plasmaCrawlBalancer(File stackFile, long buffersize) {
if (stackFile.exists()) {
try {
stack = new kelondroStack(stackFile, buffersize);
} catch (IOException e) {
stack = new kelondroStack(stackFile, buffersize, new int[] {plasmaURL.urlHashLength});
stack = new kelondroStack(stackFile, buffersize, new int[] {plasmaURL.urlHashLength}, true);
}
else
stack = new kelondroStack(stackFile, buffersize, new int[] {plasmaURL.urlHashLength});
} else {
stack = new kelondroStack(stackFile, buffersize, new int[] {plasmaURL.urlHashLength}, true);
}
domainStacks = new HashMap();
}
@ -74,7 +75,7 @@ public class plasmaCrawlBalancer {
stack = null;
}
public void reset() throws IOException {
public void reset() {
stack = kelondroStack.reset(stack);
domainStacks = new HashMap();
}

21
source/de/anomic/plasma/plasmaCrawlEURL.java
Unescape View File

@ -60,7 +60,7 @@ public class plasmaCrawlEURL extends plasmaURL {
private LinkedList rejectedStack = new LinkedList(); // strings: url
public plasmaCrawlEURL(File cachePath, int bufferkb) throws IOException {
public plasmaCrawlEURL(File cachePath, int bufferkb) {
super();
int[] ce = {
urlHashLength, // the url's hash
@ -75,14 +75,17 @@ public class plasmaCrawlEURL extends plasmaURL {
urlErrorLength, // string describing load failure
urlFlagLength // extra space
};
if (cachePath.exists()) {
// open existing cache
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400);
} else {
// create new cache
cachePath.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, ce);
}
if (cachePath.exists()) try {
// open existing cache
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400);
} catch (IOException e) {
cachePath.delete();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, ce, true);
} else {
// create new cache
cachePath.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, ce, true);
}
}
public synchronized Entry newEntry(URL url, String referrer, String initiator, String executor,

11
source/de/anomic/plasma/plasmaCrawlLURL.java
Unescape View File

@ -87,7 +87,7 @@ public final class plasmaCrawlLURL extends plasmaURL {
//public static Set damagedURLS = Collections.synchronizedSet(new HashSet());
public plasmaCrawlLURL(File cachePath, int bufferkb) throws IOException {
public plasmaCrawlLURL(File cachePath, int bufferkb) {
super();
int[] ce = {
urlHashLength,
@ -108,11 +108,16 @@ public final class plasmaCrawlLURL extends plasmaURL {
for (int i = 0; i < ce.length; i++) { segmentsize += ce[i]; }
if (cachePath.exists()) {
// open existing cache
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400);
try {
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400);
} catch (IOException e) {
cachePath.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, ce, true);
}
} else {
// create new cache
cachePath.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, ce);
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, ce, true);
}
// init result stacks

88
source/de/anomic/plasma/plasmaCrawlNURL.java
Unescape View File

@ -99,7 +99,7 @@ public class plasmaCrawlNURL extends plasmaURL {
private File cacheStacksPath;
private int bufferkb;
public plasmaCrawlNURL(File cacheStacksPath, int bufferkb) throws IOException {
public plasmaCrawlNURL(File cacheStacksPath, int bufferkb) {
super();
this.cacheStacksPath = cacheStacksPath;
this.bufferkb = bufferkb;
@ -123,23 +123,23 @@ public class plasmaCrawlNURL extends plasmaURL {
if (imageStackFile.exists()) try {
imageStack = new kelondroStack(imageStackFile, 0);
} catch (IOException e) {
imageStack = new kelondroStack(imageStackFile, 0, new int[] {plasmaURL.urlHashLength});
imageStack = new kelondroStack(imageStackFile, 0, new int[] {plasmaURL.urlHashLength}, true);
} else {
imageStack = new kelondroStack(imageStackFile, 0, new int[] {plasmaURL.urlHashLength});
imageStack = new kelondroStack(imageStackFile, 0, new int[] {plasmaURL.urlHashLength}, true);
}
if (movieStackFile.exists()) try {
movieStack = new kelondroStack(movieStackFile, 0);
} catch (IOException e) {
movieStack = new kelondroStack(movieStackFile, 0, new int[] {plasmaURL.urlHashLength});
movieStack = new kelondroStack(movieStackFile, 0, new int[] {plasmaURL.urlHashLength}, true);
} else {
movieStack = new kelondroStack(movieStackFile, 0, new int[] {plasmaURL.urlHashLength});
movieStack = new kelondroStack(movieStackFile, 0, new int[] {plasmaURL.urlHashLength}, true);
}
if (musicStackFile.exists()) try {
musicStack = new kelondroStack(musicStackFile, 0);
} catch (IOException e) {
musicStack = new kelondroStack(musicStackFile, 0, new int[] {plasmaURL.urlHashLength});
musicStack = new kelondroStack(musicStackFile, 0, new int[] {plasmaURL.urlHashLength}, true);
} else {
musicStack = new kelondroStack(musicStackFile, 0, new int[] {plasmaURL.urlHashLength});
musicStack = new kelondroStack(musicStackFile, 0, new int[] {plasmaURL.urlHashLength}, true);
}
// init stack Index
@ -147,19 +147,22 @@ public class plasmaCrawlNURL extends plasmaURL {
new initStackIndex().start();
}
private void openHashCache() throws IOException {
private void openHashCache() {
File cacheFile = new File(cacheStacksPath, "urlNotice1.db");
if (cacheFile.exists()) {
if (cacheFile.exists()) try {
// open existing cache
urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400);
} catch (IOException e) {
cacheFile.delete();
urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, ce, true);
} else {
// create new cache
cacheFile.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, ce);
urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, ce, true);
}
}
private void resetHashCache() throws IOException {
private void resetHashCache() {
if (urlHashCache != null) {
try {urlHashCache.close();} catch (IOException e) {}
urlHashCache = null;
@ -186,14 +189,47 @@ public class plasmaCrawlNURL extends plasmaURL {
public void run() {
Iterator i;
try {
try { i = coreStack.iterator(); while (i.hasNext()) stackIndex.add(new String((byte[]) i.next()));} catch (Exception e) {coreStack.reset();}
try { i = limitStack.iterator(); while (i.hasNext()) stackIndex.add(new String((byte[]) i.next()));} catch (Exception e) {limitStack.reset();}
try { i = overhangStack.iterator(); while (i.hasNext()) stackIndex.add(new String((byte[]) i.next()));} catch (Exception e) {overhangStack.reset();}
try { i = remoteStack.iterator(); while (i.hasNext()) stackIndex.add(new String((byte[]) i.next()));} catch (Exception e) {remoteStack.reset();}
try { i = imageStack.iterator(); while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey()));} catch (Exception e) {imageStack = kelondroStack.reset(imageStack);}
try { i = movieStack.iterator(); while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey()));} catch (Exception e) {movieStack = kelondroStack.reset(movieStack);}
try { i = musicStack.iterator(); while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey()));} catch (Exception e) {musicStack = kelondroStack.reset(musicStack);}
} catch (IOException e) {}
i = coreStack.iterator();
while (i.hasNext()) stackIndex.add(new String((byte[]) i.next()));
} catch (Exception e) {
coreStack.reset();
}
try {
i = limitStack.iterator();
while (i.hasNext()) stackIndex.add(new String((byte[]) i.next()));
} catch (Exception e) {
limitStack.reset();
}
try {
i = overhangStack.iterator();
while (i.hasNext()) stackIndex.add(new String((byte[]) i.next()));
} catch (Exception e) {
overhangStack.reset();
}
try {
i = remoteStack.iterator();
while (i.hasNext()) stackIndex.add(new String((byte[]) i.next()));
} catch (Exception e) {
remoteStack.reset();
}
try {
i = imageStack.iterator();
while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey()));
} catch (Exception e) {
imageStack = kelondroStack.reset(imageStack);
}
try {
i = movieStack.iterator();
while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey()));
} catch (Exception e) {
movieStack = kelondroStack.reset(movieStack);
}
try {
i = musicStack.iterator();
while (i.hasNext()) stackIndex.add(new String(((kelondroRecords.Node) i.next()).getKey()));
} catch (Exception e) {
musicStack = kelondroStack.reset(musicStack);
}
}
}
@ -500,21 +536,11 @@ public class plasmaCrawlNURL extends plasmaURL {
} catch (IOException e) {
serverLog.logSevere("PLASMA", "INTERNAL ERROR AT plasmaNURL:store:" + e.toString() + ", resetting NURL-DB");
e.printStackTrace();
try {
resetHashCache();
} catch (IOException ee) {
serverLog.logSevere("PLASMA", "INTERNAL ERROR AT plasmaNURL:store:" + e.toString() + ", reset of NURL-DB failed");
System.exit(0); // this is REALLY severe
}
resetHashCache();
} catch (kelondroException e) {
serverLog.logSevere("PLASMA", "plasmaCrawlNURL.store failed: " + e.toString() + ", resetting NURL-DB");
e.printStackTrace();
try {
resetHashCache();
} catch (IOException ee) {
serverLog.logSevere("PLASMA", "INTERNAL ERROR AT plasmaNURL:store:" + e.toString() + ", reset of NURL-DB failed");
System.exit(0); // this is REALLY severe
}
resetHashCache();
}
}

55
source/de/anomic/plasma/plasmaCrawlProfile.java
Unescape View File

@ -52,7 +52,6 @@ import de.anomic.kelondro.kelondroDyn;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMap;
import de.anomic.server.serverCodings;
import de.anomic.server.logging.serverLog;
public class plasmaCrawlProfile {
@ -60,14 +59,19 @@ public class plasmaCrawlProfile {
private File profileTableFile;
private int bufferkb;
public plasmaCrawlProfile(File profileTableFile, int bufferkb) throws IOException {
this.profileTableFile = profileTableFile;
if (profileTableFile.exists()) {
profileTable = new kelondroMap(new kelondroDyn(profileTableFile, bufferkb * 1024));
public plasmaCrawlProfile(File file, int bufferkb) {
this.profileTableFile = file;
kelondroDyn dyn = null;
if (profileTableFile.exists()) try {
dyn = new kelondroDyn(file, bufferkb * 1024);
} catch (IOException e) {
profileTableFile.delete();
dyn = new kelondroDyn(file, bufferkb * 1024, plasmaURL.urlCrawlProfileHandleLength, 2000, true);
} else {
profileTableFile.getParentFile().mkdirs();
profileTable = new kelondroMap(new kelondroDyn(profileTableFile, bufferkb * 1024, plasmaURL.urlCrawlProfileHandleLength, 2000));
dyn = new kelondroDyn(file, bufferkb * 1024, plasmaURL.urlCrawlProfileHandleLength, 2000, true);
}
profileTable = new kelondroMap(dyn);
}
public int[] dbCacheChunkSize() {
@ -80,16 +84,10 @@ public class plasmaCrawlProfile {
private void resetDatabase() {
// deletes the profile database and creates a new one
if (profileTable != null) try {
profileTable.close();
} catch (IOException e) {}
if (profileTable != null) try { profileTable.close(); } catch (IOException e) {}
if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database");
try {
profileTableFile.getParentFile().mkdirs();
profileTable = new kelondroMap(new kelondroDyn(profileTableFile, bufferkb * 1024, plasmaURL.urlCrawlProfileHandleLength, 2000));
} catch (IOException e){
serverLog.logSevere("PLASMA", "plasmaCrawlProfile.resetDatabase", e);
}
profileTableFile.getParentFile().mkdirs();
profileTable = new kelondroMap(new kelondroDyn(profileTableFile, bufferkb * 1024, plasmaURL.urlCrawlProfileHandleLength, 2000, true));
}
public void close() {
@ -104,11 +102,11 @@ public class plasmaCrawlProfile {
public Iterator profiles(boolean up) {
// enumerates profile entries
try {
return new profileIterator(up);
} catch (IOException e) {
return new HashSet().iterator();
}
try {
return new profileIterator(up);
} catch (IOException e) {
return new HashSet().iterator();
}
}
public class profileIterator implements Iterator {
@ -156,7 +154,7 @@ public class plasmaCrawlProfile {
boolean crawlingQ,
boolean storeHTCache, boolean storeTXCache,
boolean localIndexing, boolean remoteIndexing,
boolean xsstopw, boolean xdstopw, boolean xpstopw) throws IOException {
boolean xsstopw, boolean xdstopw, boolean xpstopw) {
entry ne = new entry(name, startURL, generalFilter, specificFilter,
generalDepth, specificDepth,
@ -166,7 +164,20 @@ public class plasmaCrawlProfile {
profileTable.set(ne.handle(), ne.map());
} catch (kelondroException e) {
resetDatabase();
profileTable.set(ne.handle(), ne.map());
try {
profileTable.set(ne.handle(), ne.map());
} catch (IOException ee) {
e.printStackTrace();
System.exit(0);
}
} catch (IOException e) {
resetDatabase();
try {
profileTable.set(ne.handle(), ne.map());
} catch (IOException ee) {
e.printStackTrace();
System.exit(0);
}
}
return ne;
}

19
source/de/anomic/plasma/plasmaCrawlRobotsTxt.java
Unescape View File

@ -57,14 +57,13 @@ import java.util.Map;
import de.anomic.kelondro.kelondroDyn;
import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroException;
import de.anomic.server.logging.serverLog;
public class plasmaCrawlRobotsTxt {
kelondroMap robotsTable;
private final File robotsTableFile;
private int bufferkb;
public plasmaCrawlRobotsTxt(File robotsTableFile, int bufferkb) throws IOException {
public plasmaCrawlRobotsTxt(File robotsTableFile, int bufferkb) {
this.robotsTableFile = robotsTableFile;
this.bufferkb = bufferkb;
if (robotsTableFile.exists()) {
@ -72,12 +71,14 @@ public class plasmaCrawlRobotsTxt {
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024));
} catch (kelondroException e) {
robotsTableFile.delete();
robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, 256, 512));
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, 256, 512, true));
} catch (IOException e) {
robotsTableFile.delete();
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, 256, 512, true));
}
} else {
robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, 256, 512));
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, 256, 512, true));
}
}
@ -95,12 +96,8 @@ public class plasmaCrawlRobotsTxt {
robotsTable.close();
} catch (IOException e) {}
if (!(robotsTableFile.delete())) throw new RuntimeException("cannot delete robots.txt database");
try {
robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, this.bufferkb, 256, 512));
} catch (IOException e){
serverLog.logSevere("PLASMA", "robotsTxt.resetDatabase", e);
}
robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, this.bufferkb, 256, 512, true));
}
public void close() {

15
source/de/anomic/plasma/plasmaCrawlStacker.java
Unescape View File

@ -78,7 +78,7 @@ public final class plasmaCrawlStacker {
//private boolean stopped = false;
private stackCrawlQueue queue;
public plasmaCrawlStacker(plasmaSwitchboard sb, File dbPath, int dbCacheSize) throws IOException {
public plasmaCrawlStacker(plasmaSwitchboard sb, File dbPath, int dbCacheSize) {
this.sb = sb;
this.queue = new stackCrawlQueue(dbPath,dbCacheSize);
@ -500,7 +500,7 @@ public final class plasmaCrawlStacker {
private final LinkedList urlEntryHashCache;
private kelondroTree urlEntryCache;
public stackCrawlQueue(File cacheStacksPath, int bufferkb) throws IOException {
public stackCrawlQueue(File cacheStacksPath, int bufferkb) {
// init the read semaphore
this.readSync = new serverSemaphore (0);
@ -516,7 +516,12 @@ public final class plasmaCrawlStacker {
File cacheFile = new File(cacheStacksPath, "urlPreNotice.db");
if (cacheFile.exists()) {
// open existing cache
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400);
try {
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400);
} catch (IOException e) {
cacheFile.delete();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, plasmaCrawlNURL.ce, true);
}
try {
// loop through the list and fill the messageList with url hashs
Iterator iter = this.urlEntryCache.nodeIterator(true,false);
@ -538,12 +543,12 @@ public final class plasmaCrawlStacker {
// deleting old db and creating a new db
try {this.urlEntryCache.close();}catch(Exception ex){}
cacheFile.delete();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, plasmaCrawlNURL.ce);
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, plasmaCrawlNURL.ce, true);
}
} else {
// create new cache
cacheFile.getParentFile().mkdirs();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, plasmaCrawlNURL.ce);
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, plasmaCrawlNURL.ce, true);
}
}

3
source/de/anomic/plasma/plasmaDbImporter.java
Unescape View File

@ -1,7 +1,6 @@
package de.anomic.plasma;
import java.io.File;
import java.io.IOException;
import java.util.Iterator;
import java.util.Vector;
@ -142,7 +141,7 @@ public class plasmaDbImporter extends Thread {
return this.importWordIndex.size();
}
public plasmaDbImporter(plasmaWordIndex theHomeIndexDB, plasmaCrawlLURL theHomeUrlDB, String theImportPath) throws IOException {
public plasmaDbImporter(plasmaWordIndex theHomeIndexDB, plasmaCrawlLURL theHomeUrlDB, String theImportPath) {
super(runningJobs,"DB-Import_" + theImportPath);
this.log = new serverLog("DB-IMPORT");

2
source/de/anomic/plasma/plasmaHTCache.java
Unescape View File

@ -114,7 +114,7 @@ public final class plasmaHTCache {
if (dbfile.exists())
this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400));
else
this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400, plasmaURL.urlHashLength, 150));
this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400, plasmaURL.urlHashLength, 150, false));
} catch (IOException e) {
this.log.logSevere("the request header database could not be opened: " + e.getMessage());
System.exit(0);

38
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -118,6 +118,7 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.logging.Level;
import de.anomic.data.messageBoard;
import de.anomic.data.wikiBoard;
import de.anomic.data.userDB;
@ -220,7 +221,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
private boolean crawlingIsPaused = false;
private static plasmaSwitchboard sb;
public plasmaSwitchboard(String rootPath, String initPath, String configPath) throws IOException {
public plasmaSwitchboard(String rootPath, String initPath, String configPath) {
super(rootPath, initPath, configPath);
// set loglevel and log
@ -370,16 +371,19 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// going through the sbQueue Entries and registering all content files as in use
int count = 0;
ArrayList sbQueueEntries = this.sbQueue.list();
for (int i=0; i<sbQueueEntries.size(); i++) {
plasmaSwitchboardQueue.Entry entry = (plasmaSwitchboardQueue.Entry) sbQueueEntries.get(i);
if ((entry != null)&&(entry.url() != null)&&(entry.cacheFile().exists())) {
plasmaHTCache.filesInUse.add(entry.cacheFile());
count++;
try {
ArrayList sbQueueEntries = this.sbQueue.list();
for (int i = 0; i < sbQueueEntries.size(); i++) {
plasmaSwitchboardQueue.Entry entry = (plasmaSwitchboardQueue.Entry) sbQueueEntries.get(i);
if ((entry != null) && (entry.url() != null) && (entry.cacheFile().exists())) {
plasmaHTCache.filesInUse.add(entry.cacheFile());
count++;
}
}
this.log.logConfig(count + " files in htcache reported to the cachemanager as in use.");
} catch (IOException e) {
this.log.logSevere("cannot find any files in htcache reported to the cachemanager: " + e.getMessage());
}
this.log.logConfig(count + " files in htcache reported to the cachemanager as in use.");
// define an extension-blacklist
log.logConfig("Parser: Initializing Extension Mappings for Media/Parser");
plasmaParser.initMediaExt(plasmaParser.extString2extList(getConfig("mediaExt","")));
@ -588,10 +592,10 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
return (bytes / 1024) + "TByte";
}
private void initProfiles() throws IOException {
private void initProfiles() {
if ((profiles.size() == 0) ||
(getConfig(STR_PROXYPROFILE, "").length() == 0) ||
(profiles.getEntry(getConfig(STR_PROXYPROFILE, "")) == null)) {
(getConfig(STR_PROXYPROFILE, "").length() == 0) ||
(profiles.getEntry(getConfig(STR_PROXYPROFILE, "")) == null)) {
// generate new default entry for proxy crawling
defaultProxyProfile = profiles.newEntry("proxy", "", ".*", ".*", Integer.parseInt(getConfig("proxyPrefetchDepth", "0")), Integer.parseInt(getConfig("proxyPrefetchDepth", "0")), false, true, true, true, false, true, true, true);
setConfig(STR_PROXYPROFILE, defaultProxyProfile.handle());
@ -608,15 +612,15 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
defaultRemoteProfile = profiles.getEntry(getConfig(STR_REMOTEPROFILE, ""));
}
}
private void resetProfiles() {
final File pdb = new File(plasmaPath, "crawlProfiles0.db");
if (pdb.exists()) pdb.delete();
try {
int ramProfiles = (int) getConfigLong("ramCacheProfiles",1024) / 1024;
profiles = new plasmaCrawlProfile(pdb, ramProfiles);
initProfiles();
} catch (IOException e) {}
int ramProfiles = (int) getConfigLong("ramCacheProfiles", 1024) / 1024;
profiles = new plasmaCrawlProfile(pdb, ramProfiles);
initProfiles();
}
public boolean cleanProfiles() {
if ((sbQueue.size() > 0) || (cacheLoader.size() > 0) || (urlPool.noticeURL.stackSize() > 0)) return false;
final Iterator iter = profiles.profiles(true);

21
source/de/anomic/plasma/plasmaSwitchboardQueue.java
Unescape View File

@ -65,14 +65,26 @@ public class plasmaSwitchboardQueue {
private plasmaHTCache htCache;
private plasmaCrawlLURL lurls;
public plasmaSwitchboardQueue(plasmaHTCache htCache, plasmaCrawlLURL lurls, File sbQueueStackPath, plasmaCrawlProfile profiles) throws IOException {
public plasmaSwitchboardQueue(plasmaHTCache htCache, plasmaCrawlLURL lurls, File sbQueueStackPath, plasmaCrawlProfile profiles) {
this.profiles = profiles;
this.htCache = htCache;
this.lurls = lurls;
if (sbQueueStackPath.exists())
if (sbQueueStackPath.exists()) try {
sbQueueStack = new kelondroStack(sbQueueStackPath, 0);
else
} catch (IOException e) {
sbQueueStackPath.delete();
sbQueueStack = new kelondroStack(sbQueueStackPath, 0, new int[] {
plasmaURL.urlStringLength,
plasmaURL.urlHashLength,
11,
1,
yacySeedDB.commonHashLength,
plasmaURL.urlCrawlDepthLength,
plasmaURL.urlCrawlProfileHandleLength,
plasmaURL.urlDescrLength
}, true);
} else {
sbQueueStack = new kelondroStack(sbQueueStackPath, 0, new int[] {
plasmaURL.urlStringLength,
plasmaURL.urlHashLength,
@ -82,7 +94,8 @@ public class plasmaSwitchboardQueue {
plasmaURL.urlCrawlDepthLength,
plasmaURL.urlCrawlProfileHandleLength,
plasmaURL.urlDescrLength
});
}, true);
}
}
public int size() {

2
source/de/anomic/plasma/plasmaURLPool.java
Unescape View File

@ -55,7 +55,7 @@ public class plasmaURLPool {
public final plasmaCrawlNURL noticeURL;
public final plasmaCrawlEURL errorURL;
public plasmaURLPool(File plasmaPath, int ramLURL, int ramNURL, int ramEURL) throws IOException {
public plasmaURLPool(File plasmaPath, int ramLURL, int ramNURL, int ramEURL) {
loadedURL = new plasmaCrawlLURL(new File(plasmaPath, "urlHash.db"), ramLURL);
noticeURL = new plasmaCrawlNURL(plasmaPath, ramNURL);
errorURL = new plasmaCrawlEURL(new File(plasmaPath, "urlErr0.db"), ramEURL);

12
source/de/anomic/plasma/plasmaWordConnotation.java
Unescape View File

@ -58,12 +58,14 @@ public class plasmaWordConnotation {
private static final int nodesize = 4048;
private kelondroDynTree refDB;
public plasmaWordConnotation(File refDBfile, int bufferkb) throws IOException {
if (refDBfile.exists())
public plasmaWordConnotation(File refDBfile, int bufferkb) {
if (refDBfile.exists()) try {
refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400);
else
refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, wordlength, nodesize, new int[] {wordlength, countlength});
} catch (IOException e) {
refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, wordlength, nodesize, new int[] {wordlength, countlength}, true);
} else {
refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, wordlength, nodesize, new int[] {wordlength, countlength}, true);
}
}
private void addSingleRef(String word, String reference) throws IOException {

14
source/de/anomic/plasma/plasmaWordIndexAssortment.java
Unescape View File

@ -120,12 +120,8 @@ public final class plasmaWordIndexAssortment {
assortmentFile.delete(); // make space for new one
}
// create new assortment tree file
try {
assortments = new kelondroTree(assortmentFile, bufferSize, bufferStructure(assortmentLength));
if (log != null) log.logConfig("Created new Assortment Database, width " + assortmentLength + ", " + bufferkb + "kb buffer");
} catch (IOException e){
serverLog.logSevere("PLASMA", "unable to create assortment database: " + e.getMessage(), e);
}
assortments = new kelondroTree(assortmentFile, bufferSize, bufferStructure(assortmentLength), true);
if (log != null) log.logConfig("Created new Assortment Database, width " + assortmentLength + ", " + bufferkb + "kb buffer");
}
public void store(String wordHash, plasmaWordIndexEntryContainer newContainer) {
@ -194,11 +190,7 @@ public final class plasmaWordIndexAssortment {
assortments.close();
} catch (IOException e) {}
if (!(assortmentFile.delete())) throw new RuntimeException("cannot delete assortment database");
try {
assortments = new kelondroTree(assortmentFile, bufferSize, bufferStructure(assortmentLength));
} catch (IOException e){
log.logSevere("unable to re-create assortment database: " + e.getMessage(), e);
}
assortments = new kelondroTree(assortmentFile, bufferSize, bufferStructure(assortmentLength), true);
}
public Iterator hashes(String startWordHash, boolean up, boolean rot) {

6
source/de/anomic/plasma/plasmaWordIndexCache.java
Unescape View File

@ -134,8 +134,7 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
File indexDumpFile = new File(databaseRoot, indexArrayFileName);
if (indexDumpFile.exists()) indexDumpFile.delete();
kelondroArray dumpArray = null;
try {
dumpArray = new kelondroArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0);
dumpArray = new kelondroArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0, false);
long startTime = System.currentTimeMillis();
long messageTime = System.currentTimeMillis() + 5000;
long wordsPerSecond = 0, wordcount = 0, urlcount = 0;
@ -182,9 +181,6 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
dumpArray.close();
dumpArray = null;
log.logConfig("dumped " + urlcount + " word/URL relations in " + ((System.currentTimeMillis() - startTime) / 1000) + " seconds");
} finally {
if (dumpArray != null) try {dumpArray.close();}catch(Exception e){}
}
}
private long restore() throws IOException {

7
source/de/anomic/plasma/plasmaWordIndexClassicDB.java
Unescape View File

@ -182,12 +182,7 @@ public class plasmaWordIndexClassicDB implements plasmaWordIndexInterface {
}
public plasmaWordIndexEntity getIndex(String wordHash, boolean deleteIfEmpty, long maxTime) {
try {
return new plasmaWordIndexEntity(databaseRoot, wordHash, deleteIfEmpty);
} catch (IOException e) {
log.logSevere("plasmaWordIndexClassic.getIndex: " + e.getMessage());
return null;
}
return new plasmaWordIndexEntity(databaseRoot, wordHash, deleteIfEmpty);
}
public long getUpdateTime(String wordHash) {

13
source/de/anomic/plasma/plasmaWordIndexEntity.java
Unescape View File

@ -62,7 +62,7 @@ public final class plasmaWordIndexEntity {
private File theLocation;
private boolean delete;
public plasmaWordIndexEntity(File databaseRoot, String wordHash, boolean deleteIfEmpty) throws IOException {
public plasmaWordIndexEntity(File databaseRoot, String wordHash, boolean deleteIfEmpty) {
theWordHash = wordHash;
theIndex = indexFile(databaseRoot, wordHash);
theTmpMap = null;
@ -82,20 +82,23 @@ public final class plasmaWordIndexEntity {
return success;
}
private kelondroTree indexFile(File databaseRoot, String wordHash) throws IOException {
if (wordHash.length() < 12) throw new IOException("word hash wrong: '" + wordHash + "'");
private kelondroTree indexFile(File databaseRoot, String wordHash) {
if (wordHash.length() < 12) throw new RuntimeException("word hash wrong: '" + wordHash + "'");
theLocation = wordHash2path(databaseRoot, wordHash);
File fp = theLocation.getParentFile();
if (fp != null) fp.mkdirs();
kelondroTree kt;
long cacheSize = theLocation.length();
if (cacheSize > 1048576) cacheSize = 1048576;
if (theLocation.exists()) {
if (theLocation.exists()) try {
// open existing index file
kt = new kelondroTree(theLocation, cacheSize);
} catch (IOException e) {
theLocation.delete();
kt = new kelondroTree(theLocation, cacheSize, plasmaURL.urlHashLength, plasmaWordIndexEntry.attrSpaceShort, false);
} else {
// create new index file
kt = new kelondroTree(theLocation, cacheSize, plasmaURL.urlHashLength, plasmaWordIndexEntry.attrSpaceShort);
kt = new kelondroTree(theLocation, cacheSize, plasmaURL.urlHashLength, plasmaWordIndexEntry.attrSpaceShort, false);
}
return kt; // everyone who get this should close it when finished!
}

4
source/de/anomic/urlRedirector/urlRedirectord.java
Unescape View File

@ -34,7 +34,6 @@ public class urlRedirectord implements serverHandler {
}
if (profile == null) {
try {
profile = switchboard.profiles.newEntry(
// name
"URL Redirector",
@ -63,9 +62,6 @@ public class urlRedirectord implements serverHandler {
// xpstopw
true
);
} catch (IOException e) {
this.theLogger.logSevere("Unable to create a crawling profile for the URL-Redirector",e);
}
}
}

2
source/de/anomic/yacy/yacyCore.java
Unescape View File

@ -147,7 +147,7 @@ public class yacyCore {
return (int) ((System.currentTimeMillis() - startupTime) / 1000);
}
public yacyCore(plasmaSwitchboard sb) throws IOException {
public yacyCore(plasmaSwitchboard sb) {
long time = System.currentTimeMillis();
this.switchboard = sb;

17
source/de/anomic/yacy/yacyNewsDB.java
Unescape View File

@ -63,27 +63,30 @@ public class yacyNewsDB {
- yacyCore.universalDateShortPattern.length()
- 2;
public yacyNewsDB(File path, int bufferkb) throws IOException {
public yacyNewsDB(File path, int bufferkb) {
this.path = path;
this.bufferkb = bufferkb;
if (path.exists())
if (path.exists()) try {
news = new kelondroTree(path, bufferkb * 0x400);
else
news = createDB(path, bufferkb);
} catch (IOException e) {
news = createDB(path, bufferkb);
} else {
news = createDB(path, bufferkb);
}
}
private static kelondroTree createDB(File path, int bufferkb) throws IOException {
private static kelondroTree createDB(File path, int bufferkb) {
return new kelondroTree(path, bufferkb * 0x400, new int[] {
yacyNewsRecord.idLength(), // id = created + originator
yacyNewsRecord.categoryStringLength, // category
yacyCore.universalDateShortPattern.length(), // received
2,
attributesMaxLength
});
}, true);
}
private void resetDB() throws IOException {
private void resetDB() {
try {close();} catch (Exception e) {}
if (path.exists()) path.delete();
news = createDB(path, bufferkb);

4
source/de/anomic/yacy/yacyNewsPool.java
Unescape View File

@ -93,7 +93,7 @@ public class yacyNewsPool {
private int maxDistribution;
public yacyNewsPool(File yacyDBPath, int bufferkb) throws IOException {
public yacyNewsPool(File yacyDBPath, int bufferkb) {
newsDB = new yacyNewsDB(new File(yacyDBPath, "news1.db"), bufferkb);
outgoingNews = new yacyNewsQueue(new File(yacyDBPath, "newsOut1.stack"), newsDB);
publishedNews = new yacyNewsQueue(new File(yacyDBPath, "newsPublished1.stack"), newsDB);
@ -243,7 +243,7 @@ public class yacyNewsPool {
return null;
}
public void clear(int dbKey) throws IOException {
public void clear(int dbKey) {
// this is called if a queue element shall be moved to another queue or off the queue
// it depends on the dbKey how the record is handled
switch (dbKey) {

28
source/de/anomic/yacy/yacyNewsQueue.java
Unescape View File

@ -54,35 +54,37 @@ public class yacyNewsQueue {
private kelondroStack queueStack;
private yacyNewsDB newsDB;
public yacyNewsQueue(File path, yacyNewsDB newsDB) throws IOException {
public yacyNewsQueue(File path, yacyNewsDB newsDB) {
this.path = path;
this.newsDB = newsDB;
if (path.exists()) {
try {
queueStack = new kelondroStack(path, 0);
} catch (kelondroException e) {
path.delete();
queueStack = createStack(path);
}
} else
if (path.exists()) try {
queueStack = new kelondroStack(path, 0);
} catch (kelondroException e) {
path.delete();
queueStack = createStack(path);
} catch (IOException e) {
path.delete();
queueStack = createStack(path);
} else {
queueStack = createStack(path);
}
}
private static kelondroStack createStack(File path) throws IOException {
private static kelondroStack createStack(File path) {
return new kelondroStack(path, 0, new int[] {
yacyNewsRecord.idLength(), // id = created + originator
yacyCore.universalDateShortPattern.length() // last touched
});
}, true);
}
private void resetDB() throws IOException {
private void resetDB() {
try {close();} catch (Exception e) {}
if (path.exists()) path.delete();
queueStack = createStack(path);
}
public void clear() throws IOException {
public void clear() {
resetDB();
}

26
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -100,7 +100,7 @@ public final class yacySeedDB {
File seedActiveDBFile,
File seedPassiveDBFile,
File seedPotentialDBFile,
int bufferkb) throws IOException {
int bufferkb) {
this.seedDBBufferKB = bufferkb;
this.seedActiveDBFile = seedActiveDBFile;
@ -116,15 +116,27 @@ public final class yacySeedDB {
// create or init own seed
myOwnSeedFile = sb.getOwnSeedFile();
if (myOwnSeedFile.length() > 0) {
if (myOwnSeedFile.length() > 0) try {
// load existing identity
mySeed = yacySeed.load(myOwnSeedFile);
} catch (IOException e) {
// create new identity
mySeed = yacySeed.genLocalSeed(sb);
try {
mySeed.save(myOwnSeedFile);
} catch (IOException ee) {
ee.printStackTrace();
System.exit(-1);
}
} else {
// create new identity
mySeed = yacySeed.genLocalSeed(sb);
// save of for later use
mySeed.save(myOwnSeedFile); // in a file
//writeMap(mySeed.hash, mySeed.dna, "new"); // in a database
try {
mySeed.save(myOwnSeedFile);
} catch (IOException ee) {
ee.printStackTrace();
System.exit(-1);
}
}
if (sb.getConfig("portForwardingEnabled","false").equalsIgnoreCase("true")) {
@ -172,7 +184,7 @@ public final class yacySeedDB {
return new int[]{ac[0] + pa[0] + po[0], ac[1] + pa[1] + po[1], ac[2] + pa[2] + po[2], ac[3] + pa[3] + po[3]};
}
private synchronized kelondroMap openSeedTable(File seedDBFile) throws IOException {
private synchronized kelondroMap openSeedTable(File seedDBFile) {
if (seedDBFile.exists()) try {
// open existing seed database
return new kelondroMap(new kelondroDyn(seedDBFile, (seedDBBufferKB * 0x400) / 3), sortFields, accFields);
@ -185,7 +197,7 @@ public final class yacySeedDB {
}
// create new seed database
new File(seedDBFile.getParent()).mkdir();
return new kelondroMap(new kelondroDyn(seedDBFile, (seedDBBufferKB * 0x400) / 3, commonHashLength, 480), sortFields, accFields);
return new kelondroMap(new kelondroDyn(seedDBFile, (seedDBBufferKB * 0x400) / 3, commonHashLength, 480, true), sortFields, accFields);
}
private synchronized kelondroMap resetSeedTable(kelondroMap seedDB, File seedDBFile) {

Write Preview
Loading…
Cancel
Save