boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fix for bug that caused the peer-counter problem.

Browse Source 
See http://www.yacy-forum.de/viewtopic.php?p=16016#16016
The kelondroDyn now uses a generic fill character.
kelondroDyn-Tables containing peer/word/url-hashes must not use '_'
as fill character.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1498 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent 9bc62bfdac
commit 3419b3bcdd
13 changed files with 67 additions and 64 deletions
Show Stats Download Patch File Download Diff File

18
source/de/anomic/data/bookmarksDB.java
Unescape View File

@ -71,37 +71,37 @@ public class bookmarksDB {
if(bookmarksFile.exists()){ if(bookmarksFile.exists()){
try { try {
//open it //open it
this.bookmarksTable=new kelondroMap(new kelondroDyn(bookmarksFile, 1024*bufferkb)); this.bookmarksTable=new kelondroMap(new kelondroDyn(bookmarksFile, 1024*bufferkb, '_'));
} catch (IOException e) { } catch (IOException e) {
//database reset :-(( //database reset :-((
bookmarksFile.delete(); bookmarksFile.delete();
bookmarksFile.getParentFile().mkdirs(); bookmarksFile.getParentFile().mkdirs();
//urlHash is 12 bytes long //urlHash is 12 bytes long
this.bookmarksTable = new kelondroMap(new kelondroDyn(bookmarksFile, bufferkb * 1024, 12, 256, true)); this.bookmarksTable = new kelondroMap(new kelondroDyn(bookmarksFile, bufferkb * 1024, 12, 256, '_', true));
} }
}else{ }else{
//new database //new database
bookmarksFile.getParentFile().mkdirs(); bookmarksFile.getParentFile().mkdirs();
this.bookmarksTable = new kelondroMap(new kelondroDyn(bookmarksFile, bufferkb * 1024, 12, 256, true)); this.bookmarksTable = new kelondroMap(new kelondroDyn(bookmarksFile, bufferkb * 1024, 12, 256, '_', true));
} }
//tags //tags
//check if database exists //check if database exists
if(tagsFile.exists()){ if(tagsFile.exists()){
try { try {
//open it //open it
this.tagsTable=new kelondroMap(new kelondroDyn(tagsFile, 1024*bufferkb)); this.tagsTable=new kelondroMap(new kelondroDyn(tagsFile, 1024*bufferkb, '_'));
} catch (IOException e) { } catch (IOException e) {
//reset database //reset database
tagsFile.delete(); tagsFile.delete();
tagsFile.getParentFile().mkdirs(); tagsFile.getParentFile().mkdirs();
// max. 128 byte long tags // max. 128 byte long tags
this.tagsTable = new kelondroMap(new kelondroDyn(tagsFile, bufferkb * 1024, 128, 256, true)); this.tagsTable = new kelondroMap(new kelondroDyn(tagsFile, bufferkb * 1024, 128, 256, '_', true));
rebuildTags(); rebuildTags();
} }
}else{ }else{
//new database //new database
tagsFile.getParentFile().mkdirs(); tagsFile.getParentFile().mkdirs();
this.tagsTable = new kelondroMap(new kelondroDyn(tagsFile, bufferkb * 1024, 128, 256, true)); this.tagsTable = new kelondroMap(new kelondroDyn(tagsFile, bufferkb * 1024, 128, 256, '_', true));
rebuildTags(); rebuildTags();
} }
// dates // dates
@ -109,19 +109,19 @@ public class bookmarksDB {
if(datesFile.exists()){ if(datesFile.exists()){
try { try {
//open it //open it
this.datesTable=new kelondroMap(new kelondroDyn(datesFile, 1024*bufferkb)); this.datesTable=new kelondroMap(new kelondroDyn(datesFile, 1024*bufferkb, '_'));
} catch (IOException e) { } catch (IOException e) {
//reset database //reset database
datesFile.delete(); datesFile.delete();
datesFile.getParentFile().mkdirs(); datesFile.getParentFile().mkdirs();
//YYYY-MM-DDTHH:mm:ssZ = 20 byte. currently used: YYYY-MM-DD = 10 bytes //YYYY-MM-DDTHH:mm:ssZ = 20 byte. currently used: YYYY-MM-DD = 10 bytes
this.datesTable = new kelondroMap(new kelondroDyn(datesFile, bufferkb * 1024, 20, 256, true)); this.datesTable = new kelondroMap(new kelondroDyn(datesFile, bufferkb * 1024, 20, 256, '_', true));
rebuildDates(); rebuildDates();
} }
}else{ }else{
//new database //new database
datesFile.getParentFile().mkdirs(); datesFile.getParentFile().mkdirs();
this.datesTable = new kelondroMap(new kelondroDyn(datesFile, bufferkb * 1024, 20, 256, true)); this.datesTable = new kelondroMap(new kelondroDyn(datesFile, bufferkb * 1024, 20, 256, '_', true));
rebuildDates(); rebuildDates();
} }
} }

6
source/de/anomic/data/messageBoard.java
Unescape View File

@ -71,12 +71,12 @@ public class messageBoard {
new File(path.getParent()).mkdir(); new File(path.getParent()).mkdir();
if (database == null) { if (database == null) {
if (path.exists()) try { if (path.exists()) try {
database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400)); database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, '_'));
} catch (IOException e) { } catch (IOException e) {
path.delete(); path.delete();
database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, categoryLength + dateFormat.length() + 2, recordSize, true)); database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, categoryLength + dateFormat.length() + 2, recordSize, '_', true));
} else { } else {
database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, categoryLength + dateFormat.length() + 2, recordSize, true)); database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, categoryLength + dateFormat.length() + 2, recordSize, '_', true));
} }
} }
sn = 0; sn = 0;

10
source/de/anomic/data/userDB.java
Unescape View File

@ -76,19 +76,19 @@ public final class userDB {
this.bufferkb = bufferkb; this.bufferkb = bufferkb;
if (userTableFile.exists()) { if (userTableFile.exists()) {
try { try {
this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024)); this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, '_'));
} catch (kelondroException e) { } catch (kelondroException e) {
userTableFile.delete(); userTableFile.delete();
userTableFile.getParentFile().mkdirs(); userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, 128, 256, true)); this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, 128, 256, '_', true));
} catch (IOException e) { } catch (IOException e) {
userTableFile.delete(); userTableFile.delete();
userTableFile.getParentFile().mkdirs(); userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, 128, 256, true)); this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, 128, 256, '_', true));
} }
} else { } else {
userTableFile.getParentFile().mkdirs(); userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, 128, 256, true)); this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, 128, 256, '_', true));
} }
} }
@ -107,7 +107,7 @@ public final class userDB {
} catch (IOException e) {} } catch (IOException e) {}
if (!(userTableFile.delete())) throw new RuntimeException("cannot delete user database"); if (!(userTableFile.delete())) throw new RuntimeException("cannot delete user database");
userTableFile.getParentFile().mkdirs(); userTableFile.getParentFile().mkdirs();
userTable = new kelondroMap(new kelondroDyn(userTableFile, this.bufferkb, 256, 512, true)); userTable = new kelondroMap(new kelondroDyn(userTableFile, this.bufferkb, 256, 512, '_', true));
} }
public void close() { public void close() {

12
source/de/anomic/data/wikiBoard.java
Unescape View File

@ -73,21 +73,21 @@ public class wikiBoard {
new File(actpath.getParent()).mkdir(); new File(actpath.getParent()).mkdir();
if (datbase == null) { if (datbase == null) {
if (actpath.exists()) try { if (actpath.exists()) try {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x40)); datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x40, '_'));
} catch (IOException e) { } catch (IOException e) {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, keyLength, recordSize, true)); datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, keyLength, recordSize, '_', true));
} else { } else {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, keyLength, recordSize, true)); datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, keyLength, recordSize, '_', true));
} }
} }
new File(bkppath.getParent()).mkdir(); new File(bkppath.getParent()).mkdir();
if (bkpbase == null) { if (bkpbase == null) {
if (bkppath.exists()) try { if (bkppath.exists()) try {
bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400)); bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400, '_'));
} catch (IOException e) { } catch (IOException e) {
bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400, keyLength + dateFormat.length(), recordSize, true)); bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400, keyLength + dateFormat.length(), recordSize, '_', true));
} else { } else {
bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400, keyLength + dateFormat.length(), recordSize, true)); bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400, keyLength + dateFormat.length(), recordSize, '_', true));
} }
} }
} }

21
source/de/anomic/kelondro/kelondroDyn.java
Unescape View File

@ -68,16 +68,18 @@ public class kelondroDyn extends kelondroTree {
private int keylen; private int keylen;
private int reclen; private int reclen;
private int segmentCount; private int segmentCount;
private char fillChar;
public kelondroDyn(File file, long buffersize /*bytes*/, int key, int nodesize, boolean exitOnFail) { public kelondroDyn(File file, long buffersize /*bytes*/, int key, int nodesize, char fillChar, boolean exitOnFail) {
this(file, buffersize, key, nodesize, new kelondroNaturalOrder(true), exitOnFail); this(file, buffersize, key, nodesize, fillChar, new kelondroNaturalOrder(true), exitOnFail);
} }
public kelondroDyn(File file, long buffersize /*bytes*/, int key, int nodesize, kelondroOrder objectOrder, boolean exitOnFail) { public kelondroDyn(File file, long buffersize /*bytes*/, int key, int nodesize, char fillChar, kelondroOrder objectOrder, boolean exitOnFail) {
// creates a new dynamic tree // creates a new dynamic tree
super(file, buffersize, new int[] {key + counterlen, nodesize}, objectOrder, 1, 8, exitOnFail); super(file, buffersize, new int[] {key + counterlen, nodesize}, objectOrder, 1, 8, exitOnFail);
this.keylen = columnSize(0) - counterlen; this.keylen = columnSize(0) - counterlen;
this.reclen = columnSize(1); this.reclen = columnSize(1);
this.fillChar = fillChar;
this.segmentCacheKey = null; this.segmentCacheKey = null;
this.segmentCacheContent = null; this.segmentCacheContent = null;
// init counter: write into text field // init counter: write into text field
@ -85,11 +87,12 @@ public class kelondroDyn extends kelondroTree {
writeSegmentCount(); writeSegmentCount();
} }
public kelondroDyn(File file, long buffersize) throws IOException { public kelondroDyn(File file, long buffersize, char fillChar) throws IOException {
// this opens a file with an existing dynamic tree // this opens a file with an existing dynamic tree
super(file, buffersize); super(file, buffersize);
this.keylen = columnSize(0) - counterlen; this.keylen = columnSize(0) - counterlen;
this.reclen = columnSize(1); this.reclen = columnSize(1);
this.fillChar = fillChar;
this.segmentCacheKey = null; this.segmentCacheKey = null;
this.segmentCacheContent = null; this.segmentCacheContent = null;
this.segmentCount = 0; this.segmentCount = 0;
@ -132,7 +135,7 @@ public class kelondroDyn extends kelondroTree {
private byte[] dynKey(String key, int record) { private byte[] dynKey(String key, int record) {
if (key.length() > keylen) throw new RuntimeException("key len (" + key.length() + ") out of limit (" + keylen + "): '" + key + "'"); if (key.length() > keylen) throw new RuntimeException("key len (" + key.length() + ") out of limit (" + keylen + "): '" + key + "'");
while (key.length() < keylen) key = key + "_"; while (key.length() < keylen) key = key + fillChar;
key = key + counter(record); key = key + counter(record);
return key.getBytes(); return key.getBytes();
} }
@ -140,7 +143,7 @@ public class kelondroDyn extends kelondroTree {
private String origKey(byte[] rawKey) { private String origKey(byte[] rawKey) {
int n = keylen - 1; int n = keylen - 1;
if (n >= rawKey.length) n = rawKey.length - 1; if (n >= rawKey.length) n = rawKey.length - 1;
while ((n > 0) && (rawKey[n] == (byte) '_')) n--; while ((n > 0) && (rawKey[n] == (byte) fillChar)) n--;
return new String(rawKey, 0, n + 1); return new String(rawKey, 0, n + 1);
} }
@ -429,7 +432,7 @@ public class kelondroDyn extends kelondroTree {
} else if (args.length == 1) { } else if (args.length == 1) {
// open a db and list keys // open a db and list keys
try { try {
kelondroDyn kd = new kelondroDyn(new File(args[0]), 0x100000); kelondroDyn kd = new kelondroDyn(new File(args[0]), 0x100000, '_');
System.out.println(kd.size() + " elements in DB"); System.out.println(kd.size() + " elements in DB");
Iterator i = kd.dynKeys(true, false); Iterator i = kd.dynKeys(true, false);
while (i.hasNext()) System.out.println((String) i.next()); while (i.hasNext()) System.out.println((String) i.next());
@ -445,7 +448,7 @@ public class kelondroDyn extends kelondroTree {
File f = new File(args[3]); File f = new File(args[3]);
kelondroDyn kd; kelondroDyn kd;
try { try {
if (db.exists()) kd = new kelondroDyn(db, 0x100000); else kd = new kelondroDyn(db, 0x100000, 80, 200, true); if (db.exists()) kd = new kelondroDyn(db, 0x100000, '_'); else kd = new kelondroDyn(db, 0x100000, 80, 200, '_', true);
if (writeFile) kd.readFile(key, f); else kd.writeFile(key, f); if (writeFile) kd.readFile(key, f); else kd.writeFile(key, f);
} catch (IOException e) { } catch (IOException e) {
System.out.println("ERROR: " + e.toString()); System.out.println("ERROR: " + e.toString());
@ -466,7 +469,7 @@ public class kelondroDyn extends kelondroTree {
int steps = 0; int steps = 0;
while (true) { while (true) {
if (testFile.exists()) testFile.delete(); if (testFile.exists()) testFile.delete();
tt = new kelondroDyn(testFile, 0, 4 ,100, true); tt = new kelondroDyn(testFile, 0, 4 ,100, '_', true);
steps = ((int) System.currentTimeMillis() % 7) * (((int) System.currentTimeMillis() + 17) % 11); steps = ((int) System.currentTimeMillis() % 7) * (((int) System.currentTimeMillis() + 17) % 11);
t = s; t = s;
d = ""; d = "";

12
source/de/anomic/kelondro/kelondroDynTree.java
Unescape View File

@ -69,7 +69,7 @@ public class kelondroDynTree {
private Hashtable buffer, cache; private Hashtable buffer, cache;
private long cycleBuffer; private long cycleBuffer;
public kelondroDynTree(File file, long buffersize, int keylength, int nodesize, int[] columns, boolean exitOnFail) { public kelondroDynTree(File file, long buffersize, int keylength, int nodesize, int[] columns, char fillChar, boolean exitOnFail) {
// creates a new DynTree // creates a new DynTree
this.file = file; this.file = file;
this.columns = columns; this.columns = columns;
@ -78,11 +78,11 @@ public class kelondroDynTree {
//this.cycleCache = Long.MIN_VALUE; //this.cycleCache = Long.MIN_VALUE;
this.cycleBuffer = Long.MIN_VALUE; this.cycleBuffer = Long.MIN_VALUE;
if (file.exists()) file.delete(); if (file.exists()) file.delete();
this.table = new kelondroDyn(file, buffersize, keylength, nodesize, exitOnFail); this.table = new kelondroDyn(file, buffersize, keylength, nodesize, fillChar, exitOnFail);
this.treeRAHandles = new Hashtable(); this.treeRAHandles = new Hashtable();
} }
public kelondroDynTree(File file, long buffersize) throws IOException { public kelondroDynTree(File file, long buffersize, char fillChar) throws IOException {
// opens an existing DynTree // opens an existing DynTree
this.file = file; this.file = file;
this.buffer = new Hashtable(); this.buffer = new Hashtable();
@ -90,7 +90,7 @@ public class kelondroDynTree {
//this.cycleCache = Long.MIN_VALUE; //this.cycleCache = Long.MIN_VALUE;
this.cycleBuffer = Long.MIN_VALUE; this.cycleBuffer = Long.MIN_VALUE;
if (!(file.exists())) throw new IOException("DynTree " + file.toString() + " does not exist"); if (!(file.exists())) throw new IOException("DynTree " + file.toString() + " does not exist");
this.table = new kelondroDyn(file, buffersize); this.table = new kelondroDyn(file, buffersize, fillChar);
// read one element to measure the size of columns // read one element to measure the size of columns
if (table.size() == 0) throw new IOException("DynTree " + file.toString() + " is empty. Should not."); if (table.size() == 0) throw new IOException("DynTree " + file.toString() + " is empty. Should not.");
this.treeRAHandles = new Hashtable(); this.treeRAHandles = new Hashtable();
@ -349,10 +349,10 @@ public class kelondroDynTree {
System.out.println("start"); System.out.println("start");
File file = new File("D:\\bin\\testDyn.db"); File file = new File("D:\\bin\\testDyn.db");
if (file.exists()) { if (file.exists()) {
kelondroDynTree dt = new kelondroDynTree(file, 0x100000L); kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, '_');
System.out.println("opened: table keylength=" + dt.table.columnSize(0) + ", sectorsize=" + dt.table.columnSize(1) + ", " + dt.table.size() + " entries."); System.out.println("opened: table keylength=" + dt.table.columnSize(0) + ", sectorsize=" + dt.table.columnSize(1) + ", " + dt.table.size() + " entries.");
} else { } else {
kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, 16, 512, new int[] {10,20,30}, true); kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, 16, 512, new int[] {10,20,30}, '_', true);
String name; String name;
kelondroTree t; kelondroTree t;
byte[][] line = new byte[][] {"".getBytes(), "abc".getBytes(), "def".getBytes()}; byte[][] line = new byte[][] {"".getBytes(), "abc".getBytes(), "def".getBytes()};

16
source/de/anomic/kelondro/kelondroTables.java
Unescape View File

@ -61,27 +61,27 @@ public class kelondroTables {
if (!(tablesPath.exists())) tablesPath.mkdirs(); if (!(tablesPath.exists())) tablesPath.mkdirs();
} }
public void declareMaps(String tablename, int keysize, int nodesize, boolean exitOnFail) { public void declareMaps(String tablename, int keysize, int nodesize, char fillChar, boolean exitOnFail) {
declareMaps(tablename, keysize, nodesize, null, null, exitOnFail); declareMaps(tablename, keysize, nodesize, null, null, fillChar, exitOnFail);
} }
public void declareMaps(String tablename, int keysize, int nodesize, String[] sortfields, String[] accfields, boolean exitOnFail) { public void declareMaps(String tablename, int keysize, int nodesize, String[] sortfields, String[] accfields, char fillChar, boolean exitOnFail) {
declareMaps(tablename, keysize, nodesize, sortfields, accfields, 0x800, exitOnFail); declareMaps(tablename, keysize, nodesize, sortfields, accfields, fillChar, 0x800, exitOnFail);
} }
public void declareMaps(String tablename, int keysize, int nodesize, String[] sortfields, String[] accfields, long buffersize /*bytes*/, boolean exitOnFail) { public void declareMaps(String tablename, int keysize, int nodesize, String[] sortfields, String[] accfields, char fillChar, long buffersize /*bytes*/, boolean exitOnFail) {
if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared twice."); if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared twice.");
if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared already in other context."); if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared already in other context.");
File tablefile = new File(tablesPath, "table." + tablename + ".mdb"); File tablefile = new File(tablesPath, "table." + tablename + ".mdb");
kelondroDyn dyn; kelondroDyn dyn;
if (tablefile.exists()) try { if (tablefile.exists()) try {
dyn = new kelondroDyn(tablefile, buffersize); dyn = new kelondroDyn(tablefile, buffersize, fillChar);
} catch (IOException e) { } catch (IOException e) {
tablefile.getParentFile().mkdirs(); tablefile.getParentFile().mkdirs();
dyn = new kelondroDyn(tablefile, buffersize, keysize, nodesize, exitOnFail); dyn = new kelondroDyn(tablefile, buffersize, keysize, nodesize, fillChar, exitOnFail);
} else { } else {
tablefile.getParentFile().mkdirs(); tablefile.getParentFile().mkdirs();
dyn = new kelondroDyn(tablefile, buffersize, keysize, nodesize, exitOnFail); dyn = new kelondroDyn(tablefile, buffersize, keysize, nodesize, fillChar, exitOnFail);
} }
kelondroMap map = new kelondroMap(dyn, sortfields, accfields); kelondroMap map = new kelondroMap(dyn, sortfields, accfields);
mTables.put(tablename, map); mTables.put(tablename, map);

8
source/de/anomic/plasma/plasmaCrawlProfile.java
Unescape View File

@ -64,13 +64,13 @@ public class plasmaCrawlProfile {
this.profileTableFile = file; this.profileTableFile = file;
kelondroDyn dyn = null; kelondroDyn dyn = null;
if (profileTableFile.exists()) try { if (profileTableFile.exists()) try {
dyn = new kelondroDyn(file, bufferkb * 1024); dyn = new kelondroDyn(file, bufferkb * 1024, '#');
} catch (IOException e) { } catch (IOException e) {
profileTableFile.delete(); profileTableFile.delete();
dyn = new kelondroDyn(file, bufferkb * 1024, plasmaURL.urlCrawlProfileHandleLength, 2000, true); dyn = new kelondroDyn(file, bufferkb * 1024, plasmaURL.urlCrawlProfileHandleLength, 2000, '#', true);
} else { } else {
profileTableFile.getParentFile().mkdirs(); profileTableFile.getParentFile().mkdirs();
dyn = new kelondroDyn(file, bufferkb * 1024, plasmaURL.urlCrawlProfileHandleLength, 2000, true); dyn = new kelondroDyn(file, bufferkb * 1024, plasmaURL.urlCrawlProfileHandleLength, 2000, '#', true);
} }
profileTable = new kelondroMap(dyn); profileTable = new kelondroMap(dyn);
} }
@ -88,7 +88,7 @@ public class plasmaCrawlProfile {
if (profileTable != null) try { profileTable.close(); } catch (IOException e) {} if (profileTable != null) try { profileTable.close(); } catch (IOException e) {}
if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database"); if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database");
profileTableFile.getParentFile().mkdirs(); profileTableFile.getParentFile().mkdirs();
profileTable = new kelondroMap(new kelondroDyn(profileTableFile, bufferkb * 1024, plasmaURL.urlCrawlProfileHandleLength, 2000, true)); profileTable = new kelondroMap(new kelondroDyn(profileTableFile, bufferkb * 1024, plasmaURL.urlCrawlProfileHandleLength, 2000, '#', true));
} }
public void close() { public void close() {

10
source/de/anomic/plasma/plasmaCrawlRobotsTxt.java
Unescape View File

@ -68,17 +68,17 @@ public class plasmaCrawlRobotsTxt {
this.bufferkb = bufferkb; this.bufferkb = bufferkb;
if (robotsTableFile.exists()) { if (robotsTableFile.exists()) {
try { try {
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024)); robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, '_'));
} catch (kelondroException e) { } catch (kelondroException e) {
robotsTableFile.delete(); robotsTableFile.delete();
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, 256, 512, true)); robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, 256, 512, '_', true));
} catch (IOException e) { } catch (IOException e) {
robotsTableFile.delete(); robotsTableFile.delete();
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, 256, 512, true)); robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, 256, 512, '_', true));
} }
} else { } else {
robotsTableFile.getParentFile().mkdirs(); robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, 256, 512, true)); robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, 256, 512, '_', true));
} }
} }
@ -97,7 +97,7 @@ public class plasmaCrawlRobotsTxt {
} catch (IOException e) {} } catch (IOException e) {}
if (!(robotsTableFile.delete())) throw new RuntimeException("cannot delete robots.txt database"); if (!(robotsTableFile.delete())) throw new RuntimeException("cannot delete robots.txt database");
robotsTableFile.getParentFile().mkdirs(); robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, this.bufferkb, 256, 512, true)); robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, this.bufferkb, 256, 512, '_', true));
} }
public void close() { public void close() {

4
source/de/anomic/plasma/plasmaHTCache.java
Unescape View File

@ -134,9 +134,9 @@ public final class plasmaHTCache {
File dbfile = new File(this.cachePath, "responseHeader.db"); File dbfile = new File(this.cachePath, "responseHeader.db");
try { try {
if (dbfile.exists()) if (dbfile.exists())
this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400)); this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400, '#'));
else else
this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400, plasmaURL.urlHashLength, 150, false)); this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400, plasmaURL.urlHashLength, 150, '#', false));
} catch (IOException e) { } catch (IOException e) {
this.log.logSevere("the request header database could not be opened: " + e.getMessage()); this.log.logSevere("the request header database could not be opened: " + e.getMessage());
System.exit(0); System.exit(0);

8
source/de/anomic/plasma/plasmaWordConnotation.java
Unescape View File

@ -58,13 +58,13 @@ public class plasmaWordConnotation {
private static final int nodesize = 4048; private static final int nodesize = 4048;
private kelondroDynTree refDB; private kelondroDynTree refDB;
public plasmaWordConnotation(File refDBfile, int bufferkb) { public plasmaWordConnotation(File refDBfile, int bufferkb, char fillChar) {
if (refDBfile.exists()) try { if (refDBfile.exists()) try {
refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400); refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, fillChar);
} catch (IOException e) { } catch (IOException e) {
refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, wordlength, nodesize, new int[] {wordlength, countlength}, true); refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, wordlength, nodesize, new int[] {wordlength, countlength}, fillChar, true);
} else { } else {
refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, wordlength, nodesize, new int[] {wordlength, countlength}, true); refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, wordlength, nodesize, new int[] {wordlength, countlength}, fillChar, true);
} }
} }

4
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -187,7 +187,7 @@ public final class yacySeedDB {
private synchronized kelondroMap openSeedTable(File seedDBFile) { private synchronized kelondroMap openSeedTable(File seedDBFile) {
if (seedDBFile.exists()) try { if (seedDBFile.exists()) try {
// open existing seed database // open existing seed database
return new kelondroMap(new kelondroDyn(seedDBFile, (seedDBBufferKB * 0x400) / 3), sortFields, accFields); return new kelondroMap(new kelondroDyn(seedDBFile, (seedDBBufferKB * 0x400) / 3, '#'), sortFields, accFields);
} catch (kelondroException e) { } catch (kelondroException e) {
// if we have an error, we start with a fresh database // if we have an error, we start with a fresh database
if (seedDBFile.exists()) seedDBFile.delete(); if (seedDBFile.exists()) seedDBFile.delete();
@ -197,7 +197,7 @@ public final class yacySeedDB {
} }
// create new seed database // create new seed database
new File(seedDBFile.getParent()).mkdir(); new File(seedDBFile.getParent()).mkdir();
return new kelondroMap(new kelondroDyn(seedDBFile, (seedDBBufferKB * 0x400) / 3, commonHashLength, 480, true), sortFields, accFields); return new kelondroMap(new kelondroDyn(seedDBFile, (seedDBBufferKB * 0x400) / 3, commonHashLength, 480, '#', true), sortFields, accFields);
} }
private synchronized kelondroMap resetSeedTable(kelondroMap seedDB, File seedDBFile) { private synchronized kelondroMap resetSeedTable(kelondroMap seedDB, File seedDBFile) {

2
source/yacy.java
Unescape View File

@ -1396,7 +1396,7 @@ public final class yacy {
String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"}; String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"};
for (int i=0; i < dbFileNames.length; i++) { for (int i=0; i < dbFileNames.length; i++) {
File dbFile = new File(yacyDBPath,dbFileNames[i]); File dbFile = new File(yacyDBPath,dbFileNames[i]);
kelondroMap db = new kelondroMap(new kelondroDyn(dbFile, (1024 * 0x400) / 3), yacySeedDB.sortFields, yacySeedDB.accFields); kelondroMap db = new kelondroMap(new kelondroDyn(dbFile, (1024 * 0x400) / 3, '#'), yacySeedDB.sortFields, yacySeedDB.accFields);
kelondroMap.mapIterator it; kelondroMap.mapIterator it;
it = db.maps(true, false); it = db.maps(true, false);

Write Preview
Loading…
Cancel
Save