boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

* simplified initialization of database objects

Browse Source 
* replaced kelondroTree for NURLs by kelondroFlex
* replaced kelondroTree for EURLs by kelondroFlex
take care, may be very buggy
please finish crawls before updating. crawls will be lost.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2452 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent 0b73f2b132
commit db1eae0227
42 changed files with 395 additions and 644 deletions
Show Stats Download Patch File Download Diff File

5
doc/News.html
Unescape View File

@ -84,7 +84,7 @@ globalheader();
</ul>
-->
<br>v0.46_20060823_2442<p>
<br><p>v0.46_20060823_2442
<ul>
<li>Web Interface Enhancements</li>
<ul>
@ -130,7 +130,8 @@ globalheader();
</ul>
</ul>
<br><p>v0.45_20060501_2046<ul>
<br><p>v0.45_20060501_2046
<ul>
<li>Enhanced Search Functions</li>
<ul>
<li>Re-designed search page: this shows now a simplified page without the left navigation column. The column is only shown if the user is logged-in.</li>

13
source/dbtest.java
Unescape View File

@ -173,23 +173,18 @@ public class dbtest {
kelondroRow testRow = new kelondroRow("byte[] key-" + keylength + ", byte[] dummy-" + keylength + ", value-" + valuelength);
if (dbe.equals("kelondroTree")) {
File tablefile = new File(tablename + ".kelondro.db");
if (tablefile.exists()) {
table = new kelondroTree(tablefile, buffer, preload, kelondroTree.defaultObjectCachePercent);
} else {
table = new kelondroTree(tablefile, buffer, preload, kelondroTree.defaultObjectCachePercent, testRow, true);
}
table = new kelondroTree(tablefile, buffer, preload, kelondroTree.defaultObjectCachePercent, testRow);
}
if (dbe.equals("kelondroSplittedTree")) {
File tablepath = new File(tablename).getParentFile();
tablename = new File(tablename).getName();
table = kelondroSplittedTree.open(tablepath, tablename, kelondroBase64Order.enhancedCoder,
table = new kelondroSplittedTree(tablepath, tablename, kelondroBase64Order.enhancedCoder,
buffer, preload,
8, testRow, 1, 80,
true);
8, testRow, 1, 80);
}
if (dbe.equals("kelondroFlexTable")) {
File tablepath = new File(tablename).getParentFile();
table = new kelondroFlexTable(tablepath, new File(tablename).getName(), kelondroBase64Order.enhancedCoder, buffer, preload, testRow, true);
table = new kelondroFlexTable(tablepath, new File(tablename).getName(), kelondroBase64Order.enhancedCoder, buffer, preload, testRow);
}
if (dbe.equals("mysql")) {
table = new dbTable("mysql", testRow);

8
source/de/anomic/data/blogBoard.java
Unescape View File

@ -83,13 +83,7 @@ public class blogBoard {
public blogBoard(File actpath, int bufferkb, long preloadTime) {
new File(actpath.getParent()).mkdir();
if (datbase == null) {
if (actpath.exists()) try {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x40, preloadTime, '_'));
} catch (IOException e) {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_', true));
} else {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_', true));
}
datbase = new kelondroMap(kelondroDyn.open(actpath, bufferkb / 2 * 0x40, preloadTime, keyLength, recordSize, '_'));
}
}

72
source/de/anomic/data/bookmarksDB.java
Unescape View File

@ -120,67 +120,25 @@ public class bookmarksDB {
}
public bookmarksDB(File bookmarksFile, File tagsFile, File datesFile, int bufferkb, long preloadTime) {
//bookmarks
//check if database exists
// bookmarks
tagCache=new HashMap();
bookmarkCache=new HashMap();
if(bookmarksFile.exists()){
try {
//open it
this.bookmarksTable=new kelondroMap(new kelondroDyn(bookmarksFile, 1024*bufferkb, preloadTime, '_'));
} catch (IOException e) {
//database reset :-((
bookmarksFile.delete();
bookmarksFile.getParentFile().mkdirs();
//urlHash is 12 bytes long
this.bookmarksTable = new kelondroMap(new kelondroDyn(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true));
}
}else{
//new database
bookmarksFile.getParentFile().mkdirs();
this.bookmarksTable = new kelondroMap(new kelondroDyn(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true));
}
//tags
//check if database exists
if(tagsFile.exists()){
try {
//open it
this.tagsTable=new kelondroMap(new kelondroDyn(tagsFile, 1024*bufferkb, preloadTime, '_'));
} catch (IOException e) {
//reset database
tagsFile.delete();
tagsFile.getParentFile().mkdirs();
// max. 128 byte long tags
this.tagsTable = new kelondroMap(new kelondroDyn(tagsFile, bufferkb * 1024, preloadTime, 12, 256, '_', true));
rebuildTags();
}
}else{
//new database
tagsFile.getParentFile().mkdirs();
this.tagsTable = new kelondroMap(new kelondroDyn(tagsFile, bufferkb * 1024, preloadTime, 12, 256, '_', true));
rebuildTags();
}
bookmarksFile.getParentFile().mkdirs();
this.bookmarksTable = new kelondroMap(kelondroDyn.open(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_'));
// tags
tagsFile.getParentFile().mkdirs();
boolean tagsFileExisted = tagsFile.exists();
this.tagsTable = new kelondroMap(kelondroDyn.open(tagsFile, bufferkb * 1024, preloadTime, 12, 256, '_'));
if (!tagsFileExisted) rebuildTags();
// dates
//check if database exists
if(datesFile.exists()){
try {
//open it
this.datesTable=new kelondroMap(new kelondroDyn(datesFile, 1024*bufferkb, preloadTime, '_'));
} catch (IOException e) {
//reset database
datesFile.delete();
datesFile.getParentFile().mkdirs();
//YYYY-MM-DDTHH:mm:ssZ = 20 byte. currently used: YYYY-MM-DD = 10 bytes
this.datesTable = new kelondroMap(new kelondroDyn(datesFile, bufferkb * 1024, preloadTime, 20, 256, '_', true));
rebuildDates();
}
}else{
//new database
datesFile.getParentFile().mkdirs();
this.datesTable = new kelondroMap(new kelondroDyn(datesFile, bufferkb * 1024, preloadTime, 20, 256, '_', true));
rebuildDates();
}
boolean datesExisted = datesFile.exists();
this.datesTable = new kelondroMap(kelondroDyn.open(datesFile, bufferkb * 1024, preloadTime, 20, 256, '_'));
if (!datesExisted) rebuildDates();
}
public void close(){
try {
flushBookmarkCache();

9
source/de/anomic/data/messageBoard.java
Unescape View File

@ -70,14 +70,7 @@ public class messageBoard {
public messageBoard(File path, int bufferkb, long preloadTime) {
new File(path.getParent()).mkdir();
if (database == null) {
if (path.exists()) try {
database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, preloadTime, '_'));
} catch (IOException e) {
path.delete();
database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, preloadTime, categoryLength + dateFormat.length() + 2, recordSize, '_', true));
} else {
database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, preloadTime, categoryLength + dateFormat.length() + 2, recordSize, '_', true));
}
database = new kelondroMap(kelondroDyn.open(path, bufferkb * 0x400, preloadTime, categoryLength + dateFormat.length() + 2, recordSize, '_'));
}
sn = 0;
}

20
source/de/anomic/data/userDB.java
Unescape View File

@ -76,22 +76,8 @@ public final class userDB {
this.userTableFile = userTableFile;
this.bufferkb = bufferkb;
this.preloadTime = preloadTime;
if (userTableFile.exists()) {
try {
this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, preloadTime, '_'));
} catch (kelondroException e) {
userTableFile.delete();
userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, preloadTime, 128, 256, '_', true));
} catch (IOException e) {
userTableFile.delete();
userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, preloadTime, 128, 256, '_', true));
}
} else {
userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, preloadTime, 128, 256, '_', true));
}
userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMap(kelondroDyn.open(userTableFile, bufferkb * 1024, preloadTime, 128, 256, '_'));
}
public int dbCacheNodeChunkSize() {
@ -109,7 +95,7 @@ public final class userDB {
} catch (IOException e) {}
if (!(userTableFile.delete())) throw new RuntimeException("cannot delete user database");
userTableFile.getParentFile().mkdirs();
userTable = new kelondroMap(new kelondroDyn(userTableFile, this.bufferkb, preloadTime, 256, 512, '_', true));
userTable = new kelondroMap(kelondroDyn.open(userTableFile, this.bufferkb, preloadTime, 256, 512, '_'));
}
public void close() {

20
source/de/anomic/data/wikiBoard.java
Unescape View File

@ -70,25 +70,13 @@ public class wikiBoard {
private static HashMap authors = new HashMap();
public wikiBoard(File actpath, File bkppath, int bufferkb, long preloadTime) {
new File(actpath.getParent()).mkdir();
new File(actpath.getParent()).mkdirs();
if (datbase == null) {
if (actpath.exists()) try {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x40, preloadTime, '_'));
} catch (IOException e) {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_', true));
} else {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_', true));
}
datbase = new kelondroMap(kelondroDyn.open(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_'));
}
new File(bkppath.getParent()).mkdir();
new File(bkppath.getParent()).mkdirs();
if (bkpbase == null) {
if (bkppath.exists()) try {
bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400, preloadTime, '_'));
} catch (IOException e) {
bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400, preloadTime, keyLength + dateFormat.length(), recordSize, '_', true));
} else {
bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400, preloadTime, keyLength + dateFormat.length(), recordSize, '_', true));
}
bkpbase = new kelondroMap(kelondroDyn.open(bkppath, bufferkb / 2 * 0x400, preloadTime, keyLength + dateFormat.length(), recordSize, '_'));
}
}

4
source/de/anomic/index/indexContainerOrder.java
Unescape View File

@ -53,4 +53,8 @@ public class indexContainerOrder implements kelondroOrder {
return this.embeddedOrder.compare(a, aoffset, alength, b, boffset, blength);
}
public boolean equals(kelondroOrder otherOrder) {
if (!(otherOrder instanceof indexContainerOrder)) return false;
return this.embeddedOrder.equals(((indexContainerOrder) otherOrder).embeddedOrder);
}
}

4
source/de/anomic/index/indexRAMCacheRI.java
Unescape View File

@ -96,7 +96,7 @@ public final class indexRAMCacheRI extends indexAbstractRI implements indexRI {
File indexDumpFile = new File(databaseRoot, indexArrayFileName);
if (indexDumpFile.exists()) indexDumpFile.delete();
kelondroFixedWidthArray dumpArray = null;
dumpArray = new kelondroFixedWidthArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0, false);
dumpArray = new kelondroFixedWidthArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0);
long startTime = System.currentTimeMillis();
long messageTime = System.currentTimeMillis() + 5000;
long wordsPerSecond = 0, wordcount = 0, urlcount = 0;
@ -175,7 +175,7 @@ public final class indexRAMCacheRI extends indexAbstractRI implements indexRI {
private long restore() throws IOException {
File indexDumpFile = new File(databaseRoot, indexArrayFileName);
if (!(indexDumpFile.exists())) return 0;
kelondroFixedWidthArray dumpArray = new kelondroFixedWidthArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis);
kelondroFixedWidthArray dumpArray = new kelondroFixedWidthArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0);
log.logConfig("restore array dump of index cache, " + dumpArray.size() + " word/URL relations");
long startTime = System.currentTimeMillis();
long messageTime = System.currentTimeMillis() + 5000;

2
source/de/anomic/index/indexURL.java
Unescape View File

@ -451,7 +451,7 @@ public class indexURL {
public long[] cacheObjectStatus() {
if (urlHashCache instanceof kelondroTree) return ((kelondroTree) urlHashCache).cacheObjectStatus();
return null;
return new long[]{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
}
public static final int flagTypeID(String hash) {

8
source/de/anomic/kelondro/kelondroAbstractOrder.java
Unescape View File

@ -83,4 +83,12 @@ public abstract class kelondroAbstractOrder implements kelondroOrder {
public void rotate(byte[] newzero) {
this.zero = newzero;
}
public boolean equals(kelondroOrder otherOrder) {
String thisSig = this.signature();
String otherSig = otherOrder.signature();
if ((thisSig == null) || (otherSig == null)) return false;
return thisSig.equals(otherSig);
}
}

13
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -91,7 +91,7 @@ public class kelondroCollectionIndex {
this.loadfactor = loadfactor;
// create index table
index = new kelondroFlexTable(path, filenameStub + ".index.table", indexOrder, buffersize, preloadTime, indexRow(keyLength), true);
index = new kelondroFlexTable(path, filenameStub + ".index.table", indexOrder, buffersize, preloadTime, indexRow(keyLength));
// save/check property file for this array
File propfile = propertyFile(path, filenameStub, loadfactor, rowdef.objectsize());
@ -119,13 +119,8 @@ public class kelondroCollectionIndex {
"byte[] key-" + index.row().width(0) + "," +
"byte[] collection-" + (kelondroRowCollection.exportOverheadSize + load * this.playloadrow.objectsize())
);
if (f.exists()) {
return new kelondroFixedWidthArray(f, rowdef);
} else if (create) {
return new kelondroFixedWidthArray(f, rowdef, 0, true);
} else {
return null;
}
if ((!(f.exists())) && (!create)) return null;
return new kelondroFixedWidthArray(f, rowdef, 0);
}
private kelondroFixedWidthArray getArray(int partitionNumber, int serialNumber, int chunksize) {
@ -460,7 +455,7 @@ public class kelondroCollectionIndex {
collectionIndex.close();
// printout of index
kelondroFlexTable index = new kelondroFlexTable(path, filenameStub + ".index", kelondroNaturalOrder.naturalOrder, buffersize, preloadTime, indexRow(9), true);
kelondroFlexTable index = new kelondroFlexTable(path, filenameStub + ".index", kelondroNaturalOrder.naturalOrder, buffersize, preloadTime, indexRow(9));
index.print();
index.close();
} catch (IOException e) {

53
source/de/anomic/kelondro/kelondroDyn.java
Unescape View File

@ -68,31 +68,43 @@ public class kelondroDyn extends kelondroTree {
private char fillChar;
private kelondroObjectBuffer buffer;
public kelondroDyn(File file, long buffersize /*bytes*/, long preloadTime, int key, int nodesize, char fillChar, boolean exitOnFail) {
this(file, buffersize, preloadTime, key, nodesize, fillChar, new kelondroNaturalOrder(true), exitOnFail);
public kelondroDyn(File file, long buffersize /*bytes*/, long preloadTime, int key, int nodesize, char fillChar) throws IOException {
this(file, buffersize, preloadTime, key, nodesize, fillChar, new kelondroNaturalOrder(true));
}
public kelondroDyn(File file, long buffersize /* bytes */, long preloadTime, int key,
int nodesize, char fillChar, kelondroOrder objectOrder,
boolean exitOnFail) {
// creates a new dynamic tree
super(file, buffersize, preloadTime, kelondroTree.defaultObjectCachePercent, new kelondroRow("byte[] key-" + (key + counterlen) + ", byte[] node-" + nodesize), objectOrder, 1, 8, exitOnFail);
int nodesize, char fillChar, kelondroOrder objectOrder) throws IOException {
// creates or opens a dynamic tree
super(file, buffersize, preloadTime, kelondroTree.defaultObjectCachePercent, new kelondroRow("byte[] key-" + (key + counterlen) + ", byte[] node-" + nodesize), objectOrder, 1, 8);
this.keylen = row().width(0) - counterlen;
this.reclen = row().width(1);
this.fillChar = fillChar;
this.segmentCount = 0;
writeSegmentCount();
if (!(super.fileExisted)) writeSegmentCount();
buffer = new kelondroObjectBuffer(file.toString());
}
public kelondroDyn(File file, long buffersize, long preloadTime, char fillChar) throws IOException {
// this opens a file with an existing dynamic tree
super(file, buffersize, preloadTime, kelondroTree.defaultObjectCachePercent);
this.keylen = row().width(0) - counterlen;
this.reclen = row().width(1);
this.fillChar = fillChar;
this.segmentCount = 0;
buffer = new kelondroObjectBuffer(file.toString());
public static final kelondroDyn open(File file, long buffersize /* bytes */, long preloadTime, int key, int nodesize, char fillChar) {
return open(file, buffersize, preloadTime, key, nodesize, fillChar, new kelondroNaturalOrder(true));
}
public static final kelondroDyn open(File file, long buffersize /* bytes */, long preloadTime, int key,
int nodesize, char fillChar, kelondroOrder objectOrder) {
// opens new or existing file; in case that any error occur the file is deleted again and it is tried to create the file again
// if that fails, the method returns null
try {
return new kelondroDyn(file, buffersize, preloadTime, key, nodesize, fillChar, objectOrder);
} catch (IOException e) {
file.delete();
try {
return new kelondroDyn(file, buffersize, preloadTime, key, nodesize, fillChar, objectOrder);
} catch (IOException ee) {
log.severe("cannot open or create file " + file.toString());
e.printStackTrace();
ee.printStackTrace();
return null;
}
}
}
private void writeSegmentCount() {
@ -442,7 +454,7 @@ public class kelondroDyn extends kelondroTree {
} else if (args.length == 1) {
// open a db and list keys
try {
kelondroDyn kd = new kelondroDyn(new File(args[0]), 0x100000, 0, '_');
kelondroDyn kd = new kelondroDyn(new File(args[0]), 0x100000, 0, 4 ,100, '_');
System.out.println(kd.size() + " elements in DB");
Iterator i = kd.dynKeys(true, false);
while (i.hasNext())
@ -459,10 +471,7 @@ public class kelondroDyn extends kelondroTree {
File f = new File(args[3]);
kelondroDyn kd;
try {
if (db.exists())
kd = new kelondroDyn(db, 0x100000, 0, '_');
else
kd = new kelondroDyn(db, 0x100000, 0, 80, 200, '_', true);
kd = new kelondroDyn(db, 0x100000, 0, 80, 200, '_');
if (writeFile)
kd.readFile(key, f);
else
@ -486,7 +495,7 @@ public class kelondroDyn extends kelondroTree {
int steps = 0;
while (true) {
if (testFile.exists()) testFile.delete();
tt = new kelondroDyn(testFile, 0, 0, 4 ,100, '_', true);
tt = new kelondroDyn(testFile, 0, 0, 4 ,100, '_');
steps = ((int) System.currentTimeMillis() % 7) * (((int) System.currentTimeMillis() + 17) % 11);
t = s;
d = "";

35
source/de/anomic/kelondro/kelondroDynTree.java
Unescape View File

@ -48,7 +48,6 @@ import java.io.File;
import java.io.IOException;
import java.util.Enumeration;
import java.util.Hashtable;
import java.util.Iterator;
public class kelondroDynTree {
@ -70,8 +69,8 @@ public class kelondroDynTree {
private Hashtable buffer, cache;
private long cycleBuffer;
public kelondroDynTree(File file, long buffersize, long preloadTime, int keylength, int nodesize, kelondroRow rowdef, char fillChar, boolean exitOnFail) {
// creates a new DynTree
public kelondroDynTree(File file, long buffersize, long preloadTime, int keylength, int nodesize, kelondroRow rowdef, char fillChar) throws IOException {
// creates or opens a DynTree
this.file = file;
this.buffersize = buffersize;
this.preloadTime = preloadTime;
@ -80,33 +79,9 @@ public class kelondroDynTree {
this.cache = new Hashtable();
//this.cycleCache = Long.MIN_VALUE;
this.cycleBuffer = Long.MIN_VALUE;
if (file.exists()) file.delete();
this.table = new kelondroDyn(file, buffersize, preloadTime, keylength, nodesize, fillChar, exitOnFail);
this.table = new kelondroDyn(file, buffersize, preloadTime, keylength, nodesize, fillChar);
this.treeRAHandles = new Hashtable();
}
public kelondroDynTree(File file, long buffersize, long preloadTime, char fillChar) throws IOException {
// opens an existing DynTree
this.file = file;
this.buffersize = buffersize;
this.preloadTime = preloadTime;
this.buffer = new Hashtable();
this.cache = new Hashtable();
//this.cycleCache = Long.MIN_VALUE;
this.cycleBuffer = Long.MIN_VALUE;
if (!(file.exists())) throw new IOException("DynTree " + file.toString() + " does not exist");
this.table = new kelondroDyn(file, buffersize, preloadTime, fillChar);
// read one element to measure the size of columns
if (table.size() == 0) throw new IOException("DynTree " + file.toString() + " is empty. Should not.");
this.treeRAHandles = new Hashtable();
Iterator i = table.dynKeys(true, false);
String onekey = (String) i.next();
kelondroTree onetree = getTree(onekey);
kelondroColumn[] columns = new kelondroColumn[onetree.row().columns()];
for (int j = 0; j < columns.length; j++) columns[j] = onetree.row().column(j);
this.rowdef = new kelondroRow(columns);
closeTree(onekey);
}
public void close() throws IOException {
Enumeration e = treeRAHandles.keys();
@ -323,10 +298,10 @@ public class kelondroDynTree {
System.out.println("start");
File file = new File("D:\\bin\\testDyn.db");
if (file.exists()) {
kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, 0, '_');
kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, 0, 16, 512, new kelondroRow("byte[] a-10, byte[] b-20, byte[] c-30"), '_');
System.out.println("opened: table keylength=" + dt.table.row().width(0) + ", sectorsize=" + dt.table.row().width(1) + ", " + dt.table.size() + " entries.");
} else {
kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, 0, 16, 512, new kelondroRow("byte[] a-10, byte[] b-20, byte[] c-30"), '_', true);
kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, 0, 16, 512, new kelondroRow("byte[] a-10, byte[] b-20, byte[] c-30"), '_');
String name;
kelondroTree t;
kelondroRow.Entry line;

36
source/de/anomic/kelondro/kelondroFixedWidthArray.java
Unescape View File

@ -55,27 +55,19 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
private static short thisOHBytes = 0; // our record definition does not need extra bytes
private static short thisOHHandles = 0; // and no handles
public kelondroFixedWidthArray(File file, kelondroRow rowdef, int intprops, boolean exitOnFail) {
public kelondroFixedWidthArray(File file, kelondroRow rowdef, int intprops) throws IOException {
// this creates a new array
super(file, 0, 0, thisOHBytes, thisOHHandles, rowdef, intprops, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
for (int i = 0; i < intprops; i++) try {
setHandle(i, new Handle(0));
} catch (IOException e) {
super.logFailure("cannot set handle " + i + " / " + e.getMessage());
if (exitOnFail) System.exit(-1);
throw new RuntimeException("cannot set handle " + i + " / " + e.getMessage());
}
// store column description
for (int i = 0; i < rowdef.columns(); i++) {
try {super.setText(i, rowdef.column(i).toString().getBytes());} catch (IOException e) {}
super(file, 0, 0, thisOHBytes, thisOHHandles, rowdef, intprops, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */);
if (!(super.fileExisted)) {
for (int i = 0; i < intprops; i++) {
setHandle(i, new Handle(0));
}
// store column description
for (int i = 0; i < rowdef.columns(); i++) {
try {super.setText(i, rowdef.column(i).toString().getBytes());} catch (IOException e) {}
}
}
}
public kelondroFixedWidthArray(File file, kelondroRow rowdef) throws IOException{
// this opens a file with an existing array
super(file, 0, 0);
super.assignRowdef(rowdef);
}
public synchronized kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException {
@ -142,14 +134,14 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
try {
System.out.println("erster Test");
f.delete();
kelondroFixedWidthArray k = new kelondroFixedWidthArray(f, rowdef, 6, true);
kelondroFixedWidthArray k = new kelondroFixedWidthArray(f, rowdef, 6);
k.set(3, k.row().newEntry(new byte[][]{
"test123".getBytes(), "abcd".getBytes()}));
k.add(k.row().newEntry(new byte[][]{
"test456".getBytes(), "efgh".getBytes()}));
k.close();
k = new kelondroFixedWidthArray(f, rowdef);
k = new kelondroFixedWidthArray(f, rowdef, 6);
System.out.println(k.get(2).toString());
System.out.println(k.get(3).toString());
System.out.println(k.get(4).toString());
@ -157,7 +149,7 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
System.out.println("zweiter Test");
f.delete();
k = new kelondroFixedWidthArray(f, rowdef, 6, true);
k = new kelondroFixedWidthArray(f, rowdef, 6);
k.add(k.row().newEntry(new byte[][]{"a".getBytes(), "xxxx".getBytes()}));
k.add(k.row().newEntry(new byte[][]{"b".getBytes(), "xxxx".getBytes()}));
k.remove(0);
@ -176,7 +168,7 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
System.out.println("dritter Test");
f.delete();
k = new kelondroFixedWidthArray(f, rowdef, 6, true);
k = new kelondroFixedWidthArray(f, rowdef, 6);
for (int i = 1; i <= 200; i = i * 2) {
for (int j = 0; j < i*2; j++) {
k.add(k.row().newEntry(new byte[][]{(Integer.toString(i) + "-" + Integer.toString(j)).getBytes(), "xxxx".getBytes()}));

14
source/de/anomic/kelondro/kelondroFlexTable.java
Unescape View File

@ -33,8 +33,8 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
protected kelondroBytesIntMap index;
public kelondroFlexTable(File path, String tablename, kelondroOrder objectOrder, long buffersize, long preloadTime, kelondroRow rowdef, boolean exitOnFail) throws IOException {
super(path, tablename, rowdef, exitOnFail);
public kelondroFlexTable(File path, String tablename, kelondroOrder objectOrder, long buffersize, long preloadTime, kelondroRow rowdef) throws IOException {
super(path, tablename, rowdef);
File newpath = new File(path, tablename);
File indexfile = new File(newpath, "col.000.index");
kelondroIndex ki = null;
@ -52,7 +52,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
if (indexfile.exists()) {
// use existing index file
System.out.println("*** Using File index " + indexfile);
ki = new kelondroTree(indexfile, buffersize, preloadTime, 10);
ki = kelondroTree.open(indexfile, buffersize, preloadTime, 10, treeIndexRow(rowdef.width(0)), objectOrder, 2, 80);
} else if ((preloadTime >= 0) && (stt > preloadTime)) {
// generate new index file
System.out.print("*** Generating File index for " + size() + " entries from " + indexfile);
@ -104,9 +104,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
}
private kelondroTree initializeTreeIndex(File indexfile, long buffersize, long preloadTime, kelondroOrder objectOrder) throws IOException {
kelondroTree treeindex = new kelondroTree(indexfile, buffersize, preloadTime, 10,
new kelondroRow("byte[] key-" + rowdef.width(0) + ", int reference-4 {b256}"),
objectOrder, 2, 80, true);
kelondroTree treeindex = new kelondroTree(indexfile, buffersize, preloadTime, 10, treeIndexRow(rowdef.width(0)), objectOrder, 2, 80);
Iterator content = super.col[0].contentNodes(-1);
kelondroRecords.Node node;
kelondroRow.Entry indexentry;
@ -126,6 +124,10 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
return treeindex;
}
private static final kelondroRow treeIndexRow(int keywidth) {
return new kelondroRow("byte[] key-" + keywidth + ", int reference-4 {b256}");
}
public synchronized kelondroRow.Entry get(byte[] key) throws IOException {
synchronized (index) {
int i = index.geti(key);

10
source/de/anomic/kelondro/kelondroFlexWidthArray.java
Unescape View File

@ -37,7 +37,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
protected kelondroFixedWidthArray[] col;
protected kelondroRow rowdef;
public kelondroFlexWidthArray(File path, String tablename, kelondroRow rowdef, boolean exitOnFail) throws IOException {
public kelondroFlexWidthArray(File path, String tablename, kelondroRow rowdef) throws IOException {
this.rowdef = rowdef;
// initialize columns
@ -81,7 +81,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
kelondroColumn columns[] = new kelondroColumn[colend - colstart + 1];
for (int j = colstart; j <= colend; j++) columns[j-colstart] = rowdef.column(j);
col[colstart] = new kelondroFixedWidthArray(new File(tabledir, files[i]), new kelondroRow(columns));
col[colstart] = new kelondroFixedWidthArray(new File(tabledir, files[i]), new kelondroRow(columns), 16);
for (int j = colstart; j <= colend; j++) check = check.substring(0, j) + "X" + check.substring(j + 1);
}
}
@ -100,7 +100,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
columns[j - p] = rowdef.column(j);
check = check.substring(0, j) + "X" + check.substring(j + 1);
}
col[p] = new kelondroFixedWidthArray(new File(tabledir, colfilename(p, q)), new kelondroRow(columns), 16, true);
col[p] = new kelondroFixedWidthArray(new File(tabledir, colfilename(p, q)), new kelondroRow(columns), 16);
}
}
@ -224,7 +224,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
System.out.println("erster Test");
new File(f, testname).delete();
kelondroFlexWidthArray k = new kelondroFlexWidthArray(f, "flextest", rowdef, true);
kelondroFlexWidthArray k = new kelondroFlexWidthArray(f, "flextest", rowdef);
k.add(k.row().newEntry(new byte[][]{"a".getBytes(), "xxxx".getBytes()}));
k.add(k.row().newEntry(new byte[][]{"b".getBytes(), "xxxx".getBytes()}));
k.remove(0);
@ -244,7 +244,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
System.out.println("zweiter Test");
new File(f, testname).delete();
k = new kelondroFlexWidthArray(f, "flextest", rowdef, true);
k = new kelondroFlexWidthArray(f, "flextest", rowdef);
for (int i = 1; i <= 20; i = i * 2) {
for (int j = 0; j < i*2; j++) {
k.add(k.row().newEntry(new byte[][]{(Integer.toString(i) + "-" + Integer.toString(j)).getBytes(), "xxxx".getBytes()}));

37
source/de/anomic/kelondro/kelondroHashtable.java
Unescape View File

@ -142,7 +142,7 @@ public class kelondroHashtable {
private static final byte[] dummyKey = kelondroBase64Order.enhancedCoder.encodeLong(0, 5).getBytes();
public kelondroHashtable(File file, kelondroRow rowdef, int offset, int maxsize, int maxrehash, boolean exitOnFail) {
public kelondroHashtable(File file, kelondroRow rowdef, int offset, int maxsize, int maxrehash) throws IOException {
// this creates a new hashtable
// the key element is not part of the columns array
// this is unlike the kelondroTree, where the key is part of a row
@ -152,32 +152,25 @@ public class kelondroHashtable {
// this number is needed to omit grow of the table in case of re-hashing
// the maxsize is re-computed to a virtual folding height and will result in a tablesize
// less than the given maxsize. The actual maxsize can be retrieved by maxsize()
this.hashArray = new kelondroFixedWidthArray(file, extCol(rowdef), 6, exitOnFail);
this.offset = offset;
this.maxk = kelondroMSetTools.log2a(maxsize); // equal to |log2(maxsize)| + 1
if (this.maxk >= kelondroMSetTools.log2a(maxsize + power2(offset + 1) + 1) - 1) this.maxk--;
this.maxrehash = maxrehash;
dummyRow = this.hashArray.row().newEntry();
dummyRow.setCol(0, dummyKey);
for (int i = 0; i < hashArray.row().columns(); i++)
try {
boolean fileExisted = file.exists();
this.hashArray = new kelondroFixedWidthArray(file, extCol(rowdef), 6);
if (fileExisted) {
this.offset = hashArray.geti(0);
this.maxk = hashArray.geti(1);
this.maxrehash = hashArray.geti(2);
} else {
this.offset = offset;
this.maxk = kelondroMSetTools.log2a(maxsize); // equal to |log2(maxsize)| + 1
if (this.maxk >= kelondroMSetTools.log2a(maxsize + power2(offset + 1) + 1) - 1) this.maxk--;
this.maxrehash = maxrehash;
dummyRow = this.hashArray.row().newEntry();
dummyRow.setCol(0, dummyKey);
//for (int i = 0; i < hashArray.row().columns(); i++)
hashArray.seti(0, this.offset);
hashArray.seti(1, this.maxk);
hashArray.seti(2, this.maxrehash);
} catch (IOException e) {
hashArray.logFailure("cannot set properties / " + e.getMessage());
if (exitOnFail) System.exit(-1);
throw new RuntimeException("cannot set properties / " + e.getMessage());
}
}
public kelondroHashtable(File file, kelondroRow rowdef) throws IOException{
// this opens a file with an existing hashtable
this.hashArray = new kelondroFixedWidthArray(file, rowdef);
this.offset = hashArray.geti(0);
this.maxk = hashArray.geti(1);
this.maxrehash = hashArray.geti(2);
}
private kelondroRow extCol(kelondroRow rowdef) {
kelondroColumn[] newCol = new kelondroColumn[rowdef.columns() + 1];

34
source/de/anomic/kelondro/kelondroMapTable.java
Unescape View File

@ -63,51 +63,35 @@ public class kelondroMapTable {
public void declareMaps(
String tablename, int keysize, int nodesize,
char fillChar, boolean exitOnFail) {
declareMaps(tablename, keysize, nodesize, null, null, fillChar, exitOnFail);
char fillChar) throws IOException {
declareMaps(tablename, keysize, nodesize, null, null, fillChar);
}
public void declareMaps(
String tablename, int keysize, int nodesize,
String[] sortfields, String[] accfields, char fillChar, boolean exitOnFail) {
declareMaps(tablename, keysize, nodesize, sortfields, accfields, fillChar, 0x800, 0, exitOnFail);
String[] sortfields, String[] accfields, char fillChar) throws IOException {
declareMaps(tablename, keysize, nodesize, sortfields, accfields, fillChar, 0x800, 0);
}
public void declareMaps(
String tablename, int keysize, int nodesize,
String[] sortfields, String[] accfields, char fillChar,
long buffersize /*bytes*/, long preloadTime, boolean exitOnFail) {
long buffersize /*bytes*/, long preloadTime) throws IOException {
if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared twice.");
if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared already in other context.");
File tablefile = new File(tablesPath, "table." + tablename + ".mdb");
kelondroDyn dyn;
if (tablefile.exists()) try {
dyn = new kelondroDyn(tablefile, buffersize, preloadTime, fillChar);
} catch (IOException e) {
tablefile.getParentFile().mkdirs();
dyn = new kelondroDyn(tablefile, buffersize, preloadTime, keysize, nodesize, fillChar, exitOnFail);
} else {
tablefile.getParentFile().mkdirs();
dyn = new kelondroDyn(tablefile, buffersize, preloadTime, keysize, nodesize, fillChar, exitOnFail);
}
if (!(tablefile.exists())) tablefile.getParentFile().mkdirs();
dyn = new kelondroDyn(tablefile, buffersize, preloadTime, keysize, nodesize, fillChar);
kelondroMap map = new kelondroMap(dyn, sortfields, accfields);
mTables.put(tablename, map);
}
public void declareTree(String tablename, kelondroRow rowdef, long buffersize /*bytes*/, long preloadTime, boolean exitOnFail) {
public void declareTree(String tablename, kelondroRow rowdef, long buffersize /*bytes*/, long preloadTime) {
if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared already in other context.");
if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared twice.");
File tablefile = new File(tablesPath, "table." + tablename + ".tdb");
kelondroTree Tree;
if (tablefile.exists()) try {
Tree = new kelondroTree(tablefile, buffersize, preloadTime, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) {
tablefile.getParentFile().mkdirs();
Tree = new kelondroTree(tablefile, buffersize, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, exitOnFail);
} else {
tablefile.getParentFile().mkdirs();
Tree = new kelondroTree(tablefile, buffersize, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, exitOnFail);
}
kelondroTree Tree = kelondroTree.open(tablefile, buffersize, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef);
tTables.put(tablename, Tree);
}

2
source/de/anomic/kelondro/kelondroOrder.java
Unescape View File

@ -66,4 +66,6 @@ public interface kelondroOrder extends Comparator {
public byte[] zero(); // returns the zero point of the Ordering; null if not defined
public void rotate(byte[] zero); // defines that the ordering rotates, and sets the zero point for the rotation
public boolean equals(kelondroOrder o); // used to compare different order objects; they may define the same ordering
}

43
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -150,6 +150,9 @@ public class kelondroRecords {
// optional logger
protected Logger theLogger = null;
// tracking of file cration
protected boolean fileExisted;
// Random. This is used to shift flush-times of write-buffers to differrent time
private static Random random = new Random(System.currentTimeMillis());
@ -191,30 +194,34 @@ public class kelondroRecords {
public kelondroRecords(File file, long buffersize /* bytes */, long preloadTime,
short ohbytec, short ohhandlec,
kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth,
boolean exitOnFail) {
// creates a new file
kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth) throws IOException {
// opens an existing file or creates a new file
// file: the file that shall be created
// oha : overhead size array of four bytes: oha[0]=# of bytes, oha[1]=# of shorts, oha[2]=# of ints, oha[3]=# of longs,
// columns: array with size of column width; columns.length is number of columns
// FHandles: number of integer properties
// txtProps: number of text properties
assert (!file.exists()) : "file " + file + " already exist";
try {
this.fileExisted = file.exists(); // can be used by extending class to track if this class created the file
if (file.exists()) {
// opens an existing tree
this.filename = file.getCanonicalPath();
kelondroRA raf = new kelondroFileRA(this.filename);
//kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100);
//kelondroRA raf = new kelondroCachedRA(new kelondroFileRA(this.filename), 5000000, 1000);
//kelondroRA raf = new kelondroNIOFileRA(this.filename, (file.length() < 4000000), 10000);
initExistingFile(raf, buffersize / 10);
} else {
this.filename = file.getCanonicalPath();
kelondroRA raf = new kelondroFileRA(this.filename);
// kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100);
// kelondroRA raf = new kelondroNIOFileRA(this.filename, false, 10000);
initNewFile(raf, ohbytec, ohhandlec, rowdef, FHandles, txtProps, txtPropWidth, buffersize / 10);
} catch (IOException e) {
logFailure("cannot create / " + e.getMessage());
if (exitOnFail)
System.exit(-1);
}
initCache(buffersize / 10 * 9, preloadTime);
assignRowdef(rowdef);
initCache(buffersize / 10 * 9, preloadTime);
}
public kelondroRecords(kelondroRA ra, long buffersize /* bytes */, long preloadTime,
short ohbytec, short ohhandlec,
kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth,
@ -347,18 +354,6 @@ public class kelondroRecords {
this.USAGE.write();
}
public kelondroRecords(File file, long buffersize, long preloadTime) throws IOException{
// opens an existing tree
assert (file.exists()): "file " + file.getAbsoluteFile().toString() + " does not exist";
this.filename = file.getCanonicalPath();
kelondroRA raf = new kelondroFileRA(this.filename);
//kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100);
//kelondroRA raf = new kelondroCachedRA(new kelondroFileRA(this.filename), 5000000, 1000);
//kelondroRA raf = new kelondroNIOFileRA(this.filename, (file.length() < 4000000), 10000);
initExistingFile(raf, buffersize / 10);
initCache(buffersize / 10 * 9, preloadTime);
}
public kelondroRecords(kelondroRA ra, long buffersize, long preloadTime) throws IOException{
this.filename = null;
initExistingFile(ra, buffersize / 10);
@ -912,7 +907,7 @@ public class kelondroRecords {
return this.ROW;
}
public final void assignRowdef(kelondroRow rowdef) {
private final void assignRowdef(kelondroRow rowdef) {
// overwrites a given rowdef
// the new rowdef must be compatible
if (rowdef.columns() < ROW.columns())

49
source/de/anomic/kelondro/kelondroSplittedTree.java
Unescape View File

@ -65,64 +65,27 @@ public class kelondroSplittedTree implements kelondroIndex {
return new File(path, filenameStub + "." + ff + "." + co + "." + ns + ".ktc");
}
/*
private static boolean existsAll(File pathToFiles, String filenameStub, int forkfactor, int columns){
for (int i = 0; i < forkfactor; i++) {
if (!(dbFile(pathToFiles, filenameStub, forkfactor, columns, i)).exists()) return false;
}
return true;
}
*/
public kelondroSplittedTree(File pathToFiles, String filenameStub, kelondroOrder objectOrder,
long buffersize, long preloadTime,
int forkfactor, kelondroRow rowdef, int txtProps, int txtPropsWidth,
boolean exitOnFail) {
int forkfactor, kelondroRow rowdef, int txtProps, int txtPropsWidth) {
ktfs = new kelondroTree[forkfactor];
File f;
for (int i = 0; i < forkfactor; i++) {
f = dbFile(pathToFiles, filenameStub, forkfactor, rowdef.columns(), i);
if (f.exists()) {
try {
ktfs[i] = new kelondroTree(f, buffersize/forkfactor, preloadTime / forkfactor, kelondroTree.defaultObjectCachePercent);
this.order = ktfs[i].order();
} catch (IOException e) {
ktfs[i] = new kelondroTree(f, buffersize/forkfactor, preloadTime / forkfactor, kelondroTree.defaultObjectCachePercent,
rowdef, objectOrder, txtProps, txtPropsWidth, exitOnFail);
this.order = objectOrder;
}
} else {
ktfs[i] = new kelondroTree(f, buffersize/forkfactor, preloadTime / forkfactor, kelondroTree.defaultObjectCachePercent,
rowdef, objectOrder, txtProps, txtPropsWidth, exitOnFail);
this.order = objectOrder;
}
ktfs[i] = kelondroTree.open(f, buffersize/forkfactor, preloadTime / forkfactor, kelondroTree.defaultObjectCachePercent,
rowdef, objectOrder, txtProps, txtPropsWidth);
}
ff = forkfactor;
}
public kelondroSplittedTree(File pathToFiles, String filenameStub, kelondroOrder objectOrder,
long buffersize, long preloadTime, int forkfactor, int columns) throws IOException {
ktfs = new kelondroTree[forkfactor];
for (int i = 0; i < forkfactor; i++) {
ktfs[i] = new kelondroTree(dbFile(pathToFiles, filenameStub, forkfactor, columns, i),
buffersize/forkfactor, preloadTime / forkfactor, kelondroTree.defaultObjectCachePercent);
}
ff = forkfactor;
this.order = objectOrder;
}
public static kelondroSplittedTree open(File pathToFiles, String filenameStub, kelondroOrder objectOrder,
long buffersize, long preloadTime,
int forkfactor, kelondroRow rowdef, int txtProps, int txtPropsWidth,
boolean exitOnFail) throws IOException {
// generated a new splittet tree if it not exists or
// opens an existing one
if (existsAll(pathToFiles, filenameStub, forkfactor, rowdef.columns())) {
return new kelondroSplittedTree(pathToFiles, filenameStub, objectOrder, buffersize, preloadTime, forkfactor, rowdef.columns());
} else {
return new kelondroSplittedTree(pathToFiles, filenameStub, objectOrder,
buffersize, preloadTime,
forkfactor, rowdef, txtProps, txtPropsWidth,
exitOnFail);
}
ff = forkfactor;
}
public void close() throws IOException {

60
source/de/anomic/kelondro/kelondroStack.java
Unescape View File

@ -67,31 +67,33 @@ public final class kelondroStack extends kelondroRecords {
protected static final int root = 0; // pointer for FHandles-array: pointer to root node
protected static final int toor = 1; // pointer for FHandles-array: pointer to root node
public kelondroStack(File file, kelondroRow rowdef, boolean exitOnFail) {
public kelondroStack(File file, kelondroRow rowdef) throws IOException {
// this creates a new stack
super(file, 0, 0, thisOHBytes, thisOHHandles, rowdef, thisFHandles, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
try {
super(file, 0, 0, thisOHBytes, thisOHHandles, rowdef, thisFHandles, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */);
if (super.fileExisted) {
if ((getHandle(root) == null) && (getHandle(toor) == null)) clear();
} else {
setHandle(root, null); // define the root value
setHandle(toor, null); // define the toor value
} catch (IOException e) {
super.logFailure("cannot set root/toor handles / " + e.getMessage());
if (exitOnFail) System.exit(-1);
throw new RuntimeException("cannot set root/toor handles / " + e.getMessage());
}
}
public kelondroStack(File file) throws IOException{
// this opens a file with an existing stack
super(file, 0, 0);
if ((getHandle(root) == null) && (getHandle(toor) == null)) clear();
}
public void clear() throws IOException {
super.clear();
setHandle(root, null); // reset the root value
setHandle(toor, null); // reset the toor value
public static final kelondroStack open(File file, kelondroRow rowdef) {
try {
return new kelondroStack(file, rowdef);
} catch (IOException e) {
file.delete();
try {
return new kelondroStack(file, rowdef);
} catch (IOException ee) {
System.out.println("kelondroStack: cannot open or create file " + file.toString());
e.printStackTrace();
ee.printStackTrace();
return null;
}
}
}
public static kelondroStack reset(kelondroStack stack) {
// memorize settings to this file
File f = new File(stack.filename);
@ -102,9 +104,15 @@ public final class kelondroStack extends kelondroRecords {
if (f.exists()) f.delete();
// re-open a database with same settings as before
return new kelondroStack(f, row, true);
return open(f, row);
}
public void clear() throws IOException {
super.clear();
setHandle(root, null); // reset the root value
setHandle(toor, null); // reset the toor value
}
public class Counter implements Iterator {
Handle nextHandle = null;
@ -356,18 +364,19 @@ public final class kelondroStack extends kelondroRecords {
for (int i = 0; i < args.length; i++) System.out.print(args[i] + " ");
System.out.println("");
byte[] ret = null;
kelondroRow lens = new kelondroRow("byte[] key-" + Integer.parseInt(args[1]) + ", byte[] value-" + Integer.parseInt(args[2]));
try {
if ((args.length > 4) || (args.length < 2)) {
System.err.println("usage: kelondroStack -c|-p|-v|-g|-i|-s [file]|[key [value]] <db-file>");
System.err.println("( create, push, view, (g)pop, imp, shell)");
System.exit(0);
} else if (args.length == 2) {
kelondroStack fm = new kelondroStack(new File(args[1]));
kelondroStack fm = new kelondroStack(new File(args[1]), lens);
if (args[0].equals("-v")) {
fm.print();
ret = null;
} else if (args[0].equals("-g")) {
fm = new kelondroStack(new File(args[1]));
fm = new kelondroStack(new File(args[1]), lens);
kelondroRow.Entry ret2 = fm.pop();
ret = ((ret2 == null) ? null : ret2.getColBytes(1));
fm.close();
@ -375,7 +384,7 @@ public final class kelondroStack extends kelondroRecords {
fm.close();
} else if (args.length == 3) {
if (args[0].equals("-i")) {
kelondroStack fm = new kelondroStack(new File(args[2]));
kelondroStack fm = new kelondroStack(new File(args[2]), lens);
int i = fm.imp(new File(args[1]),";");
fm.close();
ret = (i + " records imported").getBytes();
@ -399,7 +408,7 @@ public final class kelondroStack extends kelondroRecords {
if (f != null) try {f.close();}catch(Exception e) {}
}
} else if (args[0].equals("-g")) {
kelondroStack fm = new kelondroStack(new File(args[2]));
kelondroStack fm = new kelondroStack(new File(args[2]), lens);
kelondroRow.Entry ret2 = fm.pop(Integer.parseInt(args[1]));
ret = ((ret2 == null) ? null : ret2.getColBytes(1));
fm.close();
@ -409,11 +418,10 @@ public final class kelondroStack extends kelondroRecords {
// create <keylen> <valuelen> <filename>
File f = new File(args[3]);
if (f.exists()) f.delete();
kelondroRow lens = new kelondroRow("byte[] key-" + Integer.parseInt(args[1]) + ", byte[] value-" + Integer.parseInt(args[2]));
kelondroStack fm = new kelondroStack(f, lens, true);
kelondroStack fm = new kelondroStack(f, lens);
fm.close();
} else if (args[0].equals("-p")) {
kelondroStack fm = new kelondroStack(new File(args[3]));
kelondroStack fm = new kelondroStack(new File(args[3]), lens);
fm.push(fm.row().newEntry(new byte[][] {args[1].getBytes(), args[2].getBytes()}));
fm.close();
}

91
source/de/anomic/kelondro/kelondroTree.java
Unescape View File

@ -91,30 +91,55 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
protected long lastIteratorCount = readAheadChunkSize;
private kelondroObjectCache objectCache;
public kelondroTree(File file, long buffersize, long preloadTime, int objectCachePercent, kelondroRow rowdef, boolean exitOnFail) {
// this creates a new tree file
this(file, buffersize, preloadTime, objectCachePercent, rowdef, new kelondroNaturalOrder(true), rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
public kelondroTree(File file, long buffersize, long preloadTime, int objectCachePercent, kelondroRow rowdef) throws IOException {
this(file, buffersize, preloadTime, objectCachePercent, rowdef, new kelondroNaturalOrder(true), rowdef.columns() /* txtProps */, 80 /* txtPropWidth */);
}
public kelondroTree(File file, long buffersize, long preloadTime, int objectCachePercent, kelondroRow rowdef, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) {
// this creates a new tree file
public kelondroTree(File file, long buffersize, long preloadTime, int objectCachePercent, kelondroRow rowdef,
kelondroOrder objectOrder, int txtProps, int txtPropsWidth) throws IOException {
// opens an existing tree file or creates a new tree file
super(file,
(100 - objectCachePercent) * buffersize / 100, preloadTime,
thisOHBytes, thisOHHandles, rowdef,
thisFHandles, txtProps, txtPropsWidth, exitOnFail);
try {
thisFHandles, txtProps, txtPropsWidth);
super.setLogger(log);
if (super.fileExisted) {
this.objectOrder = objectOrder;
kelondroOrder oldOrder = readOrderType();
if ((oldOrder != null) && (!(oldOrder.equals(objectOrder)))) throw new IOException("wrong object order upon initialization. new order is " + objectOrder.toString() + ", old order was " + oldOrder.toString());
} else {
// create new file structure
setHandle(root, null); // define the root value
} catch (IOException e) {
super.logFailure("cannot set root handle / " + e.getMessage());
if (exitOnFail) System.exit(-1);
throw new RuntimeException("cannot set root handle / " + e.getMessage());
this.objectOrder = objectOrder;
writeOrderType();
}
this.objectOrder = objectOrder;
writeOrderType();
super.setLogger(log);
initObjectCache(buffersize, objectCachePercent);
}
public static final kelondroTree open(File file, long buffersize, long preloadTime, int objectCachePercent, kelondroRow rowdef) {
return open(file, buffersize, preloadTime, objectCachePercent, rowdef, new kelondroNaturalOrder(true), rowdef.columns() /* txtProps */, 80 /* txtPropWidth */);
}
public static final kelondroTree open(File file, long buffersize, long preloadTime, int objectCachePercent, kelondroRow rowdef,
kelondroOrder objectOrder, int txtProps, int txtPropsWidth) {
// opens new or existing file; in case that any error occur the file is deleted again and it is tried to create the file again
// if that fails, the method returns null
try {
return new kelondroTree(file, buffersize, preloadTime, objectCachePercent, rowdef, objectOrder, txtProps, txtPropsWidth);
} catch (IOException e) {
file.delete();
try {
return new kelondroTree(file, buffersize, preloadTime, objectCachePercent, rowdef, objectOrder, txtProps, txtPropsWidth);
} catch (IOException ee) {
log.severe("cannot open or create file " + file.toString());
e.printStackTrace();
ee.printStackTrace();
return null;
}
}
}
public kelondroTree(kelondroRA ra, long buffersize, long preloadTime, int objectCachePercent, kelondroRow rowdef, boolean exitOnFail) {
// this creates a new tree within a kelondroRA
this(ra, buffersize, preloadTime, objectCachePercent, rowdef, new kelondroNaturalOrder(true), rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
@ -139,14 +164,6 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
initObjectCache(buffersize, objectCachePercent);
}
public kelondroTree(File file, long buffersize, long preloadTime, int objectCachePercent) throws IOException {
// this opens a file with an existing tree file
super(file, (100 - objectCachePercent) * buffersize / 100, preloadTime);
readOrderType();
super.setLogger(log);
initObjectCache(buffersize, objectCachePercent);
}
public kelondroTree(kelondroRA ra, long buffersize, long preloadTime, int objectCachePercent) throws IOException {
// this opens a file with an existing tree in a kelondroRA
super(ra, (100 - objectCachePercent) * buffersize / 100, preloadTime);
@ -183,12 +200,14 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
} catch (IOException e) {}
}
private void readOrderType() {
private kelondroOrder readOrderType() {
try {
byte[] d = super.getDescription();
String s = new String(d).substring(0, 2);
this.objectOrder = orderBySignature(s);
} catch (IOException e) {}
return orderBySignature(s);
} catch (IOException e) {
return null;
}
}
public static kelondroOrder orderBySignature(String signature) {
@ -1238,7 +1257,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
// test script
File testFile = new File("test.db");
while (testFile.exists()) testFile.delete();
kelondroTree fm = new kelondroTree(testFile, 0x100000, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"), true);
kelondroTree fm = new kelondroTree(testFile, 0x100000, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"));
byte[] dummy = "".getBytes();
fm.put("abc0".getBytes(), dummy); fm.put("bcd0".getBytes(), dummy);
fm.put("def0".getBytes(), dummy); fm.put("bab0".getBytes(), dummy);
@ -1260,7 +1279,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
ret = null;
}
} else if (args.length == 2) {
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10);
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"));
if (args[0].equals("-v")) {
fm.print();
ret = null;
@ -1268,11 +1287,11 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
fm.close();
} else if (args.length == 3) {
if (args[0].equals("-d")) {
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10);
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"));
fm.remove(args[2].getBytes());
fm.close();
} else if (args[0].equals("-i")) {
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10);
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"));
int i = fm.imp(new File(args[1]),";");
fm.close();
ret = (i + " records imported").getBytes();
@ -1295,12 +1314,12 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
if (f != null) try {f.close();}catch(Exception e){}
}
} else if (args[0].equals("-g")) {
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10);
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"));
kelondroRow.Entry ret2 = fm.get(args[2].getBytes());
ret = ((ret2 == null) ? null : ret2.getColBytes(1));
fm.close();
} else if (args[0].equals("-n")) {
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10);
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"));
//byte[][] keys = fm.getSequentialKeys(args[2].getBytes(), 500, true);
Iterator rowIt = fm.rows(true, false, (args[2].length() == 0) ? null : args[2].getBytes());
Vector v = new Vector();
@ -1314,10 +1333,10 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
File f = new File(args[3]);
if (f.exists()) f.delete();
kelondroRow lens = new kelondroRow("byte[] key-" + Integer.parseInt(args[1]) + ", byte[] value-" + Integer.parseInt(args[2]));
kelondroTree fm = new kelondroTree(f, 0x100000, 0, 10, lens, true);
kelondroTree fm = new kelondroTree(f, 0x100000, 0, 10, lens);
fm.close();
} else if (args[0].equals("-u")) {
kelondroTree fm = new kelondroTree(new File(args[3]), 0x100000, 0, 10);
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"));
ret = fm.put(args[1].getBytes(), args[2].getBytes());
fm.close();
}
@ -1380,7 +1399,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
int steps = 0;
while (true) {
if (testFile.exists()) testFile.delete();
tt = new kelondroTree(testFile, 200, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"), true);
tt = new kelondroTree(testFile, 200, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"));
steps = 10 + ((int) System.currentTimeMillis() % 7) * (((int) System.currentTimeMillis() + 17) % 11);
t = s;
d = "";
@ -1446,7 +1465,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
File f = new File("test.db");
if (f.exists()) f.delete();
try {
kelondroTree tt = new kelondroTree(f, 1000, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"), true);
kelondroTree tt = new kelondroTree(f, 1000, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"));
byte[] b;
b = testWord('B'); tt.put(b, b); //tt.print();
b = testWord('C'); tt.put(b, b); //tt.print();
@ -1511,7 +1530,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
public static kelondroTree testTree(File f, String testentities) throws IOException {
if (f.exists()) f.delete();
kelondroTree tt = new kelondroTree(f, 0, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"), true);
kelondroTree tt = new kelondroTree(f, 0, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"));
byte[] b;
for (int i = 0; i < testentities.length(); i++) {
b = testWord(testentities.charAt(i));

10
source/de/anomic/plasma/plasmaCrawlBalancer.java
Unescape View File

@ -59,15 +59,7 @@ public class plasmaCrawlBalancer {
private HashMap domainStacks;
public plasmaCrawlBalancer(File stackFile) {
if (stackFile.exists()) {
try {
stack = new kelondroStack(stackFile);
} catch (IOException e) {
stack = new kelondroStack(stackFile, new kelondroRow("byte[] urlhash-" + indexURL.urlHashLength), true);
}
} else {
stack = new kelondroStack(stackFile, new kelondroRow("byte[] urlhash-" + indexURL.urlHashLength), true);
}
stack = kelondroStack.open(stackFile, new kelondroRow("byte[] urlhash-" + indexURL.urlHashLength));
domainStacks = new HashMap();
}

27
source/de/anomic/plasma/plasmaCrawlEURL.java
Unescape View File

@ -57,8 +57,8 @@ import java.util.Iterator;
import de.anomic.index.indexURL;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroFlexTable;
import de.anomic.kelondro.kelondroRow;
import de.anomic.kelondro.kelondroTree;
import de.anomic.tools.bitfield;
public class plasmaCrawlEURL extends indexURL {
@ -138,19 +138,32 @@ public class plasmaCrawlEURL extends indexURL {
"String failcause-" + urlErrorLength + ", " + // string describing load failure
"byte[] flags-" + urlFlagLength); // extra space
if (cachePath.exists()) try {
String newCacheName = "urlErr3.table";
cachePath.mkdirs();
try {
urlHashCache = new kelondroFlexTable(cachePath, newCacheName, kelondroBase64Order.enhancedCoder, bufferkb * 0x400, preloadTime, rowdef);
} catch (IOException e) {
e.printStackTrace();
System.exit(-1);
}
/*
File oldCacheFile = new File(cachePath, "urlErr0.db");
if (oldCacheFile.exists()) try {
// open existing cache
kelondroTree tree = new kelondroTree(cachePath, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent);
kelondroTree tree = new kelondroTree(oldCacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent);
tree.assignRowdef(rowdef);
urlHashCache = tree;
} catch (IOException e) {
cachePath.delete();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, true);
oldCacheFile.delete();
urlHashCache = new kelondroTree(oldCacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, true);
} else {
// create new cache
cachePath.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, true);
oldCacheFile.getParentFile().mkdirs();
urlHashCache = new kelondroTree(oldCacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, true);
}
*/
}
public synchronized Entry newEntry(URL url, String referrer, String initiator, String executor,

22
source/de/anomic/plasma/plasmaCrawlLURL.java
Unescape View File

@ -113,20 +113,14 @@ public final class plasmaCrawlLURL extends indexURL {
"Cardinal size-" + urlSizeLength + " {b64e}, " + // size of file in bytes
"Cardinal wc-" + urlWordCountLength + " {b64e}"); // word count
if (cachePath.exists()) {
// open existing cache
try {
kelondroTree tree = new kelondroTree(cachePath, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent);
tree.assignRowdef(rowdef);
urlHashCache = tree;
} catch (IOException e) {
cachePath.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, true);
}
} else {
// create new cache
cachePath.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, true);
File cacheFile = new File(cachePath, "urlHash.db");
cacheFile.getParentFile().mkdirs();
try {
urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef);
} catch (IOException e) {
e.printStackTrace();
System.exit(-1);
}
// init result stacks

67
source/de/anomic/plasma/plasmaCrawlNURL.java
Unescape View File

@ -55,9 +55,9 @@ import java.util.Iterator;
import de.anomic.index.indexURL;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroFlexTable;
import de.anomic.kelondro.kelondroRecords;
import de.anomic.kelondro.kelondroStack;
import de.anomic.kelondro.kelondroTree;
import de.anomic.kelondro.kelondroRow;
import de.anomic.server.logging.serverLog;
import de.anomic.tools.bitfield;
@ -104,50 +104,32 @@ public class plasmaCrawlNURL extends indexURL {
private long preloadTime;
initStackIndex initThead;
public plasmaCrawlNURL(File cacheStacksPath, int bufferkb, long preloadTime) {
public plasmaCrawlNURL(File cachePath, int bufferkb, long preloadTime) {
super();
this.cacheStacksPath = cacheStacksPath;
this.cacheStacksPath = cachePath;
this.bufferkb = bufferkb;
this.preloadTime = preloadTime;
// create a stack for newly entered entries
if (!(cacheStacksPath.exists())) cacheStacksPath.mkdir(); // make the path
if (!(cachePath.exists())) cachePath.mkdir(); // make the path
openHashCache();
File coreStackFile = new File(cacheStacksPath, "urlNoticeLocal0.stack");
File limitStackFile = new File(cacheStacksPath, "urlNoticeLimit0.stack");
File overhangStackFile = new File(cacheStacksPath, "urlNoticeOverhang0.stack");
File remoteStackFile = new File(cacheStacksPath, "urlNoticeRemote0.stack");
File imageStackFile = new File(cacheStacksPath, "urlNoticeImage0.stack");
File movieStackFile = new File(cacheStacksPath, "urlNoticeMovie0.stack");
File musicStackFile = new File(cacheStacksPath, "urlNoticeMusic0.stack");
File coreStackFile = new File(cachePath, "urlNoticeLocal0.stack");
File limitStackFile = new File(cachePath, "urlNoticeLimit0.stack");
File overhangStackFile = new File(cachePath, "urlNoticeOverhang0.stack");
File remoteStackFile = new File(cachePath, "urlNoticeRemote0.stack");
File imageStackFile = new File(cachePath, "urlNoticeImage0.stack");
File movieStackFile = new File(cachePath, "urlNoticeMovie0.stack");
File musicStackFile = new File(cachePath, "urlNoticeMusic0.stack");
coreStack = new plasmaCrawlBalancer(coreStackFile);
limitStack = new plasmaCrawlBalancer(limitStackFile);
overhangStack = new plasmaCrawlBalancer(overhangStackFile);
remoteStack = new plasmaCrawlBalancer(remoteStackFile);
kelondroRow rowdef = new kelondroRow("byte[] urlhash-" + indexURL.urlHashLength);
if (imageStackFile.exists()) try {
imageStack = new kelondroStack(imageStackFile);
} catch (IOException e) {
imageStack = new kelondroStack(imageStackFile, rowdef, true);
} else {
imageStack = new kelondroStack(imageStackFile, rowdef, true);
}
if (movieStackFile.exists()) try {
movieStack = new kelondroStack(movieStackFile);
} catch (IOException e) {
movieStack = new kelondroStack(movieStackFile, rowdef, true);
} else {
movieStack = new kelondroStack(movieStackFile, rowdef, true);
}
if (musicStackFile.exists()) try {
musicStack = new kelondroStack(musicStackFile);
} catch (IOException e) {
musicStack = new kelondroStack(musicStackFile, rowdef, true);
} else {
musicStack = new kelondroStack(musicStackFile, rowdef, true);
}
imageStack = kelondroStack.open(imageStackFile, rowdef);
movieStack = kelondroStack.open(movieStackFile, rowdef);
musicStack = kelondroStack.open(musicStackFile, rowdef);
// init stack Index
stackIndex = new HashSet();
@ -165,15 +147,17 @@ public class plasmaCrawlNURL extends indexURL {
}
private void openHashCache() {
File oldCacheFile = new File(cacheStacksPath, "urlNotice1.db");
//File newCacheFile = new File(cacheStacksPath, "urlNotice2.table");
/*if (newCacheFile.exists()) try {
urlHashCache = new kelondroFlexTable(cacheStacksPath, "urlNotice2.table", kelondroBase64Order.enhancedCoder, bufferkb * 0x400, preloadTime, rowdef, true);
String newCacheName = "urlNotice3.table";
cacheStacksPath.mkdirs();
try {
urlHashCache = new kelondroFlexTable(cacheStacksPath, newCacheName, kelondroBase64Order.enhancedCoder, bufferkb * 0x400, preloadTime, rowdef);
} catch (IOException e) {
e.printStackTrace();
oldCacheFile.delete();
urlHashCache = new kelondroTree(oldCacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, true);
} else*/ if (oldCacheFile.exists()) try {
System.exit(-1);
}
/*
File oldCacheFile = new File(cacheStacksPath, "urlNotice1.db");
if (oldCacheFile.exists()) try {
// open existing cache
kelondroTree tree = new kelondroTree(oldCacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent);
tree.assignRowdef(rowdef);
@ -184,9 +168,9 @@ public class plasmaCrawlNURL extends indexURL {
} else {
// create new cache
oldCacheFile.getParentFile().mkdirs();
//urlHashCache = new kelondroFlexTable(cacheStacksPath, "urlNotice2.table", kelondroBase64Order.enhancedCoder, bufferkb * 0x400, preloadTime, rowdef, true);
urlHashCache = new kelondroTree(oldCacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, true);
}
*/
}
private void resetHashCache() {
@ -200,6 +184,7 @@ public class plasmaCrawlNURL extends indexURL {
}
public void close() {
try {urlHashCache.close();} catch (IOException e) {}
coreStack.close();
limitStack.close();
overhangStack.close();
@ -552,6 +537,8 @@ public class plasmaCrawlNURL extends indexURL {
this.flags.getBytes(),
normalizeHandle(this.handle).getBytes()
};
if (urlHashCache == null) System.out.println("urlHashCache is NULL");
if ((urlHashCache != null) && (urlHashCache.row() == null)) System.out.println("row() is NULL");
urlHashCache.put(urlHashCache.row().newEntry(entry));
this.stored = true;
} catch (IOException e) {

15
source/de/anomic/plasma/plasmaCrawlProfile.java
Unescape View File

@ -67,16 +67,8 @@ public class plasmaCrawlProfile {
this.profileTableFile = file;
this.bufferkb = bufferkb;
this.preloadTime = preloadTime;
kelondroDyn dyn = null;
if (profileTableFile.exists()) try {
dyn = new kelondroDyn(file, bufferkb * 1024, preloadTime, '#');
} catch (IOException e) {
profileTableFile.delete();
dyn = new kelondroDyn(file, bufferkb * 1024, preloadTime, indexURL.urlCrawlProfileHandleLength, 2000, '#', true);
} else {
profileTableFile.getParentFile().mkdirs();
dyn = new kelondroDyn(file, bufferkb * 1024, preloadTime, indexURL.urlCrawlProfileHandleLength, 2000, '#', true);
}
profileTableFile.getParentFile().mkdirs();
kelondroDyn dyn = kelondroDyn.open(profileTableFile, bufferkb * 1024, preloadTime, indexURL.urlCrawlProfileHandleLength, 2000, '#');
profileTable = new kelondroMap(dyn);
domsCache = new HashMap();
}
@ -102,7 +94,8 @@ public class plasmaCrawlProfile {
if (profileTable != null) try { profileTable.close(); } catch (IOException e) {}
if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database");
profileTableFile.getParentFile().mkdirs();
profileTable = new kelondroMap(new kelondroDyn(profileTableFile, bufferkb * 1024, preloadTime, indexURL.urlCrawlProfileHandleLength, 2000, '#', true));
kelondroDyn dyn = kelondroDyn.open(profileTableFile, bufferkb * 1024, preloadTime, indexURL.urlCrawlProfileHandleLength, 2000, '#');
profileTable = new kelondroMap(dyn);
}
public void close() {

18
source/de/anomic/plasma/plasmaCrawlRobotsTxt.java
Unescape View File

@ -71,20 +71,8 @@ public class plasmaCrawlRobotsTxt {
this.robotsTableFile = robotsTableFile;
this.bufferkb = bufferkb;
this.preloadTime = preloadTime;
if (robotsTableFile.exists()) {
try {
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, preloadTime, '_'));
} catch (kelondroException e) {
robotsTableFile.delete();
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, preloadTime, 256, 512, '_', true));
} catch (IOException e) {
robotsTableFile.delete();
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, preloadTime, 256, 512, '_', true));
}
} else {
robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, preloadTime, 256, 512, '_', true));
}
robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMap(kelondroDyn.open(robotsTableFile, bufferkb * 1024, preloadTime, 256, 512, '_'));
}
public int cacheNodeChunkSize() {
@ -110,7 +98,7 @@ public class plasmaCrawlRobotsTxt {
} catch (IOException e) {}
if (!(robotsTableFile.delete())) throw new RuntimeException("cannot delete robots.txt database");
robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, this.bufferkb, preloadTime, 256, 512, '_', true));
robotsTable = new kelondroMap(kelondroDyn.open(robotsTableFile, this.bufferkb, preloadTime, 256, 512, '_'));
}
public void close() {

93
source/de/anomic/plasma/plasmaCrawlStacker.java
Unescape View File

@ -202,7 +202,7 @@ public final class plasmaCrawlStacker {
Date loadDate,
int currentdepth,
plasmaCrawlProfile.entry profile) {
try {
if (profile != null) try {
this.queue.addMessage(new stackCrawlMessage(
initiatorHash,
nexturlString,
@ -579,6 +579,9 @@ public final class plasmaCrawlStacker {
private final serverSemaphore writeSync;
private final LinkedList urlEntryHashCache;
private kelondroTree urlEntryCache;
private File cacheStacksPath;
private int bufferkb;
private long preloadTime;
public stackCrawlQueue(File cacheStacksPath, int bufferkb, long preloadTime) {
// init the read semaphore
@ -591,54 +594,54 @@ public final class plasmaCrawlStacker {
this.urlEntryHashCache = new LinkedList();
// create a stack for newly entered entries
if (!(cacheStacksPath.exists())) cacheStacksPath.mkdir(); // make the path
this.cacheStacksPath = cacheStacksPath;
this.bufferkb = bufferkb;
this.preloadTime = preloadTime;
File cacheFile = new File(cacheStacksPath, "urlPreNotice.db");
if (cacheFile.exists()) {
// open existing cache
try {
kelondroTree tree = new kelondroTree(cacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent);
tree.assignRowdef(plasmaCrawlNURL.rowdef);
this.urlEntryCache = tree;
} catch (IOException e) {
cacheFile.delete();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.rowdef, true);
}
try {
// loop through the list and fill the messageList with url hashs
Iterator rows = this.urlEntryCache.rows(true, false, null);
kelondroRow.Entry entry;
while (rows.hasNext()) {
entry = (kelondroRow.Entry) rows.next();
if (entry == null) {
System.out.println("ERROR! null element found");
continue;
}
this.urlEntryHashCache.add(entry.getColString(0, null));
this.readSync.V();
openDB();
try {
// loop through the list and fill the messageList with url hashs
Iterator rows = this.urlEntryCache.rows(true, false, null);
kelondroRow.Entry entry;
while (rows.hasNext()) {
entry = (kelondroRow.Entry) rows.next();
if (entry == null) {
System.out.println("ERROR! null element found");
continue;
}
} catch (kelondroException e) {
/* if we have an error, we start with a fresh database */
plasmaCrawlStacker.this.log.logSevere("Unable to initialize crawl stacker queue, kelondroException:" + e.getMessage() + ". Reseting DB.\n",e);
// deleting old db and creating a new db
try {this.urlEntryCache.close();}catch(Exception ex){}
cacheFile.delete();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.rowdef, true);
} catch (IOException e) {
/* if we have an error, we start with a fresh database */
plasmaCrawlStacker.this.log.logSevere("Unable to initialize crawl stacker queue, IOException:" + e.getMessage() + ". Reseting DB.\n",e);
// deleting old db and creating a new db
try {this.urlEntryCache.close();}catch(Exception ex){}
cacheFile.delete();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.rowdef, true);
this.urlEntryHashCache.add(entry.getColString(0, null));
this.readSync.V();
}
} else {
// create new cache
cacheFile.getParentFile().mkdirs();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.rowdef, true);
} catch (kelondroException e) {
/* if we have an error, we start with a fresh database */
plasmaCrawlStacker.this.log.logSevere("Unable to initialize crawl stacker queue, kelondroException:" + e.getMessage() + ". Reseting DB.\n", e);
// deleting old db and creating a new db
try {this.urlEntryCache.close();} catch (Exception ex) {}
deleteDB();
openDB();
} catch (IOException e) {
/* if we have an error, we start with a fresh database */
plasmaCrawlStacker.this.log.logSevere("Unable to initialize crawl stacker queue, IOException:" + e.getMessage() + ". Reseting DB.\n", e);
// deleting old db and creating a new db
try {this.urlEntryCache.close();} catch (Exception ex) {}
deleteDB();
openDB();
}
}
private void deleteDB() {
File cacheFile = new File(cacheStacksPath, "urlPreNotice.db");
cacheFile.delete();
}
private void openDB() {
if (!(cacheStacksPath.exists())) cacheStacksPath.mkdir(); // make the path
File cacheFile = new File(cacheStacksPath, "urlPreNotice.db");
cacheFile.getParentFile().mkdirs();
this.urlEntryCache = kelondroTree.open(cacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.rowdef);
}
public int cacheNodeChunkSize() {

5
source/de/anomic/plasma/plasmaHTCache.java
Unescape View File

@ -154,10 +154,7 @@ public final class plasmaHTCache {
// open the response header database
File dbfile = new File(this.cachePath, "responseHeader.db");
try {
if (dbfile.exists())
this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400, preloadTime, '#'));
else
this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400, preloadTime, indexURL.urlHashLength, 150, '#', false));
this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400, preloadTime, indexURL.urlHashLength, 150, '#'));
} catch (IOException e) {
this.log.logSevere("the request header database could not be opened: " + e.getMessage());
System.exit(0);

2
source/de/anomic/plasma/plasmaRankingCRProcess.java
Unescape View File

@ -260,7 +260,7 @@ public class plasmaRankingCRProcess {
kelondroCollectionIndex newseq = null;
if (newdb) {
File path = to_file.getParentFile(); // path to storage place
newacc = new kelondroFlexTable(path, CRG_accname, kelondroBase64Order.enhancedCoder, 128 * 1024 * 1024, -1, CRG_accrow, true);
newacc = new kelondroFlexTable(path, CRG_accname, kelondroBase64Order.enhancedCoder, 128 * 1024 * 1024, -1, CRG_accrow);
newseq = new kelondroCollectionIndex(path, CRG_seqname, 12, kelondroBase64Order.enhancedCoder, 128 * 1024 * 1024, -1, 2, CRG_colrow);
} else {
if (!(to_file.exists())) {

10
source/de/anomic/plasma/plasmaSwitchboardQueue.java
Unescape View File

@ -89,15 +89,7 @@ public class plasmaSwitchboardQueue {
"String profile-" + indexURL.urlCrawlProfileHandleLength + ", " + // the name of the prefetch profile handle
"String urldescr-" + indexURL.urlDescrLength); //
if (sbQueueStackPath.exists()) try {
sbQueueStack = new kelondroStack(sbQueueStackPath);
sbQueueStack.assignRowdef(rowdef);
} catch (IOException e) {
sbQueueStackPath.delete();
sbQueueStack = new kelondroStack(sbQueueStackPath, rowdef, true);
} else {
sbQueueStack = new kelondroStack(sbQueueStackPath, rowdef, true);
}
sbQueueStack = kelondroStack.open(sbQueueStackPath, rowdef);
}
private void resetQueueStack() {

4
source/de/anomic/plasma/plasmaURLPool.java
Unescape View File

@ -58,9 +58,9 @@ public class plasmaURLPool {
public final plasmaCrawlEURL errorURL;
public plasmaURLPool(File plasmaPath, int ramLURL, int ramNURL, int ramEURL, long preloadTime) {
loadedURL = new plasmaCrawlLURL(new File(plasmaPath, "urlHash.db"), ramLURL, preloadTime);
loadedURL = new plasmaCrawlLURL(plasmaPath, ramLURL, preloadTime);
noticeURL = new plasmaCrawlNURL(plasmaPath, ramNURL, -1);
errorURL = new plasmaCrawlEURL(new File(plasmaPath, "urlErr0.db"), ramEURL, -1);
errorURL = new plasmaCrawlEURL(plasmaPath, ramEURL, -1);
}
public String exists(String hash) {

10
source/de/anomic/plasma/plasmaWordConnotation.java
Unescape View File

@ -58,14 +58,8 @@ public class plasmaWordConnotation {
private static final int nodesize = 4048;
private kelondroDynTree refDB;
public plasmaWordConnotation(File refDBfile, int bufferkb, long preloadTime, char fillChar) {
if (refDBfile.exists()) try {
refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, preloadTime, fillChar);
} catch (IOException e) {
refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, preloadTime, wordlength, nodesize, new kelondroRow("byte[] word-" + wordlength + ", Cardinal count-" + countlength), fillChar, true);
} else {
refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, preloadTime, wordlength, nodesize, new kelondroRow("byte[] word-" + wordlength + ", Cardinal count-" + countlength), fillChar, true);
}
public plasmaWordConnotation(File refDBfile, int bufferkb, long preloadTime, char fillChar) throws IOException {
refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, preloadTime, wordlength, nodesize, new kelondroRow("byte[] word-" + wordlength + ", Cardinal count-" + countlength), fillChar);
}
private void addSingleRef(String word, String reference) throws IOException {

29
source/de/anomic/plasma/plasmaWordIndexAssortment.java
Unescape View File

@ -116,33 +116,16 @@ public final class plasmaWordIndexAssortment {
this.bufferSize = bufferkb * 1024;
this.preloadTime = preloadTime;
this.log = log;
if (assortmentFile.exists()) {
// open existing assortment tree file
try {
long start = System.currentTimeMillis();
assortments = new kelondroTree(assortmentFile, bufferSize, preloadTime, kelondroTree.defaultObjectCachePercent);
assortments.assignRowdef(bufferStructure(assortmentLength));
long stop = System.currentTimeMillis();
if (log != null)
log.logConfig("Opened Assortment, " +
// open assortment tree file
long start = System.currentTimeMillis();
assortments = kelondroTree.open(assortmentFile, bufferSize, preloadTime, kelondroTree.defaultObjectCachePercent, bufferStructure(assortmentLength));
long stop = System.currentTimeMillis();
if (log != null) log.logConfig("Opened Assortment, " +
assortments.size() + " entries, width " +
assortmentLength + ", " + bufferkb + "kb buffer, " +
preloadTime + " ms preloadTime, " +
(stop - start) + " ms effective, " +
assortments.cacheNodeStatus()[1] + " preloaded");
return;
} catch (IOException e){
serverLog.logSevere("PLASMA", "unable to open assortment database " + assortmentLength + ", creating new: " + e.getMessage(), e);
} catch (IndexOutOfBoundsException e){
serverLog.logSevere("PLASMA", "assortment database " + assortmentLength + " corupted, creating new: " + e.getMessage(), e);
} catch (kelondroException e) {
serverLog.logSevere("PLASMA", "assortment database " + assortmentLength + " corupted, creating new: " + e.getMessage(), e);
}
assortmentFile.delete(); // make space for new one
}
// create new assortment tree file
assortments = new kelondroTree(assortmentFile, bufferSize, preloadTime, kelondroTree.defaultObjectCachePercent, bufferStructure(assortmentLength), true);
if (log != null) log.logConfig("Created new Assortment, width " + assortmentLength + ", " + bufferkb + "kb buffer");
}
public void store(indexContainer newContainer) {
@ -269,7 +252,7 @@ public final class plasmaWordIndexAssortment {
if (!(assortmentFile.delete())) throw new RuntimeException("cannot delete assortment database");
}
if (assortmentFile.exists()) assortmentFile.delete();
assortments = new kelondroTree(assortmentFile, bufferSize, preloadTime, kelondroTree.defaultObjectCachePercent, bufferStructure(assortmentLength), true);
assortments = kelondroTree.open(assortmentFile, bufferSize, preloadTime, kelondroTree.defaultObjectCachePercent, bufferStructure(assortmentLength));
}
public Iterator containers(String startWordHash, boolean up, boolean rot) throws IOException {

24
source/de/anomic/plasma/plasmaWordIndexFile.java
Unescape View File

@ -85,23 +85,13 @@ public final class plasmaWordIndexFile {
private kelondroTree indexFile(File databaseRoot, String wordHash) {
if (wordHash.length() < 12) throw new RuntimeException("word hash wrong: '" + wordHash + "'");
theLocation = wordHash2path(databaseRoot, wordHash);
File fp = theLocation.getParentFile();
if (fp != null) fp.mkdirs();
kelondroTree kt;
long cacheSize = theLocation.length();
if (cacheSize > 1048576) cacheSize = 1048576;
if (theLocation.exists()) try {
// open existing index file
kt = new kelondroTree(theLocation, cacheSize, 0, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) {
theLocation.delete();
kt = new kelondroTree(theLocation, cacheSize, 0, kelondroTree.defaultObjectCachePercent, new kelondroRow("byte[] urlhash-" + indexURL.urlHashLength + ", byte[] ba-" + indexURLEntry.encodedByteArrayFormLength(false)), false);
} else {
// create new index file
kt = new kelondroTree(theLocation, cacheSize, 0, kelondroTree.defaultObjectCachePercent, new kelondroRow("byte[] urlhash-" + indexURL.urlHashLength + ", byte[] ba-" + indexURLEntry.encodedByteArrayFormLength(false)), false);
}
return kt; // everyone who get this should close it when finished!
theLocation = wordHash2path(databaseRoot, wordHash);
File fp = theLocation.getParentFile();
if (fp != null) fp.mkdirs();
long cacheSize = theLocation.length();
if (cacheSize > 1048576) cacheSize = 1048576;
return kelondroTree.open(theLocation, cacheSize, 0, kelondroTree.defaultObjectCachePercent,
new kelondroRow("byte[] urlhash-" + indexURL.urlHashLength + ", byte[] ba-" + indexURLEntry.encodedByteArrayFormLength(false)));
}
public static File wordHash2path(File databaseRoot, String hash) {

16
source/de/anomic/yacy/yacyNewsDB.java
Unescape View File

@ -67,25 +67,13 @@ public class yacyNewsDB {
this.path = path;
this.bufferkb = bufferkb;
this.preloadTime = preloadTime;
if (path.exists()) try {
news = new kelondroTree(path, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent);
news.assignRowdef(yacyNewsRecord.rowdef);
} catch (IOException e) {
news = createDB(path, bufferkb, preloadTime);
} else {
news = createDB(path, bufferkb, preloadTime);
}
}
private static kelondroTree createDB(File path, int bufferkb, long preloadTime) {
return new kelondroTree(path, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, yacyNewsRecord.rowdef, true);
this.news = kelondroTree.open(path, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, yacyNewsRecord.rowdef);
}
private void resetDB() {
try {close();} catch (Exception e) {}
if (path.exists()) path.delete();
news = createDB(path, bufferkb, preloadTime);
this.news = kelondroTree.open(path, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, yacyNewsRecord.rowdef);
}
public int cacheNodeChunkSize() {

26
source/de/anomic/yacy/yacyNewsQueue.java
Unescape View File

@ -51,44 +51,28 @@ import java.util.Date;
import de.anomic.kelondro.kelondroColumn;
import de.anomic.kelondro.kelondroRow;
import de.anomic.kelondro.kelondroStack;
import de.anomic.kelondro.kelondroException;
public class yacyNewsQueue {
private File path;
private kelondroStack queueStack;
private yacyNewsDB newsDB;
public yacyNewsQueue(File path, yacyNewsDB newsDB) {
this.path = path;
this.newsDB = newsDB;
if (path.exists()) try {
queueStack = new kelondroStack(path);
} catch (kelondroException e) {
path.delete();
queueStack = createStack(path);
} catch (IOException e) {
path.delete();
queueStack = createStack(path);
} else {
queueStack = createStack(path);
}
}
public static final kelondroRow rowdef = new kelondroRow(new kelondroColumn[]{
new kelondroColumn("newsid", kelondroColumn.celltype_string, kelondroColumn.encoder_bytes, yacyNewsRecord.idLength, "id = created + originator"),
new kelondroColumn("last touched", kelondroColumn.celltype_string, kelondroColumn.encoder_bytes, yacyCore.universalDateShortPattern.length(), "")
});
private static kelondroStack createStack(File path) {
return new kelondroStack(path, rowdef, true);
public yacyNewsQueue(File path, yacyNewsDB newsDB) {
this.path = path;
this.newsDB = newsDB;
this.queueStack = kelondroStack.open(path, rowdef);
}
private void resetDB() {
try {close();} catch (Exception e) {}
if (path.exists()) path.delete();
queueStack = createStack(path);
queueStack = kelondroStack.open(path, rowdef);
}
public void clear() {

17
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -202,25 +202,14 @@ public final class yacySeedDB {
}
private synchronized kelondroMap openSeedTable(File seedDBFile) {
if (seedDBFile.exists()) try {
// open existing seed database
return new kelondroMap(new kelondroDyn(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, '#'), sortFields, accFields);
} catch (kelondroException e) {
// if we have an error, we start with a fresh database
if (seedDBFile.exists()) seedDBFile.delete();
} catch (IOException e) {
// if we have an error, we start with a fresh database
if (seedDBFile.exists()) seedDBFile.delete();
}
// create new seed database
new File(seedDBFile.getParent()).mkdir();
return new kelondroMap(new kelondroDyn(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', true), sortFields, accFields);
new File(seedDBFile.getParent()).mkdirs();
return new kelondroMap(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#'), sortFields, accFields);
}
protected synchronized kelondroMap resetSeedTable(kelondroMap seedDB, File seedDBFile) {
// this is an emergency function that should only be used if any problem with the
// seed.db is detected
yacyCore.log.logFine("seed-db " + seedDBFile.toString() + " reset (on-the-fly)");
yacyCore.log.logFine("seed-db " + seedDBFile.toString() + " reset (on-the-fly)");
try {
seedDB.close();
seedDBFile.delete();

2
source/yacy.java
Unescape View File

@ -1223,7 +1223,7 @@ public final class yacy {
String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"};
for (int i=0; i < dbFileNames.length; i++) {
File dbFile = new File(yacyDBPath,dbFileNames[i]);
kelondroMap db = new kelondroMap(new kelondroDyn(dbFile, (1024 * 0x400) / 3, 3000, '#'), yacySeedDB.sortFields, yacySeedDB.accFields);
kelondroMap db = new kelondroMap(new kelondroDyn(dbFile, (1024 * 0x400) / 3, 3000, yacySeedDB.commonHashLength, 480, '#'), yacySeedDB.sortFields, yacySeedDB.accFields);
kelondroMap.mapIterator it;
it = db.maps(true, false);

Write Preview
Loading…
Cancel
Save