boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

* simplified initialization of database objects

Browse Source 
* replaced kelondroTree for NURLs by kelondroFlex
* replaced kelondroTree for EURLs by kelondroFlex
take care, may be very buggy
please finish crawls before updating. crawls will be lost.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2452 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent 0b73f2b132
commit db1eae0227
42 changed files with 395 additions and 644 deletions
Show Stats Download Patch File Download Diff File

5
doc/News.html
Unescape View File

@ -84,7 +84,7 @@ globalheader();
</ul> </ul>
--> -->
<br>v0.46_20060823_2442<p> <br><p>v0.46_20060823_2442
<ul> <ul>
<li>Web Interface Enhancements</li> <li>Web Interface Enhancements</li>
<ul> <ul>
@ -130,7 +130,8 @@ globalheader();
</ul> </ul>
</ul> </ul>
<br><p>v0.45_20060501_2046<ul> <br><p>v0.45_20060501_2046
<ul>
<li>Enhanced Search Functions</li> <li>Enhanced Search Functions</li>
<ul> <ul>
<li>Re-designed search page: this shows now a simplified page without the left navigation column. The column is only shown if the user is logged-in.</li> <li>Re-designed search page: this shows now a simplified page without the left navigation column. The column is only shown if the user is logged-in.</li>

13
source/dbtest.java
Unescape View File

@ -173,23 +173,18 @@ public class dbtest {
kelondroRow testRow = new kelondroRow("byte[] key-" + keylength + ", byte[] dummy-" + keylength + ", value-" + valuelength); kelondroRow testRow = new kelondroRow("byte[] key-" + keylength + ", byte[] dummy-" + keylength + ", value-" + valuelength);
if (dbe.equals("kelondroTree")) { if (dbe.equals("kelondroTree")) {
File tablefile = new File(tablename + ".kelondro.db"); File tablefile = new File(tablename + ".kelondro.db");
if (tablefile.exists()) { table = new kelondroTree(tablefile, buffer, preload, kelondroTree.defaultObjectCachePercent, testRow);
table = new kelondroTree(tablefile, buffer, preload, kelondroTree.defaultObjectCachePercent);
} else {
table = new kelondroTree(tablefile, buffer, preload, kelondroTree.defaultObjectCachePercent, testRow, true);
}
} }
if (dbe.equals("kelondroSplittedTree")) { if (dbe.equals("kelondroSplittedTree")) {
File tablepath = new File(tablename).getParentFile(); File tablepath = new File(tablename).getParentFile();
tablename = new File(tablename).getName(); tablename = new File(tablename).getName();
table = kelondroSplittedTree.open(tablepath, tablename, kelondroBase64Order.enhancedCoder, table = new kelondroSplittedTree(tablepath, tablename, kelondroBase64Order.enhancedCoder,
buffer, preload, buffer, preload,
8, testRow, 1, 80, 8, testRow, 1, 80);
true);
} }
if (dbe.equals("kelondroFlexTable")) { if (dbe.equals("kelondroFlexTable")) {
File tablepath = new File(tablename).getParentFile(); File tablepath = new File(tablename).getParentFile();
table = new kelondroFlexTable(tablepath, new File(tablename).getName(), kelondroBase64Order.enhancedCoder, buffer, preload, testRow, true); table = new kelondroFlexTable(tablepath, new File(tablename).getName(), kelondroBase64Order.enhancedCoder, buffer, preload, testRow);
} }
if (dbe.equals("mysql")) { if (dbe.equals("mysql")) {
table = new dbTable("mysql", testRow); table = new dbTable("mysql", testRow);

8
source/de/anomic/data/blogBoard.java
Unescape View File

@ -83,13 +83,7 @@ public class blogBoard {
public blogBoard(File actpath, int bufferkb, long preloadTime) { public blogBoard(File actpath, int bufferkb, long preloadTime) {
new File(actpath.getParent()).mkdir(); new File(actpath.getParent()).mkdir();
if (datbase == null) { if (datbase == null) {
if (actpath.exists()) try { datbase = new kelondroMap(kelondroDyn.open(actpath, bufferkb / 2 * 0x40, preloadTime, keyLength, recordSize, '_'));
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x40, preloadTime, '_'));
} catch (IOException e) {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_', true));
} else {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_', true));
}
} }
} }

72
source/de/anomic/data/bookmarksDB.java
Unescape View File

@ -120,67 +120,25 @@ public class bookmarksDB {
} }
public bookmarksDB(File bookmarksFile, File tagsFile, File datesFile, int bufferkb, long preloadTime) { public bookmarksDB(File bookmarksFile, File tagsFile, File datesFile, int bufferkb, long preloadTime) {
//bookmarks // bookmarks
//check if database exists
tagCache=new HashMap(); tagCache=new HashMap();
bookmarkCache=new HashMap(); bookmarkCache=new HashMap();
if(bookmarksFile.exists()){ bookmarksFile.getParentFile().mkdirs();
try { this.bookmarksTable = new kelondroMap(kelondroDyn.open(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_'));
//open it
this.bookmarksTable=new kelondroMap(new kelondroDyn(bookmarksFile, 1024*bufferkb, preloadTime, '_')); // tags
} catch (IOException e) { tagsFile.getParentFile().mkdirs();
//database reset :-(( boolean tagsFileExisted = tagsFile.exists();
bookmarksFile.delete(); this.tagsTable = new kelondroMap(kelondroDyn.open(tagsFile, bufferkb * 1024, preloadTime, 12, 256, '_'));
bookmarksFile.getParentFile().mkdirs(); if (!tagsFileExisted) rebuildTags();
//urlHash is 12 bytes long
this.bookmarksTable = new kelondroMap(new kelondroDyn(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true));
}
}else{
//new database
bookmarksFile.getParentFile().mkdirs();
this.bookmarksTable = new kelondroMap(new kelondroDyn(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true));
}
//tags
//check if database exists
if(tagsFile.exists()){
try {
//open it
this.tagsTable=new kelondroMap(new kelondroDyn(tagsFile, 1024*bufferkb, preloadTime, '_'));
} catch (IOException e) {
//reset database
tagsFile.delete();
tagsFile.getParentFile().mkdirs();
// max. 128 byte long tags
this.tagsTable = new kelondroMap(new kelondroDyn(tagsFile, bufferkb * 1024, preloadTime, 12, 256, '_', true));
rebuildTags();
}
}else{
//new database
tagsFile.getParentFile().mkdirs();
this.tagsTable = new kelondroMap(new kelondroDyn(tagsFile, bufferkb * 1024, preloadTime, 12, 256, '_', true));
rebuildTags();
}
// dates // dates
//check if database exists boolean datesExisted = datesFile.exists();
if(datesFile.exists()){ this.datesTable = new kelondroMap(kelondroDyn.open(datesFile, bufferkb * 1024, preloadTime, 20, 256, '_'));
try { if (!datesExisted) rebuildDates();
//open it
this.datesTable=new kelondroMap(new kelondroDyn(datesFile, 1024*bufferkb, preloadTime, '_'));
} catch (IOException e) {
//reset database
datesFile.delete();
datesFile.getParentFile().mkdirs();
//YYYY-MM-DDTHH:mm:ssZ = 20 byte. currently used: YYYY-MM-DD = 10 bytes
this.datesTable = new kelondroMap(new kelondroDyn(datesFile, bufferkb * 1024, preloadTime, 20, 256, '_', true));
rebuildDates();
}
}else{
//new database
datesFile.getParentFile().mkdirs();
this.datesTable = new kelondroMap(new kelondroDyn(datesFile, bufferkb * 1024, preloadTime, 20, 256, '_', true));
rebuildDates();
}
} }
public void close(){ public void close(){
try { try {
flushBookmarkCache(); flushBookmarkCache();

9
source/de/anomic/data/messageBoard.java
Unescape View File

@ -70,14 +70,7 @@ public class messageBoard {
public messageBoard(File path, int bufferkb, long preloadTime) { public messageBoard(File path, int bufferkb, long preloadTime) {
new File(path.getParent()).mkdir(); new File(path.getParent()).mkdir();
if (database == null) { if (database == null) {
if (path.exists()) try { database = new kelondroMap(kelondroDyn.open(path, bufferkb * 0x400, preloadTime, categoryLength + dateFormat.length() + 2, recordSize, '_'));
database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, preloadTime, '_'));
} catch (IOException e) {
path.delete();
database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, preloadTime, categoryLength + dateFormat.length() + 2, recordSize, '_', true));
} else {
database = new kelondroMap(new kelondroDyn(path, bufferkb * 0x400, preloadTime, categoryLength + dateFormat.length() + 2, recordSize, '_', true));
}
} }
sn = 0; sn = 0;
} }

20
source/de/anomic/data/userDB.java
Unescape View File

@ -76,22 +76,8 @@ public final class userDB {
this.userTableFile = userTableFile; this.userTableFile = userTableFile;
this.bufferkb = bufferkb; this.bufferkb = bufferkb;
this.preloadTime = preloadTime; this.preloadTime = preloadTime;
if (userTableFile.exists()) { userTableFile.getParentFile().mkdirs();
try { this.userTable = new kelondroMap(kelondroDyn.open(userTableFile, bufferkb * 1024, preloadTime, 128, 256, '_'));
this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, preloadTime, '_'));
} catch (kelondroException e) {
userTableFile.delete();
userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, preloadTime, 128, 256, '_', true));
} catch (IOException e) {
userTableFile.delete();
userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, preloadTime, 128, 256, '_', true));
}
} else {
userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMap(new kelondroDyn(userTableFile, bufferkb * 1024, preloadTime, 128, 256, '_', true));
}
} }
public int dbCacheNodeChunkSize() { public int dbCacheNodeChunkSize() {
@ -109,7 +95,7 @@ public final class userDB {
} catch (IOException e) {} } catch (IOException e) {}
if (!(userTableFile.delete())) throw new RuntimeException("cannot delete user database"); if (!(userTableFile.delete())) throw new RuntimeException("cannot delete user database");
userTableFile.getParentFile().mkdirs(); userTableFile.getParentFile().mkdirs();
userTable = new kelondroMap(new kelondroDyn(userTableFile, this.bufferkb, preloadTime, 256, 512, '_', true)); userTable = new kelondroMap(kelondroDyn.open(userTableFile, this.bufferkb, preloadTime, 256, 512, '_'));
} }
public void close() { public void close() {

20
source/de/anomic/data/wikiBoard.java
Unescape View File

@ -70,25 +70,13 @@ public class wikiBoard {
private static HashMap authors = new HashMap(); private static HashMap authors = new HashMap();
public wikiBoard(File actpath, File bkppath, int bufferkb, long preloadTime) { public wikiBoard(File actpath, File bkppath, int bufferkb, long preloadTime) {
new File(actpath.getParent()).mkdir(); new File(actpath.getParent()).mkdirs();
if (datbase == null) { if (datbase == null) {
if (actpath.exists()) try { datbase = new kelondroMap(kelondroDyn.open(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_'));
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x40, preloadTime, '_'));
} catch (IOException e) {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_', true));
} else {
datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_', true));
}
} }
new File(bkppath.getParent()).mkdir(); new File(bkppath.getParent()).mkdirs();
if (bkpbase == null) { if (bkpbase == null) {
if (bkppath.exists()) try { bkpbase = new kelondroMap(kelondroDyn.open(bkppath, bufferkb / 2 * 0x400, preloadTime, keyLength + dateFormat.length(), recordSize, '_'));
bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400, preloadTime, '_'));
} catch (IOException e) {
bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400, preloadTime, keyLength + dateFormat.length(), recordSize, '_', true));
} else {
bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb / 2 * 0x400, preloadTime, keyLength + dateFormat.length(), recordSize, '_', true));
}
} }
} }

4
source/de/anomic/index/indexContainerOrder.java
Unescape View File

@ -53,4 +53,8 @@ public class indexContainerOrder implements kelondroOrder {
return this.embeddedOrder.compare(a, aoffset, alength, b, boffset, blength); return this.embeddedOrder.compare(a, aoffset, alength, b, boffset, blength);
} }
public boolean equals(kelondroOrder otherOrder) {
if (!(otherOrder instanceof indexContainerOrder)) return false;
return this.embeddedOrder.equals(((indexContainerOrder) otherOrder).embeddedOrder);
}
} }

4
source/de/anomic/index/indexRAMCacheRI.java
Unescape View File

@ -96,7 +96,7 @@ public final class indexRAMCacheRI extends indexAbstractRI implements indexRI {
File indexDumpFile = new File(databaseRoot, indexArrayFileName); File indexDumpFile = new File(databaseRoot, indexArrayFileName);
if (indexDumpFile.exists()) indexDumpFile.delete(); if (indexDumpFile.exists()) indexDumpFile.delete();
kelondroFixedWidthArray dumpArray = null; kelondroFixedWidthArray dumpArray = null;
dumpArray = new kelondroFixedWidthArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0, false); dumpArray = new kelondroFixedWidthArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0);
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
long messageTime = System.currentTimeMillis() + 5000; long messageTime = System.currentTimeMillis() + 5000;
long wordsPerSecond = 0, wordcount = 0, urlcount = 0; long wordsPerSecond = 0, wordcount = 0, urlcount = 0;
@ -175,7 +175,7 @@ public final class indexRAMCacheRI extends indexAbstractRI implements indexRI {
private long restore() throws IOException { private long restore() throws IOException {
File indexDumpFile = new File(databaseRoot, indexArrayFileName); File indexDumpFile = new File(databaseRoot, indexArrayFileName);
if (!(indexDumpFile.exists())) return 0; if (!(indexDumpFile.exists())) return 0;
kelondroFixedWidthArray dumpArray = new kelondroFixedWidthArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis); kelondroFixedWidthArray dumpArray = new kelondroFixedWidthArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0);
log.logConfig("restore array dump of index cache, " + dumpArray.size() + " word/URL relations"); log.logConfig("restore array dump of index cache, " + dumpArray.size() + " word/URL relations");
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
long messageTime = System.currentTimeMillis() + 5000; long messageTime = System.currentTimeMillis() + 5000;

2
source/de/anomic/index/indexURL.java
Unescape View File

@ -451,7 +451,7 @@ public class indexURL {
public long[] cacheObjectStatus() { public long[] cacheObjectStatus() {
if (urlHashCache instanceof kelondroTree) return ((kelondroTree) urlHashCache).cacheObjectStatus(); if (urlHashCache instanceof kelondroTree) return ((kelondroTree) urlHashCache).cacheObjectStatus();
return null; return new long[]{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
} }
public static final int flagTypeID(String hash) { public static final int flagTypeID(String hash) {

8
source/de/anomic/kelondro/kelondroAbstractOrder.java
Unescape View File

@ -83,4 +83,12 @@ public abstract class kelondroAbstractOrder implements kelondroOrder {
public void rotate(byte[] newzero) { public void rotate(byte[] newzero) {
this.zero = newzero; this.zero = newzero;
} }
public boolean equals(kelondroOrder otherOrder) {
String thisSig = this.signature();
String otherSig = otherOrder.signature();
if ((thisSig == null) || (otherSig == null)) return false;
return thisSig.equals(otherSig);
}
} }

13
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -91,7 +91,7 @@ public class kelondroCollectionIndex {
this.loadfactor = loadfactor; this.loadfactor = loadfactor;
// create index table // create index table
index = new kelondroFlexTable(path, filenameStub + ".index.table", indexOrder, buffersize, preloadTime, indexRow(keyLength), true); index = new kelondroFlexTable(path, filenameStub + ".index.table", indexOrder, buffersize, preloadTime, indexRow(keyLength));
// save/check property file for this array // save/check property file for this array
File propfile = propertyFile(path, filenameStub, loadfactor, rowdef.objectsize()); File propfile = propertyFile(path, filenameStub, loadfactor, rowdef.objectsize());
@ -119,13 +119,8 @@ public class kelondroCollectionIndex {
"byte[] key-" + index.row().width(0) + "," + "byte[] key-" + index.row().width(0) + "," +
"byte[] collection-" + (kelondroRowCollection.exportOverheadSize + load * this.playloadrow.objectsize()) "byte[] collection-" + (kelondroRowCollection.exportOverheadSize + load * this.playloadrow.objectsize())
); );
if (f.exists()) { if ((!(f.exists())) && (!create)) return null;
return new kelondroFixedWidthArray(f, rowdef); return new kelondroFixedWidthArray(f, rowdef, 0);
} else if (create) {
return new kelondroFixedWidthArray(f, rowdef, 0, true);
} else {
return null;
}
} }
private kelondroFixedWidthArray getArray(int partitionNumber, int serialNumber, int chunksize) { private kelondroFixedWidthArray getArray(int partitionNumber, int serialNumber, int chunksize) {
@ -460,7 +455,7 @@ public class kelondroCollectionIndex {
collectionIndex.close(); collectionIndex.close();
// printout of index // printout of index
kelondroFlexTable index = new kelondroFlexTable(path, filenameStub + ".index", kelondroNaturalOrder.naturalOrder, buffersize, preloadTime, indexRow(9), true); kelondroFlexTable index = new kelondroFlexTable(path, filenameStub + ".index", kelondroNaturalOrder.naturalOrder, buffersize, preloadTime, indexRow(9));
index.print(); index.print();
index.close(); index.close();
} catch (IOException e) { } catch (IOException e) {

53
source/de/anomic/kelondro/kelondroDyn.java
Unescape View File

@ -68,31 +68,43 @@ public class kelondroDyn extends kelondroTree {
private char fillChar; private char fillChar;
private kelondroObjectBuffer buffer; private kelondroObjectBuffer buffer;
public kelondroDyn(File file, long buffersize /*bytes*/, long preloadTime, int key, int nodesize, char fillChar, boolean exitOnFail) { public kelondroDyn(File file, long buffersize /*bytes*/, long preloadTime, int key, int nodesize, char fillChar) throws IOException {
this(file, buffersize, preloadTime, key, nodesize, fillChar, new kelondroNaturalOrder(true), exitOnFail); this(file, buffersize, preloadTime, key, nodesize, fillChar, new kelondroNaturalOrder(true));
} }
public kelondroDyn(File file, long buffersize /* bytes */, long preloadTime, int key, public kelondroDyn(File file, long buffersize /* bytes */, long preloadTime, int key,
int nodesize, char fillChar, kelondroOrder objectOrder, int nodesize, char fillChar, kelondroOrder objectOrder) throws IOException {
boolean exitOnFail) { // creates or opens a dynamic tree
// creates a new dynamic tree super(file, buffersize, preloadTime, kelondroTree.defaultObjectCachePercent, new kelondroRow("byte[] key-" + (key + counterlen) + ", byte[] node-" + nodesize), objectOrder, 1, 8);
super(file, buffersize, preloadTime, kelondroTree.defaultObjectCachePercent, new kelondroRow("byte[] key-" + (key + counterlen) + ", byte[] node-" + nodesize), objectOrder, 1, 8, exitOnFail);
this.keylen = row().width(0) - counterlen; this.keylen = row().width(0) - counterlen;
this.reclen = row().width(1); this.reclen = row().width(1);
this.fillChar = fillChar; this.fillChar = fillChar;
this.segmentCount = 0; this.segmentCount = 0;
writeSegmentCount(); if (!(super.fileExisted)) writeSegmentCount();
buffer = new kelondroObjectBuffer(file.toString()); buffer = new kelondroObjectBuffer(file.toString());
} }
public kelondroDyn(File file, long buffersize, long preloadTime, char fillChar) throws IOException { public static final kelondroDyn open(File file, long buffersize /* bytes */, long preloadTime, int key, int nodesize, char fillChar) {
// this opens a file with an existing dynamic tree return open(file, buffersize, preloadTime, key, nodesize, fillChar, new kelondroNaturalOrder(true));
super(file, buffersize, preloadTime, kelondroTree.defaultObjectCachePercent); }
this.keylen = row().width(0) - counterlen;
this.reclen = row().width(1); public static final kelondroDyn open(File file, long buffersize /* bytes */, long preloadTime, int key,
this.fillChar = fillChar; int nodesize, char fillChar, kelondroOrder objectOrder) {
this.segmentCount = 0; // opens new or existing file; in case that any error occur the file is deleted again and it is tried to create the file again
buffer = new kelondroObjectBuffer(file.toString()); // if that fails, the method returns null
try {
return new kelondroDyn(file, buffersize, preloadTime, key, nodesize, fillChar, objectOrder);
} catch (IOException e) {
file.delete();
try {
return new kelondroDyn(file, buffersize, preloadTime, key, nodesize, fillChar, objectOrder);
} catch (IOException ee) {
log.severe("cannot open or create file " + file.toString());
e.printStackTrace();
ee.printStackTrace();
return null;
}
}
} }
private void writeSegmentCount() { private void writeSegmentCount() {
@ -442,7 +454,7 @@ public class kelondroDyn extends kelondroTree {
} else if (args.length == 1) { } else if (args.length == 1) {
// open a db and list keys // open a db and list keys
try { try {
kelondroDyn kd = new kelondroDyn(new File(args[0]), 0x100000, 0, '_'); kelondroDyn kd = new kelondroDyn(new File(args[0]), 0x100000, 0, 4 ,100, '_');
System.out.println(kd.size() + " elements in DB"); System.out.println(kd.size() + " elements in DB");
Iterator i = kd.dynKeys(true, false); Iterator i = kd.dynKeys(true, false);
while (i.hasNext()) while (i.hasNext())
@ -459,10 +471,7 @@ public class kelondroDyn extends kelondroTree {
File f = new File(args[3]); File f = new File(args[3]);
kelondroDyn kd; kelondroDyn kd;
try { try {
if (db.exists()) kd = new kelondroDyn(db, 0x100000, 0, 80, 200, '_');
kd = new kelondroDyn(db, 0x100000, 0, '_');
else
kd = new kelondroDyn(db, 0x100000, 0, 80, 200, '_', true);
if (writeFile) if (writeFile)
kd.readFile(key, f); kd.readFile(key, f);
else else
@ -486,7 +495,7 @@ public class kelondroDyn extends kelondroTree {
int steps = 0; int steps = 0;
while (true) { while (true) {
if (testFile.exists()) testFile.delete(); if (testFile.exists()) testFile.delete();
tt = new kelondroDyn(testFile, 0, 0, 4 ,100, '_', true); tt = new kelondroDyn(testFile, 0, 0, 4 ,100, '_');
steps = ((int) System.currentTimeMillis() % 7) * (((int) System.currentTimeMillis() + 17) % 11); steps = ((int) System.currentTimeMillis() % 7) * (((int) System.currentTimeMillis() + 17) % 11);
t = s; t = s;
d = ""; d = "";

35
source/de/anomic/kelondro/kelondroDynTree.java
Unescape View File

@ -48,7 +48,6 @@ import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.Enumeration; import java.util.Enumeration;
import java.util.Hashtable; import java.util.Hashtable;
import java.util.Iterator;
public class kelondroDynTree { public class kelondroDynTree {
@ -70,8 +69,8 @@ public class kelondroDynTree {
private Hashtable buffer, cache; private Hashtable buffer, cache;
private long cycleBuffer; private long cycleBuffer;
public kelondroDynTree(File file, long buffersize, long preloadTime, int keylength, int nodesize, kelondroRow rowdef, char fillChar, boolean exitOnFail) { public kelondroDynTree(File file, long buffersize, long preloadTime, int keylength, int nodesize, kelondroRow rowdef, char fillChar) throws IOException {
// creates a new DynTree // creates or opens a DynTree
this.file = file; this.file = file;
this.buffersize = buffersize; this.buffersize = buffersize;
this.preloadTime = preloadTime; this.preloadTime = preloadTime;
@ -80,33 +79,9 @@ public class kelondroDynTree {
this.cache = new Hashtable(); this.cache = new Hashtable();
//this.cycleCache = Long.MIN_VALUE; //this.cycleCache = Long.MIN_VALUE;
this.cycleBuffer = Long.MIN_VALUE; this.cycleBuffer = Long.MIN_VALUE;
if (file.exists()) file.delete(); this.table = new kelondroDyn(file, buffersize, preloadTime, keylength, nodesize, fillChar);
this.table = new kelondroDyn(file, buffersize, preloadTime, keylength, nodesize, fillChar, exitOnFail);
this.treeRAHandles = new Hashtable(); this.treeRAHandles = new Hashtable();
} }
public kelondroDynTree(File file, long buffersize, long preloadTime, char fillChar) throws IOException {
// opens an existing DynTree
this.file = file;
this.buffersize = buffersize;
this.preloadTime = preloadTime;
this.buffer = new Hashtable();
this.cache = new Hashtable();
//this.cycleCache = Long.MIN_VALUE;
this.cycleBuffer = Long.MIN_VALUE;
if (!(file.exists())) throw new IOException("DynTree " + file.toString() + " does not exist");
this.table = new kelondroDyn(file, buffersize, preloadTime, fillChar);
// read one element to measure the size of columns
if (table.size() == 0) throw new IOException("DynTree " + file.toString() + " is empty. Should not.");
this.treeRAHandles = new Hashtable();
Iterator i = table.dynKeys(true, false);
String onekey = (String) i.next();
kelondroTree onetree = getTree(onekey);
kelondroColumn[] columns = new kelondroColumn[onetree.row().columns()];
for (int j = 0; j < columns.length; j++) columns[j] = onetree.row().column(j);
this.rowdef = new kelondroRow(columns);
closeTree(onekey);
}
public void close() throws IOException { public void close() throws IOException {
Enumeration e = treeRAHandles.keys(); Enumeration e = treeRAHandles.keys();
@ -323,10 +298,10 @@ public class kelondroDynTree {
System.out.println("start"); System.out.println("start");
File file = new File("D:\\bin\\testDyn.db"); File file = new File("D:\\bin\\testDyn.db");
if (file.exists()) { if (file.exists()) {
kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, 0, '_'); kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, 0, 16, 512, new kelondroRow("byte[] a-10, byte[] b-20, byte[] c-30"), '_');
System.out.println("opened: table keylength=" + dt.table.row().width(0) + ", sectorsize=" + dt.table.row().width(1) + ", " + dt.table.size() + " entries."); System.out.println("opened: table keylength=" + dt.table.row().width(0) + ", sectorsize=" + dt.table.row().width(1) + ", " + dt.table.size() + " entries.");
} else { } else {
kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, 0, 16, 512, new kelondroRow("byte[] a-10, byte[] b-20, byte[] c-30"), '_', true); kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, 0, 16, 512, new kelondroRow("byte[] a-10, byte[] b-20, byte[] c-30"), '_');
String name; String name;
kelondroTree t; kelondroTree t;
kelondroRow.Entry line; kelondroRow.Entry line;

36
source/de/anomic/kelondro/kelondroFixedWidthArray.java
Unescape View File

@ -55,27 +55,19 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
private static short thisOHBytes = 0; // our record definition does not need extra bytes private static short thisOHBytes = 0; // our record definition does not need extra bytes
private static short thisOHHandles = 0; // and no handles private static short thisOHHandles = 0; // and no handles
public kelondroFixedWidthArray(File file, kelondroRow rowdef, int intprops, boolean exitOnFail) { public kelondroFixedWidthArray(File file, kelondroRow rowdef, int intprops) throws IOException {
// this creates a new array // this creates a new array
super(file, 0, 0, thisOHBytes, thisOHHandles, rowdef, intprops, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail); super(file, 0, 0, thisOHBytes, thisOHHandles, rowdef, intprops, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */);
for (int i = 0; i < intprops; i++) try { if (!(super.fileExisted)) {
setHandle(i, new Handle(0)); for (int i = 0; i < intprops; i++) {
} catch (IOException e) { setHandle(i, new Handle(0));
super.logFailure("cannot set handle " + i + " / " + e.getMessage()); }
if (exitOnFail) System.exit(-1); // store column description
throw new RuntimeException("cannot set handle " + i + " / " + e.getMessage()); for (int i = 0; i < rowdef.columns(); i++) {
} try {super.setText(i, rowdef.column(i).toString().getBytes());} catch (IOException e) {}
// store column description }
for (int i = 0; i < rowdef.columns(); i++) {
try {super.setText(i, rowdef.column(i).toString().getBytes());} catch (IOException e) {}
} }
} }
public kelondroFixedWidthArray(File file, kelondroRow rowdef) throws IOException{
// this opens a file with an existing array
super(file, 0, 0);
super.assignRowdef(rowdef);
}
public synchronized kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException { public synchronized kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException {
@ -142,14 +134,14 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
try { try {
System.out.println("erster Test"); System.out.println("erster Test");
f.delete(); f.delete();
kelondroFixedWidthArray k = new kelondroFixedWidthArray(f, rowdef, 6, true); kelondroFixedWidthArray k = new kelondroFixedWidthArray(f, rowdef, 6);
k.set(3, k.row().newEntry(new byte[][]{ k.set(3, k.row().newEntry(new byte[][]{
"test123".getBytes(), "abcd".getBytes()})); "test123".getBytes(), "abcd".getBytes()}));
k.add(k.row().newEntry(new byte[][]{ k.add(k.row().newEntry(new byte[][]{
"test456".getBytes(), "efgh".getBytes()})); "test456".getBytes(), "efgh".getBytes()}));
k.close(); k.close();
k = new kelondroFixedWidthArray(f, rowdef); k = new kelondroFixedWidthArray(f, rowdef, 6);
System.out.println(k.get(2).toString()); System.out.println(k.get(2).toString());
System.out.println(k.get(3).toString()); System.out.println(k.get(3).toString());
System.out.println(k.get(4).toString()); System.out.println(k.get(4).toString());
@ -157,7 +149,7 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
System.out.println("zweiter Test"); System.out.println("zweiter Test");
f.delete(); f.delete();
k = new kelondroFixedWidthArray(f, rowdef, 6, true); k = new kelondroFixedWidthArray(f, rowdef, 6);
k.add(k.row().newEntry(new byte[][]{"a".getBytes(), "xxxx".getBytes()})); k.add(k.row().newEntry(new byte[][]{"a".getBytes(), "xxxx".getBytes()}));
k.add(k.row().newEntry(new byte[][]{"b".getBytes(), "xxxx".getBytes()})); k.add(k.row().newEntry(new byte[][]{"b".getBytes(), "xxxx".getBytes()}));
k.remove(0); k.remove(0);
@ -176,7 +168,7 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
System.out.println("dritter Test"); System.out.println("dritter Test");
f.delete(); f.delete();
k = new kelondroFixedWidthArray(f, rowdef, 6, true); k = new kelondroFixedWidthArray(f, rowdef, 6);
for (int i = 1; i <= 200; i = i * 2) { for (int i = 1; i <= 200; i = i * 2) {
for (int j = 0; j < i*2; j++) { for (int j = 0; j < i*2; j++) {
k.add(k.row().newEntry(new byte[][]{(Integer.toString(i) + "-" + Integer.toString(j)).getBytes(), "xxxx".getBytes()})); k.add(k.row().newEntry(new byte[][]{(Integer.toString(i) + "-" + Integer.toString(j)).getBytes(), "xxxx".getBytes()}));

14
source/de/anomic/kelondro/kelondroFlexTable.java
Unescape View File

@ -33,8 +33,8 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
protected kelondroBytesIntMap index; protected kelondroBytesIntMap index;
public kelondroFlexTable(File path, String tablename, kelondroOrder objectOrder, long buffersize, long preloadTime, kelondroRow rowdef, boolean exitOnFail) throws IOException { public kelondroFlexTable(File path, String tablename, kelondroOrder objectOrder, long buffersize, long preloadTime, kelondroRow rowdef) throws IOException {
super(path, tablename, rowdef, exitOnFail); super(path, tablename, rowdef);
File newpath = new File(path, tablename); File newpath = new File(path, tablename);
File indexfile = new File(newpath, "col.000.index"); File indexfile = new File(newpath, "col.000.index");
kelondroIndex ki = null; kelondroIndex ki = null;
@ -52,7 +52,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
if (indexfile.exists()) { if (indexfile.exists()) {
// use existing index file // use existing index file
System.out.println("*** Using File index " + indexfile); System.out.println("*** Using File index " + indexfile);
ki = new kelondroTree(indexfile, buffersize, preloadTime, 10); ki = kelondroTree.open(indexfile, buffersize, preloadTime, 10, treeIndexRow(rowdef.width(0)), objectOrder, 2, 80);
} else if ((preloadTime >= 0) && (stt > preloadTime)) { } else if ((preloadTime >= 0) && (stt > preloadTime)) {
// generate new index file // generate new index file
System.out.print("*** Generating File index for " + size() + " entries from " + indexfile); System.out.print("*** Generating File index for " + size() + " entries from " + indexfile);
@ -104,9 +104,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
} }
private kelondroTree initializeTreeIndex(File indexfile, long buffersize, long preloadTime, kelondroOrder objectOrder) throws IOException { private kelondroTree initializeTreeIndex(File indexfile, long buffersize, long preloadTime, kelondroOrder objectOrder) throws IOException {
kelondroTree treeindex = new kelondroTree(indexfile, buffersize, preloadTime, 10, kelondroTree treeindex = new kelondroTree(indexfile, buffersize, preloadTime, 10, treeIndexRow(rowdef.width(0)), objectOrder, 2, 80);
new kelondroRow("byte[] key-" + rowdef.width(0) + ", int reference-4 {b256}"),
objectOrder, 2, 80, true);
Iterator content = super.col[0].contentNodes(-1); Iterator content = super.col[0].contentNodes(-1);
kelondroRecords.Node node; kelondroRecords.Node node;
kelondroRow.Entry indexentry; kelondroRow.Entry indexentry;
@ -126,6 +124,10 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
return treeindex; return treeindex;
} }
private static final kelondroRow treeIndexRow(int keywidth) {
return new kelondroRow("byte[] key-" + keywidth + ", int reference-4 {b256}");
}
public synchronized kelondroRow.Entry get(byte[] key) throws IOException { public synchronized kelondroRow.Entry get(byte[] key) throws IOException {
synchronized (index) { synchronized (index) {
int i = index.geti(key); int i = index.geti(key);

10
source/de/anomic/kelondro/kelondroFlexWidthArray.java
Unescape View File

@ -37,7 +37,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
protected kelondroFixedWidthArray[] col; protected kelondroFixedWidthArray[] col;
protected kelondroRow rowdef; protected kelondroRow rowdef;
public kelondroFlexWidthArray(File path, String tablename, kelondroRow rowdef, boolean exitOnFail) throws IOException { public kelondroFlexWidthArray(File path, String tablename, kelondroRow rowdef) throws IOException {
this.rowdef = rowdef; this.rowdef = rowdef;
// initialize columns // initialize columns
@ -81,7 +81,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
kelondroColumn columns[] = new kelondroColumn[colend - colstart + 1]; kelondroColumn columns[] = new kelondroColumn[colend - colstart + 1];
for (int j = colstart; j <= colend; j++) columns[j-colstart] = rowdef.column(j); for (int j = colstart; j <= colend; j++) columns[j-colstart] = rowdef.column(j);
col[colstart] = new kelondroFixedWidthArray(new File(tabledir, files[i]), new kelondroRow(columns)); col[colstart] = new kelondroFixedWidthArray(new File(tabledir, files[i]), new kelondroRow(columns), 16);
for (int j = colstart; j <= colend; j++) check = check.substring(0, j) + "X" + check.substring(j + 1); for (int j = colstart; j <= colend; j++) check = check.substring(0, j) + "X" + check.substring(j + 1);
} }
} }
@ -100,7 +100,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
columns[j - p] = rowdef.column(j); columns[j - p] = rowdef.column(j);
check = check.substring(0, j) + "X" + check.substring(j + 1); check = check.substring(0, j) + "X" + check.substring(j + 1);
} }
col[p] = new kelondroFixedWidthArray(new File(tabledir, colfilename(p, q)), new kelondroRow(columns), 16, true); col[p] = new kelondroFixedWidthArray(new File(tabledir, colfilename(p, q)), new kelondroRow(columns), 16);
} }
} }
@ -224,7 +224,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
System.out.println("erster Test"); System.out.println("erster Test");
new File(f, testname).delete(); new File(f, testname).delete();
kelondroFlexWidthArray k = new kelondroFlexWidthArray(f, "flextest", rowdef, true); kelondroFlexWidthArray k = new kelondroFlexWidthArray(f, "flextest", rowdef);
k.add(k.row().newEntry(new byte[][]{"a".getBytes(), "xxxx".getBytes()})); k.add(k.row().newEntry(new byte[][]{"a".getBytes(), "xxxx".getBytes()}));
k.add(k.row().newEntry(new byte[][]{"b".getBytes(), "xxxx".getBytes()})); k.add(k.row().newEntry(new byte[][]{"b".getBytes(), "xxxx".getBytes()}));
k.remove(0); k.remove(0);
@ -244,7 +244,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
System.out.println("zweiter Test"); System.out.println("zweiter Test");
new File(f, testname).delete(); new File(f, testname).delete();
k = new kelondroFlexWidthArray(f, "flextest", rowdef, true); k = new kelondroFlexWidthArray(f, "flextest", rowdef);
for (int i = 1; i <= 20; i = i * 2) { for (int i = 1; i <= 20; i = i * 2) {
for (int j = 0; j < i*2; j++) { for (int j = 0; j < i*2; j++) {
k.add(k.row().newEntry(new byte[][]{(Integer.toString(i) + "-" + Integer.toString(j)).getBytes(), "xxxx".getBytes()})); k.add(k.row().newEntry(new byte[][]{(Integer.toString(i) + "-" + Integer.toString(j)).getBytes(), "xxxx".getBytes()}));

37
source/de/anomic/kelondro/kelondroHashtable.java
Unescape View File

@ -142,7 +142,7 @@ public class kelondroHashtable {
private static final byte[] dummyKey = kelondroBase64Order.enhancedCoder.encodeLong(0, 5).getBytes(); private static final byte[] dummyKey = kelondroBase64Order.enhancedCoder.encodeLong(0, 5).getBytes();
public kelondroHashtable(File file, kelondroRow rowdef, int offset, int maxsize, int maxrehash, boolean exitOnFail) { public kelondroHashtable(File file, kelondroRow rowdef, int offset, int maxsize, int maxrehash) throws IOException {
// this creates a new hashtable // this creates a new hashtable
// the key element is not part of the columns array // the key element is not part of the columns array
// this is unlike the kelondroTree, where the key is part of a row // this is unlike the kelondroTree, where the key is part of a row
@ -152,32 +152,25 @@ public class kelondroHashtable {
// this number is needed to omit grow of the table in case of re-hashing // this number is needed to omit grow of the table in case of re-hashing
// the maxsize is re-computed to a virtual folding height and will result in a tablesize // the maxsize is re-computed to a virtual folding height and will result in a tablesize
// less than the given maxsize. The actual maxsize can be retrieved by maxsize() // less than the given maxsize. The actual maxsize can be retrieved by maxsize()
this.hashArray = new kelondroFixedWidthArray(file, extCol(rowdef), 6, exitOnFail); boolean fileExisted = file.exists();
this.offset = offset; this.hashArray = new kelondroFixedWidthArray(file, extCol(rowdef), 6);
this.maxk = kelondroMSetTools.log2a(maxsize); // equal to |log2(maxsize)| + 1 if (fileExisted) {
if (this.maxk >= kelondroMSetTools.log2a(maxsize + power2(offset + 1) + 1) - 1) this.maxk--; this.offset = hashArray.geti(0);
this.maxrehash = maxrehash; this.maxk = hashArray.geti(1);
dummyRow = this.hashArray.row().newEntry(); this.maxrehash = hashArray.geti(2);
dummyRow.setCol(0, dummyKey); } else {
for (int i = 0; i < hashArray.row().columns(); i++) this.offset = offset;
try { this.maxk = kelondroMSetTools.log2a(maxsize); // equal to |log2(maxsize)| + 1
if (this.maxk >= kelondroMSetTools.log2a(maxsize + power2(offset + 1) + 1) - 1) this.maxk--;
this.maxrehash = maxrehash;
dummyRow = this.hashArray.row().newEntry();
dummyRow.setCol(0, dummyKey);
//for (int i = 0; i < hashArray.row().columns(); i++)
hashArray.seti(0, this.offset); hashArray.seti(0, this.offset);
hashArray.seti(1, this.maxk); hashArray.seti(1, this.maxk);
hashArray.seti(2, this.maxrehash); hashArray.seti(2, this.maxrehash);
} catch (IOException e) {
hashArray.logFailure("cannot set properties / " + e.getMessage());
if (exitOnFail) System.exit(-1);
throw new RuntimeException("cannot set properties / " + e.getMessage());
} }
} }
public kelondroHashtable(File file, kelondroRow rowdef) throws IOException{
// this opens a file with an existing hashtable
this.hashArray = new kelondroFixedWidthArray(file, rowdef);
this.offset = hashArray.geti(0);
this.maxk = hashArray.geti(1);
this.maxrehash = hashArray.geti(2);
}
private kelondroRow extCol(kelondroRow rowdef) { private kelondroRow extCol(kelondroRow rowdef) {
kelondroColumn[] newCol = new kelondroColumn[rowdef.columns() + 1]; kelondroColumn[] newCol = new kelondroColumn[rowdef.columns() + 1];

34
source/de/anomic/kelondro/kelondroMapTable.java
Unescape View File

@ -63,51 +63,35 @@ public class kelondroMapTable {
public void declareMaps( public void declareMaps(
String tablename, int keysize, int nodesize, String tablename, int keysize, int nodesize,
char fillChar, boolean exitOnFail) { char fillChar) throws IOException {
declareMaps(tablename, keysize, nodesize, null, null, fillChar, exitOnFail); declareMaps(tablename, keysize, nodesize, null, null, fillChar);
} }
public void declareMaps( public void declareMaps(
String tablename, int keysize, int nodesize, String tablename, int keysize, int nodesize,
String[] sortfields, String[] accfields, char fillChar, boolean exitOnFail) { String[] sortfields, String[] accfields, char fillChar) throws IOException {
declareMaps(tablename, keysize, nodesize, sortfields, accfields, fillChar, 0x800, 0, exitOnFail); declareMaps(tablename, keysize, nodesize, sortfields, accfields, fillChar, 0x800, 0);
} }
public void declareMaps( public void declareMaps(
String tablename, int keysize, int nodesize, String tablename, int keysize, int nodesize,
String[] sortfields, String[] accfields, char fillChar, String[] sortfields, String[] accfields, char fillChar,
long buffersize /*bytes*/, long preloadTime, boolean exitOnFail) { long buffersize /*bytes*/, long preloadTime) throws IOException {
if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared twice."); if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared twice.");
if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared already in other context."); if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared already in other context.");
File tablefile = new File(tablesPath, "table." + tablename + ".mdb"); File tablefile = new File(tablesPath, "table." + tablename + ".mdb");
kelondroDyn dyn; kelondroDyn dyn;
if (tablefile.exists()) try { if (!(tablefile.exists())) tablefile.getParentFile().mkdirs();
dyn = new kelondroDyn(tablefile, buffersize, preloadTime, fillChar); dyn = new kelondroDyn(tablefile, buffersize, preloadTime, keysize, nodesize, fillChar);
} catch (IOException e) {
tablefile.getParentFile().mkdirs();
dyn = new kelondroDyn(tablefile, buffersize, preloadTime, keysize, nodesize, fillChar, exitOnFail);
} else {
tablefile.getParentFile().mkdirs();
dyn = new kelondroDyn(tablefile, buffersize, preloadTime, keysize, nodesize, fillChar, exitOnFail);
}
kelondroMap map = new kelondroMap(dyn, sortfields, accfields); kelondroMap map = new kelondroMap(dyn, sortfields, accfields);
mTables.put(tablename, map); mTables.put(tablename, map);
} }
public void declareTree(String tablename, kelondroRow rowdef, long buffersize /*bytes*/, long preloadTime, boolean exitOnFail) { public void declareTree(String tablename, kelondroRow rowdef, long buffersize /*bytes*/, long preloadTime) {
if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared already in other context."); if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared already in other context.");
if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared twice."); if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared twice.");
File tablefile = new File(tablesPath, "table." + tablename + ".tdb"); File tablefile = new File(tablesPath, "table." + tablename + ".tdb");
kelondroTree Tree; kelondroTree Tree = kelondroTree.open(tablefile, buffersize, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef);
if (tablefile.exists()) try {
Tree = new kelondroTree(tablefile, buffersize, preloadTime, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) {
tablefile.getParentFile().mkdirs();
Tree = new kelondroTree(tablefile, buffersize, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, exitOnFail);
} else {
tablefile.getParentFile().mkdirs();
Tree = new kelondroTree(tablefile, buffersize, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, exitOnFail);
}
tTables.put(tablename, Tree); tTables.put(tablename, Tree);
} }

2
source/de/anomic/kelondro/kelondroOrder.java
Unescape View File

@ -66,4 +66,6 @@ public interface kelondroOrder extends Comparator {
public byte[] zero(); // returns the zero point of the Ordering; null if not defined public byte[] zero(); // returns the zero point of the Ordering; null if not defined
public void rotate(byte[] zero); // defines that the ordering rotates, and sets the zero point for the rotation public void rotate(byte[] zero); // defines that the ordering rotates, and sets the zero point for the rotation
public boolean equals(kelondroOrder o); // used to compare different order objects; they may define the same ordering
} }

43
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -150,6 +150,9 @@ public class kelondroRecords {
// optional logger // optional logger
protected Logger theLogger = null; protected Logger theLogger = null;
// tracking of file cration
protected boolean fileExisted;
// Random. This is used to shift flush-times of write-buffers to differrent time // Random. This is used to shift flush-times of write-buffers to differrent time
private static Random random = new Random(System.currentTimeMillis()); private static Random random = new Random(System.currentTimeMillis());
@ -191,30 +194,34 @@ public class kelondroRecords {
public kelondroRecords(File file, long buffersize /* bytes */, long preloadTime, public kelondroRecords(File file, long buffersize /* bytes */, long preloadTime,
short ohbytec, short ohhandlec, short ohbytec, short ohhandlec,
kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth, kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth) throws IOException {
boolean exitOnFail) { // opens an existing file or creates a new file
// creates a new file
// file: the file that shall be created // file: the file that shall be created
// oha : overhead size array of four bytes: oha[0]=# of bytes, oha[1]=# of shorts, oha[2]=# of ints, oha[3]=# of longs, // oha : overhead size array of four bytes: oha[0]=# of bytes, oha[1]=# of shorts, oha[2]=# of ints, oha[3]=# of longs,
// columns: array with size of column width; columns.length is number of columns // columns: array with size of column width; columns.length is number of columns
// FHandles: number of integer properties // FHandles: number of integer properties
// txtProps: number of text properties // txtProps: number of text properties
assert (!file.exists()) : "file " + file + " already exist"; this.fileExisted = file.exists(); // can be used by extending class to track if this class created the file
try { if (file.exists()) {
// opens an existing tree
this.filename = file.getCanonicalPath();
kelondroRA raf = new kelondroFileRA(this.filename);
//kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100);
//kelondroRA raf = new kelondroCachedRA(new kelondroFileRA(this.filename), 5000000, 1000);
//kelondroRA raf = new kelondroNIOFileRA(this.filename, (file.length() < 4000000), 10000);
initExistingFile(raf, buffersize / 10);
} else {
this.filename = file.getCanonicalPath(); this.filename = file.getCanonicalPath();
kelondroRA raf = new kelondroFileRA(this.filename); kelondroRA raf = new kelondroFileRA(this.filename);
// kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100); // kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100);
// kelondroRA raf = new kelondroNIOFileRA(this.filename, false, 10000); // kelondroRA raf = new kelondroNIOFileRA(this.filename, false, 10000);
initNewFile(raf, ohbytec, ohhandlec, rowdef, FHandles, txtProps, txtPropWidth, buffersize / 10); initNewFile(raf, ohbytec, ohhandlec, rowdef, FHandles, txtProps, txtPropWidth, buffersize / 10);
} catch (IOException e) {
logFailure("cannot create / " + e.getMessage());
if (exitOnFail)
System.exit(-1);
} }
initCache(buffersize / 10 * 9, preloadTime); assignRowdef(rowdef);
initCache(buffersize / 10 * 9, preloadTime);
} }
public kelondroRecords(kelondroRA ra, long buffersize /* bytes */, long preloadTime, public kelondroRecords(kelondroRA ra, long buffersize /* bytes */, long preloadTime,
short ohbytec, short ohhandlec, short ohbytec, short ohhandlec,
kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth, kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth,
@ -347,18 +354,6 @@ public class kelondroRecords {
this.USAGE.write(); this.USAGE.write();
} }
public kelondroRecords(File file, long buffersize, long preloadTime) throws IOException{
// opens an existing tree
assert (file.exists()): "file " + file.getAbsoluteFile().toString() + " does not exist";
this.filename = file.getCanonicalPath();
kelondroRA raf = new kelondroFileRA(this.filename);
//kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100);
//kelondroRA raf = new kelondroCachedRA(new kelondroFileRA(this.filename), 5000000, 1000);
//kelondroRA raf = new kelondroNIOFileRA(this.filename, (file.length() < 4000000), 10000);
initExistingFile(raf, buffersize / 10);
initCache(buffersize / 10 * 9, preloadTime);
}
public kelondroRecords(kelondroRA ra, long buffersize, long preloadTime) throws IOException{ public kelondroRecords(kelondroRA ra, long buffersize, long preloadTime) throws IOException{
this.filename = null; this.filename = null;
initExistingFile(ra, buffersize / 10); initExistingFile(ra, buffersize / 10);
@ -912,7 +907,7 @@ public class kelondroRecords {
return this.ROW; return this.ROW;
} }
public final void assignRowdef(kelondroRow rowdef) { private final void assignRowdef(kelondroRow rowdef) {
// overwrites a given rowdef // overwrites a given rowdef
// the new rowdef must be compatible // the new rowdef must be compatible
if (rowdef.columns() < ROW.columns()) if (rowdef.columns() < ROW.columns())

49
source/de/anomic/kelondro/kelondroSplittedTree.java
Unescape View File

@ -65,64 +65,27 @@ public class kelondroSplittedTree implements kelondroIndex {
return new File(path, filenameStub + "." + ff + "." + co + "." + ns + ".ktc"); return new File(path, filenameStub + "." + ff + "." + co + "." + ns + ".ktc");
} }
/*
private static boolean existsAll(File pathToFiles, String filenameStub, int forkfactor, int columns){ private static boolean existsAll(File pathToFiles, String filenameStub, int forkfactor, int columns){
for (int i = 0; i < forkfactor; i++) { for (int i = 0; i < forkfactor; i++) {
if (!(dbFile(pathToFiles, filenameStub, forkfactor, columns, i)).exists()) return false; if (!(dbFile(pathToFiles, filenameStub, forkfactor, columns, i)).exists()) return false;
} }
return true; return true;
} }
*/
public kelondroSplittedTree(File pathToFiles, String filenameStub, kelondroOrder objectOrder, public kelondroSplittedTree(File pathToFiles, String filenameStub, kelondroOrder objectOrder,
long buffersize, long preloadTime, long buffersize, long preloadTime,
int forkfactor, kelondroRow rowdef, int txtProps, int txtPropsWidth, int forkfactor, kelondroRow rowdef, int txtProps, int txtPropsWidth) {
boolean exitOnFail) {
ktfs = new kelondroTree[forkfactor]; ktfs = new kelondroTree[forkfactor];
File f; File f;
for (int i = 0; i < forkfactor; i++) { for (int i = 0; i < forkfactor; i++) {
f = dbFile(pathToFiles, filenameStub, forkfactor, rowdef.columns(), i); f = dbFile(pathToFiles, filenameStub, forkfactor, rowdef.columns(), i);
if (f.exists()) { ktfs[i] = kelondroTree.open(f, buffersize/forkfactor, preloadTime / forkfactor, kelondroTree.defaultObjectCachePercent,
try { rowdef, objectOrder, txtProps, txtPropsWidth);
ktfs[i] = new kelondroTree(f, buffersize/forkfactor, preloadTime / forkfactor, kelondroTree.defaultObjectCachePercent);
this.order = ktfs[i].order();
} catch (IOException e) {
ktfs[i] = new kelondroTree(f, buffersize/forkfactor, preloadTime / forkfactor, kelondroTree.defaultObjectCachePercent,
rowdef, objectOrder, txtProps, txtPropsWidth, exitOnFail);
this.order = objectOrder;
}
} else {
ktfs[i] = new kelondroTree(f, buffersize/forkfactor, preloadTime / forkfactor, kelondroTree.defaultObjectCachePercent,
rowdef, objectOrder, txtProps, txtPropsWidth, exitOnFail);
this.order = objectOrder;
}
} }
ff = forkfactor;
}
public kelondroSplittedTree(File pathToFiles, String filenameStub, kelondroOrder objectOrder,
long buffersize, long preloadTime, int forkfactor, int columns) throws IOException {
ktfs = new kelondroTree[forkfactor];
for (int i = 0; i < forkfactor; i++) {
ktfs[i] = new kelondroTree(dbFile(pathToFiles, filenameStub, forkfactor, columns, i),
buffersize/forkfactor, preloadTime / forkfactor, kelondroTree.defaultObjectCachePercent);
}
ff = forkfactor;
this.order = objectOrder; this.order = objectOrder;
} ff = forkfactor;
public static kelondroSplittedTree open(File pathToFiles, String filenameStub, kelondroOrder objectOrder,
long buffersize, long preloadTime,
int forkfactor, kelondroRow rowdef, int txtProps, int txtPropsWidth,
boolean exitOnFail) throws IOException {
// generated a new splittet tree if it not exists or
// opens an existing one
if (existsAll(pathToFiles, filenameStub, forkfactor, rowdef.columns())) {
return new kelondroSplittedTree(pathToFiles, filenameStub, objectOrder, buffersize, preloadTime, forkfactor, rowdef.columns());
} else {
return new kelondroSplittedTree(pathToFiles, filenameStub, objectOrder,
buffersize, preloadTime,
forkfactor, rowdef, txtProps, txtPropsWidth,
exitOnFail);
}
} }
public void close() throws IOException { public void close() throws IOException {

60
source/de/anomic/kelondro/kelondroStack.java
Unescape View File

@ -67,31 +67,33 @@ public final class kelondroStack extends kelondroRecords {
protected static final int root = 0; // pointer for FHandles-array: pointer to root node protected static final int root = 0; // pointer for FHandles-array: pointer to root node
protected static final int toor = 1; // pointer for FHandles-array: pointer to root node protected static final int toor = 1; // pointer for FHandles-array: pointer to root node
public kelondroStack(File file, kelondroRow rowdef, boolean exitOnFail) { public kelondroStack(File file, kelondroRow rowdef) throws IOException {
// this creates a new stack // this creates a new stack
super(file, 0, 0, thisOHBytes, thisOHHandles, rowdef, thisFHandles, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail); super(file, 0, 0, thisOHBytes, thisOHHandles, rowdef, thisFHandles, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */);
try { if (super.fileExisted) {
if ((getHandle(root) == null) && (getHandle(toor) == null)) clear();
} else {
setHandle(root, null); // define the root value setHandle(root, null); // define the root value
setHandle(toor, null); // define the toor value setHandle(toor, null); // define the toor value
} catch (IOException e) {
super.logFailure("cannot set root/toor handles / " + e.getMessage());
if (exitOnFail) System.exit(-1);
throw new RuntimeException("cannot set root/toor handles / " + e.getMessage());
} }
} }
public kelondroStack(File file) throws IOException{ public static final kelondroStack open(File file, kelondroRow rowdef) {
// this opens a file with an existing stack try {
super(file, 0, 0); return new kelondroStack(file, rowdef);
if ((getHandle(root) == null) && (getHandle(toor) == null)) clear(); } catch (IOException e) {
} file.delete();
try {
public void clear() throws IOException { return new kelondroStack(file, rowdef);
super.clear(); } catch (IOException ee) {
setHandle(root, null); // reset the root value System.out.println("kelondroStack: cannot open or create file " + file.toString());
setHandle(toor, null); // reset the toor value e.printStackTrace();
ee.printStackTrace();
return null;
}
}
} }
public static kelondroStack reset(kelondroStack stack) { public static kelondroStack reset(kelondroStack stack) {
// memorize settings to this file // memorize settings to this file
File f = new File(stack.filename); File f = new File(stack.filename);
@ -102,9 +104,15 @@ public final class kelondroStack extends kelondroRecords {
if (f.exists()) f.delete(); if (f.exists()) f.delete();
// re-open a database with same settings as before // re-open a database with same settings as before
return new kelondroStack(f, row, true); return open(f, row);
} }
public void clear() throws IOException {
super.clear();
setHandle(root, null); // reset the root value
setHandle(toor, null); // reset the toor value
}
public class Counter implements Iterator { public class Counter implements Iterator {
Handle nextHandle = null; Handle nextHandle = null;
@ -356,18 +364,19 @@ public final class kelondroStack extends kelondroRecords {
for (int i = 0; i < args.length; i++) System.out.print(args[i] + " "); for (int i = 0; i < args.length; i++) System.out.print(args[i] + " ");
System.out.println(""); System.out.println("");
byte[] ret = null; byte[] ret = null;
kelondroRow lens = new kelondroRow("byte[] key-" + Integer.parseInt(args[1]) + ", byte[] value-" + Integer.parseInt(args[2]));
try { try {
if ((args.length > 4) || (args.length < 2)) { if ((args.length > 4) || (args.length < 2)) {
System.err.println("usage: kelondroStack -c|-p|-v|-g|-i|-s [file]|[key [value]] <db-file>"); System.err.println("usage: kelondroStack -c|-p|-v|-g|-i|-s [file]|[key [value]] <db-file>");
System.err.println("( create, push, view, (g)pop, imp, shell)"); System.err.println("( create, push, view, (g)pop, imp, shell)");
System.exit(0); System.exit(0);
} else if (args.length == 2) { } else if (args.length == 2) {
kelondroStack fm = new kelondroStack(new File(args[1])); kelondroStack fm = new kelondroStack(new File(args[1]), lens);
if (args[0].equals("-v")) { if (args[0].equals("-v")) {
fm.print(); fm.print();
ret = null; ret = null;
} else if (args[0].equals("-g")) { } else if (args[0].equals("-g")) {
fm = new kelondroStack(new File(args[1])); fm = new kelondroStack(new File(args[1]), lens);
kelondroRow.Entry ret2 = fm.pop(); kelondroRow.Entry ret2 = fm.pop();
ret = ((ret2 == null) ? null : ret2.getColBytes(1)); ret = ((ret2 == null) ? null : ret2.getColBytes(1));
fm.close(); fm.close();
@ -375,7 +384,7 @@ public final class kelondroStack extends kelondroRecords {
fm.close(); fm.close();
} else if (args.length == 3) { } else if (args.length == 3) {
if (args[0].equals("-i")) { if (args[0].equals("-i")) {
kelondroStack fm = new kelondroStack(new File(args[2])); kelondroStack fm = new kelondroStack(new File(args[2]), lens);
int i = fm.imp(new File(args[1]),";"); int i = fm.imp(new File(args[1]),";");
fm.close(); fm.close();
ret = (i + " records imported").getBytes(); ret = (i + " records imported").getBytes();
@ -399,7 +408,7 @@ public final class kelondroStack extends kelondroRecords {
if (f != null) try {f.close();}catch(Exception e) {} if (f != null) try {f.close();}catch(Exception e) {}
} }
} else if (args[0].equals("-g")) { } else if (args[0].equals("-g")) {
kelondroStack fm = new kelondroStack(new File(args[2])); kelondroStack fm = new kelondroStack(new File(args[2]), lens);
kelondroRow.Entry ret2 = fm.pop(Integer.parseInt(args[1])); kelondroRow.Entry ret2 = fm.pop(Integer.parseInt(args[1]));
ret = ((ret2 == null) ? null : ret2.getColBytes(1)); ret = ((ret2 == null) ? null : ret2.getColBytes(1));
fm.close(); fm.close();
@ -409,11 +418,10 @@ public final class kelondroStack extends kelondroRecords {
// create <keylen> <valuelen> <filename> // create <keylen> <valuelen> <filename>
File f = new File(args[3]); File f = new File(args[3]);
if (f.exists()) f.delete(); if (f.exists()) f.delete();
kelondroRow lens = new kelondroRow("byte[] key-" + Integer.parseInt(args[1]) + ", byte[] value-" + Integer.parseInt(args[2])); kelondroStack fm = new kelondroStack(f, lens);
kelondroStack fm = new kelondroStack(f, lens, true);
fm.close(); fm.close();
} else if (args[0].equals("-p")) { } else if (args[0].equals("-p")) {
kelondroStack fm = new kelondroStack(new File(args[3])); kelondroStack fm = new kelondroStack(new File(args[3]), lens);
fm.push(fm.row().newEntry(new byte[][] {args[1].getBytes(), args[2].getBytes()})); fm.push(fm.row().newEntry(new byte[][] {args[1].getBytes(), args[2].getBytes()}));
fm.close(); fm.close();
} }

91
source/de/anomic/kelondro/kelondroTree.java
Unescape View File

@ -91,30 +91,55 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
protected long lastIteratorCount = readAheadChunkSize; protected long lastIteratorCount = readAheadChunkSize;
private kelondroObjectCache objectCache; private kelondroObjectCache objectCache;
public kelondroTree(File file, long buffersize, long preloadTime, int objectCachePercent, kelondroRow rowdef, boolean exitOnFail) { public kelondroTree(File file, long buffersize, long preloadTime, int objectCachePercent, kelondroRow rowdef) throws IOException {
// this creates a new tree file this(file, buffersize, preloadTime, objectCachePercent, rowdef, new kelondroNaturalOrder(true), rowdef.columns() /* txtProps */, 80 /* txtPropWidth */);
this(file, buffersize, preloadTime, objectCachePercent, rowdef, new kelondroNaturalOrder(true), rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
} }
public kelondroTree(File file, long buffersize, long preloadTime, int objectCachePercent, kelondroRow rowdef, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) { public kelondroTree(File file, long buffersize, long preloadTime, int objectCachePercent, kelondroRow rowdef,
// this creates a new tree file kelondroOrder objectOrder, int txtProps, int txtPropsWidth) throws IOException {
// opens an existing tree file or creates a new tree file
super(file, super(file,
(100 - objectCachePercent) * buffersize / 100, preloadTime, (100 - objectCachePercent) * buffersize / 100, preloadTime,
thisOHBytes, thisOHHandles, rowdef, thisOHBytes, thisOHHandles, rowdef,
thisFHandles, txtProps, txtPropsWidth, exitOnFail); thisFHandles, txtProps, txtPropsWidth);
try {
super.setLogger(log);
if (super.fileExisted) {
this.objectOrder = objectOrder;
kelondroOrder oldOrder = readOrderType();
if ((oldOrder != null) && (!(oldOrder.equals(objectOrder)))) throw new IOException("wrong object order upon initialization. new order is " + objectOrder.toString() + ", old order was " + oldOrder.toString());
} else {
// create new file structure
setHandle(root, null); // define the root value setHandle(root, null); // define the root value
} catch (IOException e) { this.objectOrder = objectOrder;
super.logFailure("cannot set root handle / " + e.getMessage()); writeOrderType();
if (exitOnFail) System.exit(-1);
throw new RuntimeException("cannot set root handle / " + e.getMessage());
} }
this.objectOrder = objectOrder;
writeOrderType();
super.setLogger(log);
initObjectCache(buffersize, objectCachePercent); initObjectCache(buffersize, objectCachePercent);
} }
public static final kelondroTree open(File file, long buffersize, long preloadTime, int objectCachePercent, kelondroRow rowdef) {
return open(file, buffersize, preloadTime, objectCachePercent, rowdef, new kelondroNaturalOrder(true), rowdef.columns() /* txtProps */, 80 /* txtPropWidth */);
}
public static final kelondroTree open(File file, long buffersize, long preloadTime, int objectCachePercent, kelondroRow rowdef,
kelondroOrder objectOrder, int txtProps, int txtPropsWidth) {
// opens new or existing file; in case that any error occur the file is deleted again and it is tried to create the file again
// if that fails, the method returns null
try {
return new kelondroTree(file, buffersize, preloadTime, objectCachePercent, rowdef, objectOrder, txtProps, txtPropsWidth);
} catch (IOException e) {
file.delete();
try {
return new kelondroTree(file, buffersize, preloadTime, objectCachePercent, rowdef, objectOrder, txtProps, txtPropsWidth);
} catch (IOException ee) {
log.severe("cannot open or create file " + file.toString());
e.printStackTrace();
ee.printStackTrace();
return null;
}
}
}
public kelondroTree(kelondroRA ra, long buffersize, long preloadTime, int objectCachePercent, kelondroRow rowdef, boolean exitOnFail) { public kelondroTree(kelondroRA ra, long buffersize, long preloadTime, int objectCachePercent, kelondroRow rowdef, boolean exitOnFail) {
// this creates a new tree within a kelondroRA // this creates a new tree within a kelondroRA
this(ra, buffersize, preloadTime, objectCachePercent, rowdef, new kelondroNaturalOrder(true), rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail); this(ra, buffersize, preloadTime, objectCachePercent, rowdef, new kelondroNaturalOrder(true), rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
@ -139,14 +164,6 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
initObjectCache(buffersize, objectCachePercent); initObjectCache(buffersize, objectCachePercent);
} }
public kelondroTree(File file, long buffersize, long preloadTime, int objectCachePercent) throws IOException {
// this opens a file with an existing tree file
super(file, (100 - objectCachePercent) * buffersize / 100, preloadTime);
readOrderType();
super.setLogger(log);
initObjectCache(buffersize, objectCachePercent);
}
public kelondroTree(kelondroRA ra, long buffersize, long preloadTime, int objectCachePercent) throws IOException { public kelondroTree(kelondroRA ra, long buffersize, long preloadTime, int objectCachePercent) throws IOException {
// this opens a file with an existing tree in a kelondroRA // this opens a file with an existing tree in a kelondroRA
super(ra, (100 - objectCachePercent) * buffersize / 100, preloadTime); super(ra, (100 - objectCachePercent) * buffersize / 100, preloadTime);
@ -183,12 +200,14 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
} catch (IOException e) {} } catch (IOException e) {}
} }
private void readOrderType() { private kelondroOrder readOrderType() {
try { try {
byte[] d = super.getDescription(); byte[] d = super.getDescription();
String s = new String(d).substring(0, 2); String s = new String(d).substring(0, 2);
this.objectOrder = orderBySignature(s); return orderBySignature(s);
} catch (IOException e) {} } catch (IOException e) {
return null;
}
} }
public static kelondroOrder orderBySignature(String signature) { public static kelondroOrder orderBySignature(String signature) {
@ -1238,7 +1257,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
// test script // test script
File testFile = new File("test.db"); File testFile = new File("test.db");
while (testFile.exists()) testFile.delete(); while (testFile.exists()) testFile.delete();
kelondroTree fm = new kelondroTree(testFile, 0x100000, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"), true); kelondroTree fm = new kelondroTree(testFile, 0x100000, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"));
byte[] dummy = "".getBytes(); byte[] dummy = "".getBytes();
fm.put("abc0".getBytes(), dummy); fm.put("bcd0".getBytes(), dummy); fm.put("abc0".getBytes(), dummy); fm.put("bcd0".getBytes(), dummy);
fm.put("def0".getBytes(), dummy); fm.put("bab0".getBytes(), dummy); fm.put("def0".getBytes(), dummy); fm.put("bab0".getBytes(), dummy);
@ -1260,7 +1279,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
ret = null; ret = null;
} }
} else if (args.length == 2) { } else if (args.length == 2) {
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10); kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"));
if (args[0].equals("-v")) { if (args[0].equals("-v")) {
fm.print(); fm.print();
ret = null; ret = null;
@ -1268,11 +1287,11 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
fm.close(); fm.close();
} else if (args.length == 3) { } else if (args.length == 3) {
if (args[0].equals("-d")) { if (args[0].equals("-d")) {
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10); kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"));
fm.remove(args[2].getBytes()); fm.remove(args[2].getBytes());
fm.close(); fm.close();
} else if (args[0].equals("-i")) { } else if (args[0].equals("-i")) {
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10); kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"));
int i = fm.imp(new File(args[1]),";"); int i = fm.imp(new File(args[1]),";");
fm.close(); fm.close();
ret = (i + " records imported").getBytes(); ret = (i + " records imported").getBytes();
@ -1295,12 +1314,12 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
if (f != null) try {f.close();}catch(Exception e){} if (f != null) try {f.close();}catch(Exception e){}
} }
} else if (args[0].equals("-g")) { } else if (args[0].equals("-g")) {
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10); kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"));
kelondroRow.Entry ret2 = fm.get(args[2].getBytes()); kelondroRow.Entry ret2 = fm.get(args[2].getBytes());
ret = ((ret2 == null) ? null : ret2.getColBytes(1)); ret = ((ret2 == null) ? null : ret2.getColBytes(1));
fm.close(); fm.close();
} else if (args[0].equals("-n")) { } else if (args[0].equals("-n")) {
kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10); kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"));
//byte[][] keys = fm.getSequentialKeys(args[2].getBytes(), 500, true); //byte[][] keys = fm.getSequentialKeys(args[2].getBytes(), 500, true);
Iterator rowIt = fm.rows(true, false, (args[2].length() == 0) ? null : args[2].getBytes()); Iterator rowIt = fm.rows(true, false, (args[2].length() == 0) ? null : args[2].getBytes());
Vector v = new Vector(); Vector v = new Vector();
@ -1314,10 +1333,10 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
File f = new File(args[3]); File f = new File(args[3]);
if (f.exists()) f.delete(); if (f.exists()) f.delete();
kelondroRow lens = new kelondroRow("byte[] key-" + Integer.parseInt(args[1]) + ", byte[] value-" + Integer.parseInt(args[2])); kelondroRow lens = new kelondroRow("byte[] key-" + Integer.parseInt(args[1]) + ", byte[] value-" + Integer.parseInt(args[2]));
kelondroTree fm = new kelondroTree(f, 0x100000, 0, 10, lens, true); kelondroTree fm = new kelondroTree(f, 0x100000, 0, 10, lens);
fm.close(); fm.close();
} else if (args[0].equals("-u")) { } else if (args[0].equals("-u")) {
kelondroTree fm = new kelondroTree(new File(args[3]), 0x100000, 0, 10); kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"));
ret = fm.put(args[1].getBytes(), args[2].getBytes()); ret = fm.put(args[1].getBytes(), args[2].getBytes());
fm.close(); fm.close();
} }
@ -1380,7 +1399,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
int steps = 0; int steps = 0;
while (true) { while (true) {
if (testFile.exists()) testFile.delete(); if (testFile.exists()) testFile.delete();
tt = new kelondroTree(testFile, 200, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"), true); tt = new kelondroTree(testFile, 200, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"));
steps = 10 + ((int) System.currentTimeMillis() % 7) * (((int) System.currentTimeMillis() + 17) % 11); steps = 10 + ((int) System.currentTimeMillis() % 7) * (((int) System.currentTimeMillis() + 17) % 11);
t = s; t = s;
d = ""; d = "";
@ -1446,7 +1465,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
File f = new File("test.db"); File f = new File("test.db");
if (f.exists()) f.delete(); if (f.exists()) f.delete();
try { try {
kelondroTree tt = new kelondroTree(f, 1000, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"), true); kelondroTree tt = new kelondroTree(f, 1000, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"));
byte[] b; byte[] b;
b = testWord('B'); tt.put(b, b); //tt.print(); b = testWord('B'); tt.put(b, b); //tt.print();
b = testWord('C'); tt.put(b, b); //tt.print(); b = testWord('C'); tt.put(b, b); //tt.print();
@ -1511,7 +1530,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
public static kelondroTree testTree(File f, String testentities) throws IOException { public static kelondroTree testTree(File f, String testentities) throws IOException {
if (f.exists()) f.delete(); if (f.exists()) f.delete();
kelondroTree tt = new kelondroTree(f, 0, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"), true); kelondroTree tt = new kelondroTree(f, 0, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4"));
byte[] b; byte[] b;
for (int i = 0; i < testentities.length(); i++) { for (int i = 0; i < testentities.length(); i++) {
b = testWord(testentities.charAt(i)); b = testWord(testentities.charAt(i));

10
source/de/anomic/plasma/plasmaCrawlBalancer.java
Unescape View File

@ -59,15 +59,7 @@ public class plasmaCrawlBalancer {
private HashMap domainStacks; private HashMap domainStacks;
public plasmaCrawlBalancer(File stackFile) { public plasmaCrawlBalancer(File stackFile) {
if (stackFile.exists()) { stack = kelondroStack.open(stackFile, new kelondroRow("byte[] urlhash-" + indexURL.urlHashLength));
try {
stack = new kelondroStack(stackFile);
} catch (IOException e) {
stack = new kelondroStack(stackFile, new kelondroRow("byte[] urlhash-" + indexURL.urlHashLength), true);
}
} else {
stack = new kelondroStack(stackFile, new kelondroRow("byte[] urlhash-" + indexURL.urlHashLength), true);
}
domainStacks = new HashMap(); domainStacks = new HashMap();
} }

27
source/de/anomic/plasma/plasmaCrawlEURL.java
Unescape View File

@ -57,8 +57,8 @@ import java.util.Iterator;
import de.anomic.index.indexURL; import de.anomic.index.indexURL;
import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroFlexTable;
import de.anomic.kelondro.kelondroRow; import de.anomic.kelondro.kelondroRow;
import de.anomic.kelondro.kelondroTree;
import de.anomic.tools.bitfield; import de.anomic.tools.bitfield;
public class plasmaCrawlEURL extends indexURL { public class plasmaCrawlEURL extends indexURL {
@ -138,19 +138,32 @@ public class plasmaCrawlEURL extends indexURL {
"String failcause-" + urlErrorLength + ", " + // string describing load failure "String failcause-" + urlErrorLength + ", " + // string describing load failure
"byte[] flags-" + urlFlagLength); // extra space "byte[] flags-" + urlFlagLength); // extra space
if (cachePath.exists()) try {
String newCacheName = "urlErr3.table";
cachePath.mkdirs();
try {
urlHashCache = new kelondroFlexTable(cachePath, newCacheName, kelondroBase64Order.enhancedCoder, bufferkb * 0x400, preloadTime, rowdef);
} catch (IOException e) {
e.printStackTrace();
System.exit(-1);
}
/*
File oldCacheFile = new File(cachePath, "urlErr0.db");
if (oldCacheFile.exists()) try {
// open existing cache // open existing cache
kelondroTree tree = new kelondroTree(cachePath, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent); kelondroTree tree = new kelondroTree(oldCacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent);
tree.assignRowdef(rowdef); tree.assignRowdef(rowdef);
urlHashCache = tree; urlHashCache = tree;
} catch (IOException e) { } catch (IOException e) {
cachePath.delete(); oldCacheFile.delete();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, true); urlHashCache = new kelondroTree(oldCacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, true);
} else { } else {
// create new cache // create new cache
cachePath.getParentFile().mkdirs(); oldCacheFile.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, true); urlHashCache = new kelondroTree(oldCacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, true);
} }
*/
} }
public synchronized Entry newEntry(URL url, String referrer, String initiator, String executor, public synchronized Entry newEntry(URL url, String referrer, String initiator, String executor,

22
source/de/anomic/plasma/plasmaCrawlLURL.java
Unescape View File

@ -113,20 +113,14 @@ public final class plasmaCrawlLURL extends indexURL {
"Cardinal size-" + urlSizeLength + " {b64e}, " + // size of file in bytes "Cardinal size-" + urlSizeLength + " {b64e}, " + // size of file in bytes
"Cardinal wc-" + urlWordCountLength + " {b64e}"); // word count "Cardinal wc-" + urlWordCountLength + " {b64e}"); // word count
if (cachePath.exists()) { File cacheFile = new File(cachePath, "urlHash.db");
// open existing cache
try { cacheFile.getParentFile().mkdirs();
kelondroTree tree = new kelondroTree(cachePath, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent); try {
tree.assignRowdef(rowdef); urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef);
urlHashCache = tree; } catch (IOException e) {
} catch (IOException e) { e.printStackTrace();
cachePath.getParentFile().mkdirs(); System.exit(-1);
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, true);
}
} else {
// create new cache
cachePath.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, true);
} }
// init result stacks // init result stacks

67
source/de/anomic/plasma/plasmaCrawlNURL.java
Unescape View File

@ -55,9 +55,9 @@ import java.util.Iterator;
import de.anomic.index.indexURL; import de.anomic.index.indexURL;
import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroFlexTable;
import de.anomic.kelondro.kelondroRecords; import de.anomic.kelondro.kelondroRecords;
import de.anomic.kelondro.kelondroStack; import de.anomic.kelondro.kelondroStack;
import de.anomic.kelondro.kelondroTree;
import de.anomic.kelondro.kelondroRow; import de.anomic.kelondro.kelondroRow;
import de.anomic.server.logging.serverLog; import de.anomic.server.logging.serverLog;
import de.anomic.tools.bitfield; import de.anomic.tools.bitfield;
@ -104,50 +104,32 @@ public class plasmaCrawlNURL extends indexURL {
private long preloadTime; private long preloadTime;
initStackIndex initThead; initStackIndex initThead;
public plasmaCrawlNURL(File cacheStacksPath, int bufferkb, long preloadTime) { public plasmaCrawlNURL(File cachePath, int bufferkb, long preloadTime) {
super(); super();
this.cacheStacksPath = cacheStacksPath; this.cacheStacksPath = cachePath;
this.bufferkb = bufferkb; this.bufferkb = bufferkb;
this.preloadTime = preloadTime; this.preloadTime = preloadTime;
// create a stack for newly entered entries // create a stack for newly entered entries
if (!(cacheStacksPath.exists())) cacheStacksPath.mkdir(); // make the path if (!(cachePath.exists())) cachePath.mkdir(); // make the path
openHashCache(); openHashCache();
File coreStackFile = new File(cacheStacksPath, "urlNoticeLocal0.stack"); File coreStackFile = new File(cachePath, "urlNoticeLocal0.stack");
File limitStackFile = new File(cacheStacksPath, "urlNoticeLimit0.stack"); File limitStackFile = new File(cachePath, "urlNoticeLimit0.stack");
File overhangStackFile = new File(cacheStacksPath, "urlNoticeOverhang0.stack"); File overhangStackFile = new File(cachePath, "urlNoticeOverhang0.stack");
File remoteStackFile = new File(cacheStacksPath, "urlNoticeRemote0.stack"); File remoteStackFile = new File(cachePath, "urlNoticeRemote0.stack");
File imageStackFile = new File(cacheStacksPath, "urlNoticeImage0.stack"); File imageStackFile = new File(cachePath, "urlNoticeImage0.stack");
File movieStackFile = new File(cacheStacksPath, "urlNoticeMovie0.stack"); File movieStackFile = new File(cachePath, "urlNoticeMovie0.stack");
File musicStackFile = new File(cacheStacksPath, "urlNoticeMusic0.stack"); File musicStackFile = new File(cachePath, "urlNoticeMusic0.stack");
coreStack = new plasmaCrawlBalancer(coreStackFile); coreStack = new plasmaCrawlBalancer(coreStackFile);
limitStack = new plasmaCrawlBalancer(limitStackFile); limitStack = new plasmaCrawlBalancer(limitStackFile);
overhangStack = new plasmaCrawlBalancer(overhangStackFile); overhangStack = new plasmaCrawlBalancer(overhangStackFile);
remoteStack = new plasmaCrawlBalancer(remoteStackFile); remoteStack = new plasmaCrawlBalancer(remoteStackFile);
kelondroRow rowdef = new kelondroRow("byte[] urlhash-" + indexURL.urlHashLength); kelondroRow rowdef = new kelondroRow("byte[] urlhash-" + indexURL.urlHashLength);
if (imageStackFile.exists()) try { imageStack = kelondroStack.open(imageStackFile, rowdef);
imageStack = new kelondroStack(imageStackFile); movieStack = kelondroStack.open(movieStackFile, rowdef);
} catch (IOException e) { musicStack = kelondroStack.open(musicStackFile, rowdef);
imageStack = new kelondroStack(imageStackFile, rowdef, true);
} else {
imageStack = new kelondroStack(imageStackFile, rowdef, true);
}
if (movieStackFile.exists()) try {
movieStack = new kelondroStack(movieStackFile);
} catch (IOException e) {
movieStack = new kelondroStack(movieStackFile, rowdef, true);
} else {
movieStack = new kelondroStack(movieStackFile, rowdef, true);
}
if (musicStackFile.exists()) try {
musicStack = new kelondroStack(musicStackFile);
} catch (IOException e) {
musicStack = new kelondroStack(musicStackFile, rowdef, true);
} else {
musicStack = new kelondroStack(musicStackFile, rowdef, true);
}
// init stack Index // init stack Index
stackIndex = new HashSet(); stackIndex = new HashSet();
@ -165,15 +147,17 @@ public class plasmaCrawlNURL extends indexURL {
} }
private void openHashCache() { private void openHashCache() {
File oldCacheFile = new File(cacheStacksPath, "urlNotice1.db"); String newCacheName = "urlNotice3.table";
//File newCacheFile = new File(cacheStacksPath, "urlNotice2.table"); cacheStacksPath.mkdirs();
/*if (newCacheFile.exists()) try { try {
urlHashCache = new kelondroFlexTable(cacheStacksPath, "urlNotice2.table", kelondroBase64Order.enhancedCoder, bufferkb * 0x400, preloadTime, rowdef, true); urlHashCache = new kelondroFlexTable(cacheStacksPath, newCacheName, kelondroBase64Order.enhancedCoder, bufferkb * 0x400, preloadTime, rowdef);
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
oldCacheFile.delete(); System.exit(-1);
urlHashCache = new kelondroTree(oldCacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, true); }
} else*/ if (oldCacheFile.exists()) try { /*
File oldCacheFile = new File(cacheStacksPath, "urlNotice1.db");
if (oldCacheFile.exists()) try {
// open existing cache // open existing cache
kelondroTree tree = new kelondroTree(oldCacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent); kelondroTree tree = new kelondroTree(oldCacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent);
tree.assignRowdef(rowdef); tree.assignRowdef(rowdef);
@ -184,9 +168,9 @@ public class plasmaCrawlNURL extends indexURL {
} else { } else {
// create new cache // create new cache
oldCacheFile.getParentFile().mkdirs(); oldCacheFile.getParentFile().mkdirs();
//urlHashCache = new kelondroFlexTable(cacheStacksPath, "urlNotice2.table", kelondroBase64Order.enhancedCoder, bufferkb * 0x400, preloadTime, rowdef, true);
urlHashCache = new kelondroTree(oldCacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, true); urlHashCache = new kelondroTree(oldCacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, true);
} }
*/
} }
private void resetHashCache() { private void resetHashCache() {
@ -200,6 +184,7 @@ public class plasmaCrawlNURL extends indexURL {
} }
public void close() { public void close() {
try {urlHashCache.close();} catch (IOException e) {}
coreStack.close(); coreStack.close();
limitStack.close(); limitStack.close();
overhangStack.close(); overhangStack.close();
@ -552,6 +537,8 @@ public class plasmaCrawlNURL extends indexURL {
this.flags.getBytes(), this.flags.getBytes(),
normalizeHandle(this.handle).getBytes() normalizeHandle(this.handle).getBytes()
}; };
if (urlHashCache == null) System.out.println("urlHashCache is NULL");
if ((urlHashCache != null) && (urlHashCache.row() == null)) System.out.println("row() is NULL");
urlHashCache.put(urlHashCache.row().newEntry(entry)); urlHashCache.put(urlHashCache.row().newEntry(entry));
this.stored = true; this.stored = true;
} catch (IOException e) { } catch (IOException e) {

15
source/de/anomic/plasma/plasmaCrawlProfile.java
Unescape View File

@ -67,16 +67,8 @@ public class plasmaCrawlProfile {
this.profileTableFile = file; this.profileTableFile = file;
this.bufferkb = bufferkb; this.bufferkb = bufferkb;
this.preloadTime = preloadTime; this.preloadTime = preloadTime;
kelondroDyn dyn = null; profileTableFile.getParentFile().mkdirs();
if (profileTableFile.exists()) try { kelondroDyn dyn = kelondroDyn.open(profileTableFile, bufferkb * 1024, preloadTime, indexURL.urlCrawlProfileHandleLength, 2000, '#');
dyn = new kelondroDyn(file, bufferkb * 1024, preloadTime, '#');
} catch (IOException e) {
profileTableFile.delete();
dyn = new kelondroDyn(file, bufferkb * 1024, preloadTime, indexURL.urlCrawlProfileHandleLength, 2000, '#', true);
} else {
profileTableFile.getParentFile().mkdirs();
dyn = new kelondroDyn(file, bufferkb * 1024, preloadTime, indexURL.urlCrawlProfileHandleLength, 2000, '#', true);
}
profileTable = new kelondroMap(dyn); profileTable = new kelondroMap(dyn);
domsCache = new HashMap(); domsCache = new HashMap();
} }
@ -102,7 +94,8 @@ public class plasmaCrawlProfile {
if (profileTable != null) try { profileTable.close(); } catch (IOException e) {} if (profileTable != null) try { profileTable.close(); } catch (IOException e) {}
if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database"); if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database");
profileTableFile.getParentFile().mkdirs(); profileTableFile.getParentFile().mkdirs();
profileTable = new kelondroMap(new kelondroDyn(profileTableFile, bufferkb * 1024, preloadTime, indexURL.urlCrawlProfileHandleLength, 2000, '#', true)); kelondroDyn dyn = kelondroDyn.open(profileTableFile, bufferkb * 1024, preloadTime, indexURL.urlCrawlProfileHandleLength, 2000, '#');
profileTable = new kelondroMap(dyn);
} }
public void close() { public void close() {

18
source/de/anomic/plasma/plasmaCrawlRobotsTxt.java
Unescape View File

@ -71,20 +71,8 @@ public class plasmaCrawlRobotsTxt {
this.robotsTableFile = robotsTableFile; this.robotsTableFile = robotsTableFile;
this.bufferkb = bufferkb; this.bufferkb = bufferkb;
this.preloadTime = preloadTime; this.preloadTime = preloadTime;
if (robotsTableFile.exists()) { robotsTableFile.getParentFile().mkdirs();
try { robotsTable = new kelondroMap(kelondroDyn.open(robotsTableFile, bufferkb * 1024, preloadTime, 256, 512, '_'));
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, preloadTime, '_'));
} catch (kelondroException e) {
robotsTableFile.delete();
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, preloadTime, 256, 512, '_', true));
} catch (IOException e) {
robotsTableFile.delete();
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, preloadTime, 256, 512, '_', true));
}
} else {
robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, bufferkb * 1024, preloadTime, 256, 512, '_', true));
}
} }
public int cacheNodeChunkSize() { public int cacheNodeChunkSize() {
@ -110,7 +98,7 @@ public class plasmaCrawlRobotsTxt {
} catch (IOException e) {} } catch (IOException e) {}
if (!(robotsTableFile.delete())) throw new RuntimeException("cannot delete robots.txt database"); if (!(robotsTableFile.delete())) throw new RuntimeException("cannot delete robots.txt database");
robotsTableFile.getParentFile().mkdirs(); robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, this.bufferkb, preloadTime, 256, 512, '_', true)); robotsTable = new kelondroMap(kelondroDyn.open(robotsTableFile, this.bufferkb, preloadTime, 256, 512, '_'));
} }
public void close() { public void close() {

93
source/de/anomic/plasma/plasmaCrawlStacker.java
Unescape View File

@ -202,7 +202,7 @@ public final class plasmaCrawlStacker {
Date loadDate, Date loadDate,
int currentdepth, int currentdepth,
plasmaCrawlProfile.entry profile) { plasmaCrawlProfile.entry profile) {
try { if (profile != null) try {
this.queue.addMessage(new stackCrawlMessage( this.queue.addMessage(new stackCrawlMessage(
initiatorHash, initiatorHash,
nexturlString, nexturlString,
@ -579,6 +579,9 @@ public final class plasmaCrawlStacker {
private final serverSemaphore writeSync; private final serverSemaphore writeSync;
private final LinkedList urlEntryHashCache; private final LinkedList urlEntryHashCache;
private kelondroTree urlEntryCache; private kelondroTree urlEntryCache;
private File cacheStacksPath;
private int bufferkb;
private long preloadTime;
public stackCrawlQueue(File cacheStacksPath, int bufferkb, long preloadTime) { public stackCrawlQueue(File cacheStacksPath, int bufferkb, long preloadTime) {
// init the read semaphore // init the read semaphore
@ -591,54 +594,54 @@ public final class plasmaCrawlStacker {
this.urlEntryHashCache = new LinkedList(); this.urlEntryHashCache = new LinkedList();
// create a stack for newly entered entries // create a stack for newly entered entries
if (!(cacheStacksPath.exists())) cacheStacksPath.mkdir(); // make the path this.cacheStacksPath = cacheStacksPath;
this.bufferkb = bufferkb;
this.preloadTime = preloadTime;
File cacheFile = new File(cacheStacksPath, "urlPreNotice.db"); openDB();
if (cacheFile.exists()) { try {
// open existing cache // loop through the list and fill the messageList with url hashs
try { Iterator rows = this.urlEntryCache.rows(true, false, null);
kelondroTree tree = new kelondroTree(cacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent); kelondroRow.Entry entry;
tree.assignRowdef(plasmaCrawlNURL.rowdef); while (rows.hasNext()) {
this.urlEntryCache = tree; entry = (kelondroRow.Entry) rows.next();
} catch (IOException e) { if (entry == null) {
cacheFile.delete(); System.out.println("ERROR! null element found");
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.rowdef, true); continue;
}
try {
// loop through the list and fill the messageList with url hashs
Iterator rows = this.urlEntryCache.rows(true, false, null);
kelondroRow.Entry entry;
while (rows.hasNext()) {
entry = (kelondroRow.Entry) rows.next();
if (entry == null) {
System.out.println("ERROR! null element found");
continue;
}
this.urlEntryHashCache.add(entry.getColString(0, null));
this.readSync.V();
} }
} catch (kelondroException e) { this.urlEntryHashCache.add(entry.getColString(0, null));
/* if we have an error, we start with a fresh database */ this.readSync.V();
plasmaCrawlStacker.this.log.logSevere("Unable to initialize crawl stacker queue, kelondroException:" + e.getMessage() + ". Reseting DB.\n",e);
// deleting old db and creating a new db
try {this.urlEntryCache.close();}catch(Exception ex){}
cacheFile.delete();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.rowdef, true);
} catch (IOException e) {
/* if we have an error, we start with a fresh database */
plasmaCrawlStacker.this.log.logSevere("Unable to initialize crawl stacker queue, IOException:" + e.getMessage() + ". Reseting DB.\n",e);
// deleting old db and creating a new db
try {this.urlEntryCache.close();}catch(Exception ex){}
cacheFile.delete();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.rowdef, true);
} }
} else { } catch (kelondroException e) {
// create new cache /* if we have an error, we start with a fresh database */
cacheFile.getParentFile().mkdirs(); plasmaCrawlStacker.this.log.logSevere("Unable to initialize crawl stacker queue, kelondroException:" + e.getMessage() + ". Reseting DB.\n", e);
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.rowdef, true);
// deleting old db and creating a new db
try {this.urlEntryCache.close();} catch (Exception ex) {}
deleteDB();
openDB();
} catch (IOException e) {
/* if we have an error, we start with a fresh database */
plasmaCrawlStacker.this.log.logSevere("Unable to initialize crawl stacker queue, IOException:" + e.getMessage() + ". Reseting DB.\n", e);
// deleting old db and creating a new db
try {this.urlEntryCache.close();} catch (Exception ex) {}
deleteDB();
openDB();
} }
}
private void deleteDB() {
File cacheFile = new File(cacheStacksPath, "urlPreNotice.db");
cacheFile.delete();
}
private void openDB() {
if (!(cacheStacksPath.exists())) cacheStacksPath.mkdir(); // make the path
File cacheFile = new File(cacheStacksPath, "urlPreNotice.db");
cacheFile.getParentFile().mkdirs();
this.urlEntryCache = kelondroTree.open(cacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.rowdef);
} }
public int cacheNodeChunkSize() { public int cacheNodeChunkSize() {

5
source/de/anomic/plasma/plasmaHTCache.java
Unescape View File

@ -154,10 +154,7 @@ public final class plasmaHTCache {
// open the response header database // open the response header database
File dbfile = new File(this.cachePath, "responseHeader.db"); File dbfile = new File(this.cachePath, "responseHeader.db");
try { try {
if (dbfile.exists()) this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400, preloadTime, indexURL.urlHashLength, 150, '#'));
this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400, preloadTime, '#'));
else
this.responseHeaderDB = new kelondroMap(new kelondroDyn(dbfile, bufferkb * 0x400, preloadTime, indexURL.urlHashLength, 150, '#', false));
} catch (IOException e) { } catch (IOException e) {
this.log.logSevere("the request header database could not be opened: " + e.getMessage()); this.log.logSevere("the request header database could not be opened: " + e.getMessage());
System.exit(0); System.exit(0);

2
source/de/anomic/plasma/plasmaRankingCRProcess.java
Unescape View File

@ -260,7 +260,7 @@ public class plasmaRankingCRProcess {
kelondroCollectionIndex newseq = null; kelondroCollectionIndex newseq = null;
if (newdb) { if (newdb) {
File path = to_file.getParentFile(); // path to storage place File path = to_file.getParentFile(); // path to storage place
newacc = new kelondroFlexTable(path, CRG_accname, kelondroBase64Order.enhancedCoder, 128 * 1024 * 1024, -1, CRG_accrow, true); newacc = new kelondroFlexTable(path, CRG_accname, kelondroBase64Order.enhancedCoder, 128 * 1024 * 1024, -1, CRG_accrow);
newseq = new kelondroCollectionIndex(path, CRG_seqname, 12, kelondroBase64Order.enhancedCoder, 128 * 1024 * 1024, -1, 2, CRG_colrow); newseq = new kelondroCollectionIndex(path, CRG_seqname, 12, kelondroBase64Order.enhancedCoder, 128 * 1024 * 1024, -1, 2, CRG_colrow);
} else { } else {
if (!(to_file.exists())) { if (!(to_file.exists())) {

10
source/de/anomic/plasma/plasmaSwitchboardQueue.java
Unescape View File

@ -89,15 +89,7 @@ public class plasmaSwitchboardQueue {
"String profile-" + indexURL.urlCrawlProfileHandleLength + ", " + // the name of the prefetch profile handle "String profile-" + indexURL.urlCrawlProfileHandleLength + ", " + // the name of the prefetch profile handle
"String urldescr-" + indexURL.urlDescrLength); // "String urldescr-" + indexURL.urlDescrLength); //
if (sbQueueStackPath.exists()) try { sbQueueStack = kelondroStack.open(sbQueueStackPath, rowdef);
sbQueueStack = new kelondroStack(sbQueueStackPath);
sbQueueStack.assignRowdef(rowdef);
} catch (IOException e) {
sbQueueStackPath.delete();
sbQueueStack = new kelondroStack(sbQueueStackPath, rowdef, true);
} else {
sbQueueStack = new kelondroStack(sbQueueStackPath, rowdef, true);
}
} }
private void resetQueueStack() { private void resetQueueStack() {

4
source/de/anomic/plasma/plasmaURLPool.java
Unescape View File

@ -58,9 +58,9 @@ public class plasmaURLPool {
public final plasmaCrawlEURL errorURL; public final plasmaCrawlEURL errorURL;
public plasmaURLPool(File plasmaPath, int ramLURL, int ramNURL, int ramEURL, long preloadTime) { public plasmaURLPool(File plasmaPath, int ramLURL, int ramNURL, int ramEURL, long preloadTime) {
loadedURL = new plasmaCrawlLURL(new File(plasmaPath, "urlHash.db"), ramLURL, preloadTime); loadedURL = new plasmaCrawlLURL(plasmaPath, ramLURL, preloadTime);
noticeURL = new plasmaCrawlNURL(plasmaPath, ramNURL, -1); noticeURL = new plasmaCrawlNURL(plasmaPath, ramNURL, -1);
errorURL = new plasmaCrawlEURL(new File(plasmaPath, "urlErr0.db"), ramEURL, -1); errorURL = new plasmaCrawlEURL(plasmaPath, ramEURL, -1);
} }
public String exists(String hash) { public String exists(String hash) {

10
source/de/anomic/plasma/plasmaWordConnotation.java
Unescape View File

@ -58,14 +58,8 @@ public class plasmaWordConnotation {
private static final int nodesize = 4048; private static final int nodesize = 4048;
private kelondroDynTree refDB; private kelondroDynTree refDB;
public plasmaWordConnotation(File refDBfile, int bufferkb, long preloadTime, char fillChar) { public plasmaWordConnotation(File refDBfile, int bufferkb, long preloadTime, char fillChar) throws IOException {
if (refDBfile.exists()) try { refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, preloadTime, wordlength, nodesize, new kelondroRow("byte[] word-" + wordlength + ", Cardinal count-" + countlength), fillChar);
refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, preloadTime, fillChar);
} catch (IOException e) {
refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, preloadTime, wordlength, nodesize, new kelondroRow("byte[] word-" + wordlength + ", Cardinal count-" + countlength), fillChar, true);
} else {
refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, preloadTime, wordlength, nodesize, new kelondroRow("byte[] word-" + wordlength + ", Cardinal count-" + countlength), fillChar, true);
}
} }
private void addSingleRef(String word, String reference) throws IOException { private void addSingleRef(String word, String reference) throws IOException {

29
source/de/anomic/plasma/plasmaWordIndexAssortment.java
Unescape View File

@ -116,33 +116,16 @@ public final class plasmaWordIndexAssortment {
this.bufferSize = bufferkb * 1024; this.bufferSize = bufferkb * 1024;
this.preloadTime = preloadTime; this.preloadTime = preloadTime;
this.log = log; this.log = log;
if (assortmentFile.exists()) { // open assortment tree file
// open existing assortment tree file long start = System.currentTimeMillis();
try { assortments = kelondroTree.open(assortmentFile, bufferSize, preloadTime, kelondroTree.defaultObjectCachePercent, bufferStructure(assortmentLength));
long start = System.currentTimeMillis(); long stop = System.currentTimeMillis();
assortments = new kelondroTree(assortmentFile, bufferSize, preloadTime, kelondroTree.defaultObjectCachePercent); if (log != null) log.logConfig("Opened Assortment, " +
assortments.assignRowdef(bufferStructure(assortmentLength));
long stop = System.currentTimeMillis();
if (log != null)
log.logConfig("Opened Assortment, " +
assortments.size() + " entries, width " + assortments.size() + " entries, width " +
assortmentLength + ", " + bufferkb + "kb buffer, " + assortmentLength + ", " + bufferkb + "kb buffer, " +
preloadTime + " ms preloadTime, " + preloadTime + " ms preloadTime, " +
(stop - start) + " ms effective, " + (stop - start) + " ms effective, " +
assortments.cacheNodeStatus()[1] + " preloaded"); assortments.cacheNodeStatus()[1] + " preloaded");
return;
} catch (IOException e){
serverLog.logSevere("PLASMA", "unable to open assortment database " + assortmentLength + ", creating new: " + e.getMessage(), e);
} catch (IndexOutOfBoundsException e){
serverLog.logSevere("PLASMA", "assortment database " + assortmentLength + " corupted, creating new: " + e.getMessage(), e);
} catch (kelondroException e) {
serverLog.logSevere("PLASMA", "assortment database " + assortmentLength + " corupted, creating new: " + e.getMessage(), e);
}
assortmentFile.delete(); // make space for new one
}
// create new assortment tree file
assortments = new kelondroTree(assortmentFile, bufferSize, preloadTime, kelondroTree.defaultObjectCachePercent, bufferStructure(assortmentLength), true);
if (log != null) log.logConfig("Created new Assortment, width " + assortmentLength + ", " + bufferkb + "kb buffer");
} }
public void store(indexContainer newContainer) { public void store(indexContainer newContainer) {
@ -269,7 +252,7 @@ public final class plasmaWordIndexAssortment {
if (!(assortmentFile.delete())) throw new RuntimeException("cannot delete assortment database"); if (!(assortmentFile.delete())) throw new RuntimeException("cannot delete assortment database");
} }
if (assortmentFile.exists()) assortmentFile.delete(); if (assortmentFile.exists()) assortmentFile.delete();
assortments = new kelondroTree(assortmentFile, bufferSize, preloadTime, kelondroTree.defaultObjectCachePercent, bufferStructure(assortmentLength), true); assortments = kelondroTree.open(assortmentFile, bufferSize, preloadTime, kelondroTree.defaultObjectCachePercent, bufferStructure(assortmentLength));
} }
public Iterator containers(String startWordHash, boolean up, boolean rot) throws IOException { public Iterator containers(String startWordHash, boolean up, boolean rot) throws IOException {

24
source/de/anomic/plasma/plasmaWordIndexFile.java
Unescape View File

@ -85,23 +85,13 @@ public final class plasmaWordIndexFile {
private kelondroTree indexFile(File databaseRoot, String wordHash) { private kelondroTree indexFile(File databaseRoot, String wordHash) {
if (wordHash.length() < 12) throw new RuntimeException("word hash wrong: '" + wordHash + "'"); if (wordHash.length() < 12) throw new RuntimeException("word hash wrong: '" + wordHash + "'");
theLocation = wordHash2path(databaseRoot, wordHash); theLocation = wordHash2path(databaseRoot, wordHash);
File fp = theLocation.getParentFile(); File fp = theLocation.getParentFile();
if (fp != null) fp.mkdirs(); if (fp != null) fp.mkdirs();
kelondroTree kt; long cacheSize = theLocation.length();
long cacheSize = theLocation.length(); if (cacheSize > 1048576) cacheSize = 1048576;
if (cacheSize > 1048576) cacheSize = 1048576; return kelondroTree.open(theLocation, cacheSize, 0, kelondroTree.defaultObjectCachePercent,
if (theLocation.exists()) try { new kelondroRow("byte[] urlhash-" + indexURL.urlHashLength + ", byte[] ba-" + indexURLEntry.encodedByteArrayFormLength(false)));
// open existing index file
kt = new kelondroTree(theLocation, cacheSize, 0, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) {
theLocation.delete();
kt = new kelondroTree(theLocation, cacheSize, 0, kelondroTree.defaultObjectCachePercent, new kelondroRow("byte[] urlhash-" + indexURL.urlHashLength + ", byte[] ba-" + indexURLEntry.encodedByteArrayFormLength(false)), false);
} else {
// create new index file
kt = new kelondroTree(theLocation, cacheSize, 0, kelondroTree.defaultObjectCachePercent, new kelondroRow("byte[] urlhash-" + indexURL.urlHashLength + ", byte[] ba-" + indexURLEntry.encodedByteArrayFormLength(false)), false);
}
return kt; // everyone who get this should close it when finished!
} }
public static File wordHash2path(File databaseRoot, String hash) { public static File wordHash2path(File databaseRoot, String hash) {

16
source/de/anomic/yacy/yacyNewsDB.java
Unescape View File

@ -67,25 +67,13 @@ public class yacyNewsDB {
this.path = path; this.path = path;
this.bufferkb = bufferkb; this.bufferkb = bufferkb;
this.preloadTime = preloadTime; this.preloadTime = preloadTime;
this.news = kelondroTree.open(path, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, yacyNewsRecord.rowdef);
if (path.exists()) try {
news = new kelondroTree(path, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent);
news.assignRowdef(yacyNewsRecord.rowdef);
} catch (IOException e) {
news = createDB(path, bufferkb, preloadTime);
} else {
news = createDB(path, bufferkb, preloadTime);
}
}
private static kelondroTree createDB(File path, int bufferkb, long preloadTime) {
return new kelondroTree(path, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, yacyNewsRecord.rowdef, true);
} }
private void resetDB() { private void resetDB() {
try {close();} catch (Exception e) {} try {close();} catch (Exception e) {}
if (path.exists()) path.delete(); if (path.exists()) path.delete();
news = createDB(path, bufferkb, preloadTime); this.news = kelondroTree.open(path, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, yacyNewsRecord.rowdef);
} }
public int cacheNodeChunkSize() { public int cacheNodeChunkSize() {

26
source/de/anomic/yacy/yacyNewsQueue.java
Unescape View File

@ -51,44 +51,28 @@ import java.util.Date;
import de.anomic.kelondro.kelondroColumn; import de.anomic.kelondro.kelondroColumn;
import de.anomic.kelondro.kelondroRow; import de.anomic.kelondro.kelondroRow;
import de.anomic.kelondro.kelondroStack; import de.anomic.kelondro.kelondroStack;
import de.anomic.kelondro.kelondroException;
public class yacyNewsQueue { public class yacyNewsQueue {
private File path; private File path;
private kelondroStack queueStack; private kelondroStack queueStack;
private yacyNewsDB newsDB; private yacyNewsDB newsDB;
public yacyNewsQueue(File path, yacyNewsDB newsDB) {
this.path = path;
this.newsDB = newsDB;
if (path.exists()) try {
queueStack = new kelondroStack(path);
} catch (kelondroException e) {
path.delete();
queueStack = createStack(path);
} catch (IOException e) {
path.delete();
queueStack = createStack(path);
} else {
queueStack = createStack(path);
}
}
public static final kelondroRow rowdef = new kelondroRow(new kelondroColumn[]{ public static final kelondroRow rowdef = new kelondroRow(new kelondroColumn[]{
new kelondroColumn("newsid", kelondroColumn.celltype_string, kelondroColumn.encoder_bytes, yacyNewsRecord.idLength, "id = created + originator"), new kelondroColumn("newsid", kelondroColumn.celltype_string, kelondroColumn.encoder_bytes, yacyNewsRecord.idLength, "id = created + originator"),
new kelondroColumn("last touched", kelondroColumn.celltype_string, kelondroColumn.encoder_bytes, yacyCore.universalDateShortPattern.length(), "") new kelondroColumn("last touched", kelondroColumn.celltype_string, kelondroColumn.encoder_bytes, yacyCore.universalDateShortPattern.length(), "")
}); });
private static kelondroStack createStack(File path) { public yacyNewsQueue(File path, yacyNewsDB newsDB) {
return new kelondroStack(path, rowdef, true); this.path = path;
this.newsDB = newsDB;
this.queueStack = kelondroStack.open(path, rowdef);
} }
private void resetDB() { private void resetDB() {
try {close();} catch (Exception e) {} try {close();} catch (Exception e) {}
if (path.exists()) path.delete(); if (path.exists()) path.delete();
queueStack = createStack(path); queueStack = kelondroStack.open(path, rowdef);
} }
public void clear() { public void clear() {

17
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -202,25 +202,14 @@ public final class yacySeedDB {
} }
private synchronized kelondroMap openSeedTable(File seedDBFile) { private synchronized kelondroMap openSeedTable(File seedDBFile) {
if (seedDBFile.exists()) try { new File(seedDBFile.getParent()).mkdirs();
// open existing seed database return new kelondroMap(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#'), sortFields, accFields);
return new kelondroMap(new kelondroDyn(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, '#'), sortFields, accFields);
} catch (kelondroException e) {
// if we have an error, we start with a fresh database
if (seedDBFile.exists()) seedDBFile.delete();
} catch (IOException e) {
// if we have an error, we start with a fresh database
if (seedDBFile.exists()) seedDBFile.delete();
}
// create new seed database
new File(seedDBFile.getParent()).mkdir();
return new kelondroMap(new kelondroDyn(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', true), sortFields, accFields);
} }
protected synchronized kelondroMap resetSeedTable(kelondroMap seedDB, File seedDBFile) { protected synchronized kelondroMap resetSeedTable(kelondroMap seedDB, File seedDBFile) {
// this is an emergency function that should only be used if any problem with the // this is an emergency function that should only be used if any problem with the
// seed.db is detected // seed.db is detected
yacyCore.log.logFine("seed-db " + seedDBFile.toString() + " reset (on-the-fly)"); yacyCore.log.logFine("seed-db " + seedDBFile.toString() + " reset (on-the-fly)");
try { try {
seedDB.close(); seedDB.close();
seedDBFile.delete(); seedDBFile.delete();

2
source/yacy.java
Unescape View File

@ -1223,7 +1223,7 @@ public final class yacy {
String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"}; String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"};
for (int i=0; i < dbFileNames.length; i++) { for (int i=0; i < dbFileNames.length; i++) {
File dbFile = new File(yacyDBPath,dbFileNames[i]); File dbFile = new File(yacyDBPath,dbFileNames[i]);
kelondroMap db = new kelondroMap(new kelondroDyn(dbFile, (1024 * 0x400) / 3, 3000, '#'), yacySeedDB.sortFields, yacySeedDB.accFields); kelondroMap db = new kelondroMap(new kelondroDyn(dbFile, (1024 * 0x400) / 3, 3000, yacySeedDB.commonHashLength, 480, '#'), yacySeedDB.sortFields, yacySeedDB.accFields);
kelondroMap.mapIterator it; kelondroMap.mapIterator it;
it = db.maps(true, false); it = db.maps(true, false);

Write Preview
Loading…
Cancel
Save