boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

full integration of kelondroRow

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2167 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent 6e49721762
commit c36e9fc8d3
31 changed files with 343 additions and 370 deletions
Show Stats Download Patch File Download Diff File

28
source/dbtest.java
Unescape View File

@ -116,7 +116,7 @@ public class dbtest {
public void run() {
final STEntry entry = new STEntry(this.getSource());
try {
getTable().put(new byte[][] { entry.getKey(), entry.getValue() , entry.getValue() });
getTable().put(getTable().row().newEntry(new byte[][] { entry.getKey(), entry.getValue() , entry.getValue() }));
} catch (IOException e) {
System.err.println(e);
e.printStackTrace();
@ -166,12 +166,13 @@ public class dbtest {
profiler.start();
// create the database access
kelondroRow testRow = new kelondroRow(new int[]{keylength, valuelength, valuelength});
if (dbe.equals("kelondroTree")) {
File tablefile = new File(tablename + ".kelondro.db");
if (tablefile.exists()) {
table = new kelondroTree(tablefile, buffer, kelondroTree.defaultObjectCachePercent);
} else {
table = new kelondroTree(tablefile, buffer, kelondroTree.defaultObjectCachePercent, new int[]{keylength, valuelength, valuelength}, true);
table = new kelondroTree(tablefile, buffer, kelondroTree.defaultObjectCachePercent, testRow, true);
}
}
if (dbe.equals("kelondroSplittedTree")) {
@ -179,7 +180,7 @@ public class dbtest {
table = kelondroSplittedTree.open(tablepath, tablename, kelondroBase64Order.enhancedCoder,
buffer,
8,
new int[]{keylength, valuelength, valuelength}, 1, 80,
testRow, 1, 80,
true);
}
if (dbe.equals("kelondroFlexTable")) {
@ -187,11 +188,11 @@ public class dbtest {
table = new kelondroFlexTable(tablepath, new File(tablename).getName(), new kelondroRow(new int[]{keylength, valuelength, valuelength}), true);
}
if (dbe.equals("mysql")) {
table = new dbTable("mysql");
table = new dbTable("mysql", testRow);
}
if (dbe.equals("pgsql")) {
table = new dbTable("pgsql");
table = new dbTable("pgsql", testRow);
}
long afterinit = System.currentTimeMillis();
@ -210,7 +211,7 @@ public class dbtest {
long randomstart = Long.parseLong(args[4]);
Random random = new Random(randomstart);
for (int i = 0; i < count; i++) {
table.put(new byte[][]{randomHash(random), dummyvalue1, dummyvalue2});
table.put(table.row().newEntry(new byte[][]{randomHash(random), dummyvalue1, dummyvalue2}));
if (i % 500 == 0) {
System.out.println(i + " entries processed so far.");
}
@ -305,8 +306,10 @@ final class dbTable implements kelondroIndex {
private PreparedStatement sqlStatement;
private int commandCount = 0;
private int batchlimit = 1;
private kelondroRow rowdef;
public dbTable(String dbType) throws Exception {
public dbTable(String dbType, kelondroRow rowdef) throws Exception {
this.rowdef = rowdef;
openDatabaseConnection(dbType);
}
@ -357,15 +360,18 @@ final class dbTable implements kelondroIndex {
}
}
public kelondroRow row() {
return this.rowdef;
}
public kelondroRow.Entry get(byte[] key) throws IOException {
return null;
}
public byte[][] put(byte[][] row) throws IOException {
public kelondroRow.Entry put(kelondroRow.Entry row) throws IOException {
try {
this.sqlStatement.setString(1,new String(row[0]));
sqlStatement.setBytes(2,row[1]);
this.sqlStatement.setString(1, new String(row.getColString(0, null)));
sqlStatement.setBytes(2, row.getColBytes(1));
sqlStatement.addBatch();
commandCount++;
@ -380,7 +386,7 @@ final class dbTable implements kelondroIndex {
}
}
public byte[][] remove(byte[] key) throws IOException {
public kelondroRow.Entry remove(byte[] key) throws IOException {
// TODO Auto-generated method stub
return null;
}

2
source/de/anomic/index/indexRAMCacheRI.java
Unescape View File

@ -94,7 +94,7 @@ public final class indexRAMCacheRI extends indexAbstractRI implements indexRI {
File indexDumpFile = new File(databaseRoot, indexArrayFileName);
if (indexDumpFile.exists()) indexDumpFile.delete();
kelondroFixedWidthArray dumpArray = null;
dumpArray = new kelondroFixedWidthArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0, false);
dumpArray = new kelondroFixedWidthArray(indexDumpFile, new kelondroRow(plasmaWordIndexAssortment.bufferStructureBasis), 0, false);
long startTime = System.currentTimeMillis();
long messageTime = System.currentTimeMillis() + 5000;
long wordsPerSecond = 0, wordcount = 0, urlcount = 0;

2
source/de/anomic/kelondro/kelondroArray.java
Unescape View File

@ -30,7 +30,7 @@ public interface kelondroArray {
public int size();
public int columns();
public kelondroRow row();
public kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException;

42
source/de/anomic/kelondro/kelondroBase64Order.java
Unescape View File

@ -88,7 +88,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
return o;
}
public static kelondroOrder bySignature(String signature) {
public final static kelondroOrder bySignature(String signature) {
if (signature.equals("Bd")) return new kelondroBase64Order(false, false);
if (signature.equals("bd")) return new kelondroBase64Order(false, true);
if (signature.equals("Bu")) return new kelondroBase64Order(true, false);
@ -96,7 +96,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
return null;
}
public String signature() {
public final String signature() {
if ((!asc) && (!rfc1113compliant)) return "Bd";
if ((!asc) && ( rfc1113compliant)) return "bd";
if (( asc) && (!rfc1113compliant)) return "Bu";
@ -104,15 +104,15 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
return null;
}
public char encodeByte(byte b) {
public final char encodeByte(byte b) {
return (char) alpha[b];
}
public byte decodeByte(char b) {
public final byte decodeByte(char b) {
return ahpla[b];
}
public String encodeLongSmart(long c, int length) {
public final String encodeLongSmart(long c, int length) {
if (c >= max(length)) {
StringBuffer s = new StringBuffer(length);
s.setLength(length);
@ -123,7 +123,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
}
}
public String encodeLong(long c, int length) {
public final String encodeLong(long c, int length) {
StringBuffer s = new StringBuffer(length);
s.setLength(length);
while (length > 0) {
@ -133,14 +133,22 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
return s.toString();
}
public long decodeLong(String s) {
public final void encodeLong(long c, byte[] b, int offset, int length) {
assert offset + length <= b.length;
while (length > 0) {
b[--length + offset] = (byte) alpha[(byte) (c & 0x3F)];
c >>= 6;
}
}
public final long decodeLong(String s) {
while (s.endsWith("=")) s = s.substring(0, s.length() - 1);
long c = 0;
for (int i = 0; i < s.length(); i++) c = (c << 6) | ahpla[s.charAt(i)];
return c;
}
public long decodeLong(byte[] s, int offset, int length) {
public final long decodeLong(byte[] s, int offset, int length) {
while ((length > 0) && (s[offset + length - 1] == '=')) length--;
long c = 0;
for (int i = 0; i < length; i++) c = (c << 6) | ahpla[s[offset + i]];
@ -155,14 +163,14 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
return c;
}
public String encodeString(String in) {
public final String encodeString(String in) {
return encode(in.getBytes());
}
// we will use this encoding to encode strings with 2^8 values to
// b64-Strings
// we will do that by grouping each three input bytes to four output bytes.
public String encode(byte[] in) {
public final String encode(byte[] in) {
StringBuffer out = new StringBuffer(in.length / 3 * 4 + 3);
int pos = 0;
long l;
@ -181,7 +189,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
return out.toString();
}
public String decodeString(String in) {
public final String decodeString(String in) {
try {
//return new String(decode(in), "ISO-8859-1");
return new String(decode(in), "UTF-8");
@ -191,7 +199,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
}
}
public byte[] decode(String in) {
public final byte[] decode(String in) {
try {
int posIn = 0;
int posOut = 0;
@ -233,7 +241,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
}
}
private long cardinalI(byte[] key) {
private final long cardinalI(byte[] key) {
// returns a cardinal number in the range of 0 .. Long.MAX_VALUE
long c = 0;
int p = 0;
@ -243,7 +251,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
return c;
}
public long cardinal(byte[] key) {
public final long cardinal(byte[] key) {
if (this.zero == null) return cardinalI(key);
long zeroCardinal = cardinalI(this.zero);
long keyCardinal = cardinalI(key);
@ -251,11 +259,11 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
return Long.MAX_VALUE - keyCardinal + zeroCardinal + 1;
}
public int compare(byte[] a, byte[] b) {
public final int compare(byte[] a, byte[] b) {
return (asc) ? compare0(a, b) : compare0(b, a);
}
public int compare0(byte[] a, byte[] b) {
public final int compare0(byte[] a, byte[] b) {
if (zero == null) return compares(a, b);
// we have an artificial start point. check all combinations
int az = compares(a, zero); // -1 if a < z; 0 if a == z; 1 if a > z
@ -267,7 +275,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
return bz;
}
public int compares(byte[] a, byte[] b) {
public final int compares(byte[] a, byte[] b) {
int i = 0;
final int al = a.length;
final int bl = b.length;

45
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -72,7 +72,7 @@ public class kelondroCollectionIndex {
columns[2] = 4; // chunkcount (number of chunks in this collection)
columns[3] = 4; // index (position in index file)
columns[4] = 2; // update time in days since 1.1.2000
index = new kelondroSplittedTree(path, filenameStub, indexOrder, buffersize, 8, columns, 1, 80, true);
index = new kelondroSplittedTree(path, filenameStub, indexOrder, buffersize, 8, new kelondroRow(columns), 1, 80, true);
// create array files
this.array = new kelondroFixedWidthArray[partitions];
@ -97,13 +97,13 @@ public class kelondroCollectionIndex {
} else {
int load = 1; for (int i = 0; i < partitionNumber; i++) load = load * loadfactor;
int[] columns = new int[4];
columns[0] = index.columnSize(0); // add always the key
columns[0] = index.row().width(0); // add always the key
columns[1] = 4; // chunkcount (raw format)
columns[2] = 2; // last time read
columns[3] = 2; // last time wrote
columns[4] = 2; // flag string, assigns collection order as currently stored in table
columns[5] = load * genericChunkSize;
return new kelondroFixedWidthArray(f, columns, 0, true);
return new kelondroFixedWidthArray(f, new kelondroRow(columns), 0, true);
}
}
@ -134,12 +134,13 @@ public class kelondroCollectionIndex {
// write a new entry in this array
int newRowNumber = array[part].add(array[part].row().newEntry(newarrayrow));
// store the new row number in the index
index.put(new byte[][]{key,
kelondroNaturalOrder.encodeLong(this.chunksize, 4),
kelondroNaturalOrder.encodeLong(collection.size(), 4),
kelondroNaturalOrder.encodeLong((long) newRowNumber, 4),
kelondroNaturalOrder.encodeLong(daysSince2000(System.currentTimeMillis()), 2)
});
kelondroRow.Entry e = index.row().newEntry();
e.setCol(0, key);
e.setColLongB256(1, this.chunksize);
e.setColLongB256(2, collection.size());
e.setColLongB256(3, (long) newRowNumber);
e.setColLongB256(4, daysSince2000(System.currentTimeMillis()));
index.put(e);
} else {
// overwrite the old collection
// read old information
@ -154,24 +155,26 @@ public class kelondroCollectionIndex {
// we don't need a new slot, just write in the old one
array[oldPartitionNumber].set(rownumber, array[oldPartitionNumber].row().newEntry(newarrayrow));
// update the index entry
index.put(new byte[][]{key,
kelondroNaturalOrder.encodeLong(this.chunksize, 4),
kelondroNaturalOrder.encodeLong(collection.size(), 4),
kelondroNaturalOrder.encodeLong((long) rownumber, 4),
kelondroNaturalOrder.encodeLong(daysSince2000(System.currentTimeMillis()), 2)
});
kelondroRow.Entry e = index.row().newEntry();
e.setCol(0, key);
e.setColLongB256(1, this.chunksize);
e.setColLongB256(2, collection.size());
e.setColLongB256(3, (long) rownumber);
e.setColLongB256(4, daysSince2000(System.currentTimeMillis()));
index.put(e);
} else {
// we need a new slot, that means we must first delete the old entry
array[oldPartitionNumber].remove(rownumber);
// write a new entry in the other array
int newRowNumber = array[newPartitionNumber].add(array[newPartitionNumber].row().newEntry(newarrayrow));
// store the new row number in the index
index.put(new byte[][]{key,
kelondroNaturalOrder.encodeLong(this.chunksize, 4),
kelondroNaturalOrder.encodeLong(collection.size(), 4),
kelondroNaturalOrder.encodeLong((long) newRowNumber, 4),
kelondroNaturalOrder.encodeLong(daysSince2000(System.currentTimeMillis()), 2)
});
kelondroRow.Entry e = index.row().newEntry();
e.setCol(0, key);
e.setColLongB256(1, this.chunksize);
e.setColLongB256(2, collection.size());
e.setColLongB256(3, (long) newRowNumber);
e.setColLongB256(4, daysSince2000(System.currentTimeMillis()));
index.put(e);
}
}
}

16
source/de/anomic/kelondro/kelondroDyn.java
Unescape View File

@ -76,9 +76,9 @@ public class kelondroDyn extends kelondroTree {
int nodesize, char fillChar, kelondroOrder objectOrder,
boolean exitOnFail) {
// creates a new dynamic tree
super(file, buffersize, kelondroTree.defaultObjectCachePercent, new int[] { key + counterlen, nodesize }, objectOrder, 1, 8, exitOnFail);
this.keylen = columnSize(0) - counterlen;
this.reclen = columnSize(1);
super(file, buffersize, kelondroTree.defaultObjectCachePercent, new kelondroRow(new int[] { key + counterlen, nodesize }), objectOrder, 1, 8, exitOnFail);
this.keylen = row().width(0) - counterlen;
this.reclen = row().width(1);
this.fillChar = fillChar;
this.segmentCount = 0;
writeSegmentCount();
@ -88,8 +88,8 @@ public class kelondroDyn extends kelondroTree {
public kelondroDyn(File file, long buffersize, char fillChar) throws IOException {
// this opens a file with an existing dynamic tree
super(file, buffersize, kelondroTree.defaultObjectCachePercent);
this.keylen = columnSize(0) - counterlen;
this.reclen = columnSize(1);
this.keylen = row().width(0) - counterlen;
this.reclen = row().width(1);
this.fillChar = fillChar;
this.segmentCount = 0;
buffer = new kelondroObjectBuffer(file.toString());
@ -159,11 +159,11 @@ public class kelondroDyn extends kelondroTree {
String k;
String v;
int c;
byte[][] nt;
kelondroRow.Entry nt;
while (ri.hasNext()) {
nt = (byte[][]) ri.next();
nt = (kelondroRow.Entry) ri.next();
if (nt == null) throw new kelondroException(filename, "no more elements available");
g = nt[0];
g = nt.getColBytes(0);
if (g == null) return null;
k = new String(g, 0, keylen);
v = new String(g, keylen, counterlen);

75
source/de/anomic/kelondro/kelondroDynTree.java
Unescape View File

@ -53,7 +53,7 @@ import java.util.Iterator;
public class kelondroDynTree {
// basic data structures
private int[] columns;
private kelondroRow rowdef;
private kelondroDyn table;
private Hashtable treeRAHandles;
private File file;
@ -69,10 +69,10 @@ public class kelondroDynTree {
private Hashtable buffer, cache;
private long cycleBuffer;
public kelondroDynTree(File file, long buffersize, int keylength, int nodesize, int[] columns, char fillChar, boolean exitOnFail) {
public kelondroDynTree(File file, long buffersize, int keylength, int nodesize, kelondroRow rowdef, char fillChar, boolean exitOnFail) {
// creates a new DynTree
this.file = file;
this.columns = columns;
this.rowdef = rowdef;
this.buffer = new Hashtable();
this.cache = new Hashtable();
//this.cycleCache = Long.MIN_VALUE;
@ -97,8 +97,9 @@ public class kelondroDynTree {
Iterator i = table.dynKeys(true, false);
String onekey = (String) i.next();
kelondroTree onetree = getTree(onekey);
this.columns = new int[onetree.columns()];
for (int j = 0; j < columns.length; j++) columns[j] = onetree.columnSize(j);
int[] columns = new int[onetree.row().columns()];
for (int j = 0; j < columns.length; j++) columns[j] = onetree.row().width(j);
this.rowdef = new kelondroRow(columns);
closeTree(onekey);
}
@ -109,15 +110,6 @@ public class kelondroDynTree {
table.close();
if (size == 0) this.file.delete();
}
/*
public void setReadCacheAttr(int maxcount, int maxsize, long maxage, long cycletime) {
maxcountCache = maxcount;
maxsizeCache = maxsize;
maxageCache = maxage;
cycletimeCache = cycletime;
}
*/
public void setWriteBufferAttr(int maxcount, int maxsize, long maxage, long cycletime) {
maxcountBuffer = maxcount;
@ -135,7 +127,7 @@ public class kelondroDynTree {
kelondroRA ra = table.getRA(key); // works always, even with no-existing entry
treeRAHandles.put(key, ra);
try {
return new kelondroTree(ra, buffersize, kelondroTree.defaultObjectCachePercent, columns, false);
return new kelondroTree(ra, buffersize, kelondroTree.defaultObjectCachePercent, rowdef, false);
} catch (RuntimeException e) {
throw new IOException(e.getMessage());
}
@ -183,11 +175,11 @@ public class kelondroDynTree {
this.timestamp = Long.MAX_VALUE; // to flag no-update
}
public byte[][] get(byte[] key) throws IOException {
byte[][] entry = (byte[][]) tcache.get(key);
public kelondroRow.Entry get(byte[] key) throws IOException {
kelondroRow.Entry entry = (kelondroRow.Entry) tcache.get(key);
if (entry == null) {
kelondroTree t = getTree(this.tablename);
entry = t.get(key).getCols();
entry = t.get(key);
t.close();
this.tcache.put(key, entry);
this.timestamp = System.currentTimeMillis();
@ -195,8 +187,8 @@ public class kelondroDynTree {
return entry;
}
protected void put(byte[][] entry) { // this is only used internal
this.tcache.put(entry[0], entry);
protected void put(kelondroRow.Entry entry) { // this is only used internal
this.tcache.put(entry.getColBytes(0), entry);
this.timestamp = System.currentTimeMillis();
}
@ -218,8 +210,8 @@ public class kelondroDynTree {
this.timestamp = Long.MAX_VALUE; // to flag no-update
}
public void put(byte[][] entry) {
this.tbuffer.put(entry[0], entry);
public void put(kelondroRow.Entry entry) {
this.tbuffer.put(entry.getColBytes(0), entry);
this.timestamp = System.currentTimeMillis();
}
@ -233,11 +225,11 @@ public class kelondroDynTree {
if (this.tbuffer.size() == 0) return;
Enumeration e = this.tbuffer.keys();
kelondroTree t = getTree(this.tablename);
byte[][] entry;
kelondroRow.Entry entry;
byte[] key;
while (e.hasMoreElements()) {
key = (byte[]) e.nextElement();
entry = (byte[][]) this.tbuffer.get(key);
entry = (kelondroRow.Entry) this.tbuffer.get(key);
t.put(entry);
}
t.close();
@ -248,7 +240,7 @@ public class kelondroDynTree {
// read cached
public synchronized byte[][] get(String tablename, byte[] key) throws IOException {
public synchronized kelondroRow.Entry get(String tablename, byte[] key) throws IOException {
treeCache tc = (treeCache) cache.get(table);
if (tc == null) {
tc = new treeCache(tablename);
@ -256,31 +248,9 @@ public class kelondroDynTree {
}
return tc.get(key);
}
/*
// clean-up method for cache:
private void flushCache() {
if ((System.currentTimeMillis() - this.cycleCache < this.cycletimeCache) &&
(cache.size() < this.maxcountCache)) return;
this.cycleCache = System.currentTimeMillis();
// collect all caches which have a time > maxagecache
Enumeration e = cache.keys();
String tablename;
treeCache tc;
while (e.hasMoreElements()) {
tablename = (String) e.nextElement();
tc = (treeCache) cache.get(tablename);
if ((System.currentTimeMillis() - tc.timestamp > this.maxageCache) ||
(tc.cache.size() > this.maxsizeCache) ||
(cache.size() > this.maxcountCache)) {
cache.remove(tablename);
}
}
}
*/
// write buffered
public synchronized void put(String tablename, byte[][] newrow) {
public synchronized void put(String tablename, kelondroRow.Entry newrow) {
treeBuffer tb = (treeBuffer) buffer.get(tablename);
if (tb == null) {
tb = new treeBuffer(tablename);
@ -350,17 +320,18 @@ public class kelondroDynTree {
File file = new File("D:\\bin\\testDyn.db");
if (file.exists()) {
kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, '_');
System.out.println("opened: table keylength=" + dt.table.columnSize(0) + ", sectorsize=" + dt.table.columnSize(1) + ", " + dt.table.size() + " entries.");
System.out.println("opened: table keylength=" + dt.table.row().width(0) + ", sectorsize=" + dt.table.row().width(1) + ", " + dt.table.size() + " entries.");
} else {
kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, 16, 512, new int[] {10,20,30}, '_', true);
kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, 16, 512, new kelondroRow(new int[] {10,20,30}), '_', true);
String name;
kelondroTree t;
byte[][] line = new byte[][] {"".getBytes(), "abc".getBytes(), "def".getBytes()};
kelondroRow.Entry line;
for (int i = 1; i < 100; i++) {
name = "test" + i;
t = dt.newTree(name);
line = t.row().newEntry(new byte[][] {"".getBytes(), "abc".getBytes(), "def".getBytes()});
for (int j = 1; j < 10; j++) {
line[0] = ("entry" + j).getBytes();
line.setCol(0, ("entry" + j).getBytes());
t.put(line);
}
dt.closeTree(name);

8
source/de/anomic/kelondro/kelondroFScoreCluster.java
Unescape View File

@ -70,11 +70,11 @@ public class kelondroFScoreCluster {
} catch (IOException e) {
refcountDBfile.delete();
countrefDBfile.delete();
refcountDB = new kelondroTree(refcountDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new int[] {wordlength, countlength}, objectOrder, 1, countlength, exitOnFail);
countrefDB = new kelondroTree(countrefDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new int[] {countlength + wordlength, 4}, objectOrder, 1, countlength, exitOnFail);
refcountDB = new kelondroTree(refcountDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new kelondroRow(new int[] {wordlength, countlength}), objectOrder, 1, countlength, exitOnFail);
countrefDB = new kelondroTree(countrefDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new kelondroRow(new int[] {countlength + wordlength, 4}), objectOrder, 1, countlength, exitOnFail);
} else if ((!(refcountDBfile.exists())) && (!(countrefDBfile.exists()))) {
refcountDB = new kelondroTree(refcountDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new int[] {wordlength, countlength}, objectOrder, 1, countlength, exitOnFail);
countrefDB = new kelondroTree(countrefDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new int[] {countlength + wordlength, 4}, objectOrder, 1, countlength, exitOnFail);
refcountDB = new kelondroTree(refcountDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new kelondroRow(new int[] {wordlength, countlength}), objectOrder, 1, countlength, exitOnFail);
countrefDB = new kelondroTree(countrefDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new kelondroRow(new int[] {countlength + wordlength, 4}), objectOrder, 1, countlength, exitOnFail);
} else {
if (exitOnFail) {
System.exit(-1);

12
source/de/anomic/kelondro/kelondroFixedWidthArray.java
Unescape View File

@ -55,9 +55,9 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
private static short thisOHBytes = 0; // our record definition does not need extra bytes
private static short thisOHHandles = 0; // and no handles
public kelondroFixedWidthArray(File file, int[] columns, int intprops, boolean exitOnFail) {
public kelondroFixedWidthArray(File file, kelondroRow rowdef, int intprops, boolean exitOnFail) {
// this creates a new array
super(file, 0, thisOHBytes, thisOHHandles, columns, intprops, columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
super(file, 0, thisOHBytes, thisOHHandles, rowdef, intprops, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
for (int i = 0; i < intprops; i++) try {
setHandle(i, new Handle(0));
} catch (IOException e) {
@ -124,7 +124,7 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
for (int i = 0; i < size(); i++) {
System.out.print("row " + i + ": ");
row = get(i);
for (int j = 0; j < columns(); j++) System.out.print(((row.empty(j)) ? "NULL" : row.getColString(j, "UTF-8")) + ", ");
for (int j = 0; j < row.columns(); j++) System.out.print(((row.empty(j)) ? "NULL" : row.getColString(j, "UTF-8")) + ", ");
System.out.println();
}
System.out.println("EndOfTable");
@ -144,7 +144,7 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
// create <filename> <valuelen>
File f = new File(args[1]);
if (f.exists()) f.delete();
kelondroFixedWidthArray fm = new kelondroFixedWidthArray(f, new int[]{Integer.parseInt(args[2])}, 2, true);
kelondroFixedWidthArray fm = new kelondroFixedWidthArray(f, new kelondroRow(new int[]{Integer.parseInt(args[2])}), 2, true);
fm.close();
} else
if ((args.length == 2) && (args[0].equals("-v"))) {
@ -158,7 +158,7 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
// get <filename> <index>
kelondroFixedWidthArray fm = new kelondroFixedWidthArray(new File(args[1]));
kelondroRow.Entry row = fm.get(Integer.parseInt(args[2]));
for (int j = 0; j < fm.columns(); j++) System.out.print(row.getColString(j, null) + " ");
for (int j = 0; j < fm.row().columns(); j++) System.out.print(row.getColString(j, null) + " ");
System.out.println();
fm.close();
} else
@ -186,7 +186,7 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
if ((args.length == 1) && (args[0].equals("-test"))) {
File testfile = new File("test.array");
if (testfile.exists()) testfile.delete();
kelondroFixedWidthArray fm = new kelondroFixedWidthArray(testfile, new int[]{30, 50}, 9, true);
kelondroFixedWidthArray fm = new kelondroFixedWidthArray(testfile, new kelondroRow(new int[]{30, 50}), 9, true);
for (int i = 0; i < 100; i++) {
fm.set(i, fm.row().newEntry(new byte[][]{("name" + i).getBytes(), ("value" + i).getBytes()}));
}

18
source/de/anomic/kelondro/kelondroFlexTable.java
Unescape View File

@ -61,31 +61,27 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
}
*/
public int columnSize(int column) {
return rowdef.width(column);
}
public kelondroRow.Entry get(byte[] key) throws IOException {
Integer i = (Integer) this.index.get(key);
if (i == null) return null;
return super.get(i.intValue());
}
public byte[][] put(byte[][] row) throws IOException {
Integer i = (Integer) this.index.get(row[0]);
public kelondroRow.Entry put(kelondroRow.Entry row) throws IOException {
Integer i = (Integer) this.index.get(row.getColBytes(0));
if (i == null) {
i = new Integer(super.add(super.rowdef.newEntry(row)));
this.index.put(row[0], i);
i = new Integer(super.add(row));
this.index.put(row.getColBytes(0), i);
return null;
} else {
return super.set(i.intValue(), super.rowdef.newEntry(row)).getCols();
return super.set(i.intValue(), row);
}
}
public byte[][] remove(byte[] key) throws IOException {
public kelondroRow.Entry remove(byte[] key) throws IOException {
Integer i = (Integer) this.index.get(key);
if (i == null) return null;
byte[][] r = super.get(i.intValue()).getCols();
kelondroRow.Entry r = super.get(i.intValue());
super.remove(i.intValue());
return r;
}

33
source/de/anomic/kelondro/kelondroFlexWidthArray.java
Unescape View File

@ -81,7 +81,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
columns[j - p] = rowdef.width(j);
check = check.substring(0, j) + "X" + check.substring(j + 1);
}
col[p] = new kelondroFixedWidthArray(new File(tabledir, colfilename(p, q)), columns, 16, true);
col[p] = new kelondroFixedWidthArray(new File(tabledir, colfilename(p, q)), new kelondroRow(columns), 16, true);
}
}
@ -94,12 +94,13 @@ public class kelondroFlexWidthArray implements kelondroArray {
return "col." + f + ".list";
}
public int size() {
return col[0].size();
public kelondroRow row() {
return rowdef;
}
public int columns() {
return rowdef.columns();
public int size() {
return col[0].size();
}
public kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException {
@ -112,12 +113,12 @@ public class kelondroFlexWidthArray implements kelondroArray {
rowentry.bytes(),
rowdef.colstart[r],
rowdef.colstart[r]
- rowdef.colstart[r + col[r].columns() - 1]
- rowdef.colstart[r + col[r].row().columns() - 1]
+ rowdef.width(r));
e1 = col[r].set(index, e0);
for (int i = 0; i < col[r].columns(); i++)
for (int i = 0; i < col[r].row().columns(); i++)
p.setCol(r + i, e1.getColBytes(i));
r = r + col[r].columns();
r = r + col[r].row().columns();
}
}
return p;
@ -130,9 +131,9 @@ public class kelondroFlexWidthArray implements kelondroArray {
synchronized (col) {
while (r < rowdef.columns()) {
e = col[r].get(index);
for (int i = 0; i < col[r].columns(); i++)
for (int i = 0; i < col[r].row().columns(); i++)
p.setCol(r + i, e.getColBytes(i));
r = r + col[r].columns();
r = r + col[r].row().columns();
}
}
return p;
@ -144,17 +145,17 @@ public class kelondroFlexWidthArray implements kelondroArray {
synchronized (col) {
e = col[0].row().newEntry(rowentry.bytes(), 0, rowdef.width(0));
index = col[0].add(e);
int r = col[0].columns();
int r = col[0].row().columns();
while (r < rowdef.columns()) {
e = col[r].row().newEntry(
rowentry.bytes(),
rowdef.colstart[r],
rowdef.colstart[r + col[r].columns() - 1]
+ rowdef.width(r + col[r].columns() - 1)
rowdef.colstart[r + col[r].row().columns() - 1]
+ rowdef.width(r + col[r].row().columns() - 1)
- rowdef.colstart[r]);
col[r].set(index, e);
r = r + col[r].columns();
r = r + col[r].row().columns();
}
}
return index;
@ -165,7 +166,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
synchronized (col) {
while (r < rowdef.columns()) {
col[r].remove(index);
r = r + col[r].columns();
r = r + col[r].row().columns();
}
}
}
@ -176,7 +177,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
for (int i = 0; i < size(); i++) {
System.out.print("row " + i + ": ");
row = get(i);
for (int j = 0; j < columns(); j++) System.out.print(((row.empty(j)) ? "NULL" : row.getColString(j, "UTF-8")) + ", ");
for (int j = 0; j < row().columns(); j++) System.out.print(((row.empty(j)) ? "NULL" : row.getColString(j, "UTF-8")) + ", ");
System.out.println();
}
System.out.println("EndOfTable");

16
source/de/anomic/kelondro/kelondroHashtable.java
Unescape View File

@ -142,7 +142,7 @@ public class kelondroHashtable {
private static final byte[] dummyKey = kelondroBase64Order.enhancedCoder.encodeLong(0, 5).getBytes();
public kelondroHashtable(File file, int[] columns, int offset, int maxsize, int maxrehash, boolean exitOnFail) {
public kelondroHashtable(File file, kelondroRow rowdef, int offset, int maxsize, int maxrehash, boolean exitOnFail) {
// this creates a new hashtable
// the key element is not part of the columns array
// this is unlike the kelondroTree, where the key is part of a row
@ -152,14 +152,14 @@ public class kelondroHashtable {
// this number is needed to omit grow of the table in case of re-hashing
// the maxsize is re-computed to a virtual folding height and will result in a tablesize
// less than the given maxsize. The actual maxsize can be retrieved by maxsize()
this.hashArray = new kelondroFixedWidthArray(file, extCol(columns), 6, exitOnFail);
this.hashArray = new kelondroFixedWidthArray(file, extCol(rowdef), 6, exitOnFail);
this.offset = offset;
this.maxk = kelondroMSetTools.log2a(maxsize); // equal to |log2(maxsize)| + 1
if (this.maxk >= kelondroMSetTools.log2a(maxsize + power2(offset + 1) + 1) - 1) this.maxk--;
this.maxrehash = maxrehash;
dummyRow = this.hashArray.row().newEntry();
dummyRow.setCol(0, dummyKey);
for (int i = 0; i < hashArray.columns(); i++)
for (int i = 0; i < hashArray.row().columns(); i++)
try {
hashArray.seti(0, this.offset);
hashArray.seti(1, this.maxk);
@ -179,11 +179,11 @@ public class kelondroHashtable {
this.maxrehash = hashArray.geti(2);
}
private int[] extCol(int[] columns) {
int[] newCol = new int[columns.length + 1];
private kelondroRow extCol(kelondroRow rowdef) {
int[] newCol = new int[rowdef.columns() + 1];
newCol[0] = 4;
System.arraycopy(columns, 0, newCol, 1, columns.length);
return newCol;
for (int i = 0; i < rowdef.columns(); i++) newCol[i + 1] = rowdef.width(i);
return new kelondroRow(newCol);
}
public static int power2(int x) {
@ -219,7 +219,7 @@ public class kelondroHashtable {
// write row
kelondroRow.Entry newhkrow = hashArray.row().newEntry();
newhkrow.setCol(0, hash.key());
newhkrow.setColLongB256(0, hash.key());
newhkrow.setCol(1, rowentry.bytes());
hashArray.set(rowNumber, newhkrow);
return hashArray.row().newEntry(oldhkrow.getColBytes(1));

7
source/de/anomic/kelondro/kelondroIndex.java
Unescape View File

@ -54,12 +54,11 @@ import java.io.IOException;
public interface kelondroIndex {
public int columns();
public int columnSize(int column);
public kelondroRow row();
public kelondroRow.Entry get(byte[] key) throws IOException;
public byte[][] put(byte[][] row) throws IOException;
public byte[][] remove(byte[] key) throws IOException;
public kelondroRow.Entry put(kelondroRow.Entry row) throws IOException;
public kelondroRow.Entry remove(byte[] key) throws IOException;
//public Iterator rows(boolean up, boolean rotating, byte[] startKey) throws IOException; // Objects are of type byte[][]
//public Iterator keys(boolean up, boolean rotating, byte[] startKey) throws IOException; // Objects are of type String
//public TreeMap rowMap(boolean up, boolean rotating, byte[] firstKey, boolean including, int count) throws IOException;

2
source/de/anomic/kelondro/kelondroMap.java
Unescape View File

@ -130,7 +130,7 @@ public class kelondroMap {
}
public int keySize() {
return dyn.columnSize(0);
return dyn.row().width(0);
}
public int[] cacheNodeChunkSize() {

8
source/de/anomic/kelondro/kelondroMapTable.java
Unescape View File

@ -87,7 +87,7 @@ public class kelondroMapTable {
mTables.put(tablename, map);
}
public void declareTree(String tablename, int[] columns, long buffersize /*bytes*/, boolean exitOnFail) {
public void declareTree(String tablename, kelondroRow rowdef, long buffersize /*bytes*/, boolean exitOnFail) {
if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared already in other context.");
if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared twice.");
File tablefile = new File(tablesPath, "table." + tablename + ".tdb");
@ -96,10 +96,10 @@ public class kelondroMapTable {
Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) {
tablefile.getParentFile().mkdirs();
Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent, columns, exitOnFail);
Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent, rowdef, exitOnFail);
} else {
tablefile.getParentFile().mkdirs();
Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent, columns, exitOnFail);
Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent, rowdef, exitOnFail);
}
tTables.put(tablename, Tree);
}
@ -112,7 +112,7 @@ public class kelondroMapTable {
mTables.put(tablename, table);
}
public synchronized void update(String tablename, byte[][] row /* first element is the unique key = index */) throws IOException {
public synchronized void update(String tablename, kelondroRow.Entry row /* first element is the unique key = index */) throws IOException {
kelondroTree tree = (kelondroTree) tTables.get(tablename);
if (tree == null) throw new RuntimeException("kelondroTables.update: tree table '" + tablename + "' does not exist.");
tree.put(row);

14
source/de/anomic/kelondro/kelondroNaturalOrder.java
Unescape View File

@ -56,25 +56,25 @@ public class kelondroNaturalOrder extends kelondroAbstractOrder implements kelon
this.zero = null;
}
public Object clone() {
public final Object clone() {
kelondroNaturalOrder o = new kelondroNaturalOrder(this.asc);
o.rotate(this.zero);
return o;
}
public static kelondroOrder bySignature(String signature) {
public final static kelondroOrder bySignature(String signature) {
if (signature.equals("nd")) return new kelondroNaturalOrder(false);
if (signature.equals("nu")) return new kelondroNaturalOrder(true);
return null;
}
public String signature() {
public final String signature() {
if (!asc) return "nd";
if ( asc) return "nu";
return null;
}
private static long cardinalI(byte[] key) {
private final static long cardinalI(byte[] key) {
// returns a cardinal number in the range of 0 .. Long.MAX_VALUE
long c = 0;
int p = 0;
@ -84,7 +84,7 @@ public class kelondroNaturalOrder extends kelondroAbstractOrder implements kelon
return c;
}
public long cardinal(byte[] key) {
public final long cardinal(byte[] key) {
if (this.zero == null) return cardinalI(key);
long zeroCardinal = cardinalI(this.zero);
long keyCardinal = cardinalI(key);
@ -131,11 +131,11 @@ public class kelondroNaturalOrder extends kelondroAbstractOrder implements kelon
// is less than, equal to, or greater than the second.
// two arrays are also equal if one array is a subset of the other's array
// with filled-up char(0)-values
public int compare(byte[] a, byte[] b) {
public final int compare(byte[] a, byte[] b) {
return (asc) ? compare0(a, b) : compare0(b, a);
}
public int compare0(byte[] a, byte[] b) {
public final int compare0(byte[] a, byte[] b) {
if (zero == null) return compares(a, b);
// we have an artificial start point. check all combinations
int az = compares(a, zero); // -1 if a < z; 0 if a == z; 1 if a > z

39
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -193,7 +193,7 @@ public class kelondroRecords {
public kelondroRecords(File file, long buffersize /* bytes */,
short ohbytec, short ohhandlec,
int[] columns, int FHandles, int txtProps, int txtPropWidth,
kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth,
boolean exitOnFail) {
// creates a new file
// file: the file that shall be created
@ -208,7 +208,7 @@ public class kelondroRecords {
kelondroRA raf = new kelondroFileRA(this.filename);
// kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100);
// kelondroRA raf = new kelondroNIOFileRA(this.filename, false, 10000);
init(raf, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth, buffersize / 10);
init(raf, ohbytec, ohhandlec, rowdef, FHandles, txtProps, txtPropWidth, buffersize / 10);
} catch (IOException e) {
logFailure("cannot create / " + e.getMessage());
if (exitOnFail)
@ -219,11 +219,11 @@ public class kelondroRecords {
public kelondroRecords(kelondroRA ra, long buffersize /* bytes */,
short ohbytec, short ohhandlec,
int[] columns, int FHandles, int txtProps, int txtPropWidth,
kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth,
boolean exitOnFail) {
this.filename = null;
try {
init(ra, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth, buffersize / 10);
init(ra, ohbytec, ohhandlec, rowdef, FHandles, txtProps, txtPropWidth, buffersize / 10);
} catch (IOException e) {
logFailure("cannot create / " + e.getMessage());
if (exitOnFail) System.exit(-1);
@ -232,7 +232,7 @@ public class kelondroRecords {
}
private void init(kelondroRA ra, short ohbytec, short ohhandlec,
int[] columns, int FHandles, int txtProps, int txtPropWidth, long writeBufferSize) throws IOException {
kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth, long writeBufferSize) throws IOException {
// create new Chunked IO
if (useWriteBuffer) {
@ -242,16 +242,16 @@ public class kelondroRecords {
}
// create row
ROW = new kelondroRow(columns);
ROW = rowdef;
// store dynamic run-time data
this.overhead = ohbytec + 4 * ohhandlec;
this.recordsize = this.overhead + ROW.size();
this.headchunksize = overhead + columns[0];
this.recordsize = this.overhead + ROW.objectsize();
this.headchunksize = overhead + ROW.width(0);
this.tailchunksize = this.recordsize - this.headchunksize;
// store dynamic run-time seek pointers
POS_HANDLES = POS_COLWIDTHS + columns.length * 4;
POS_HANDLES = POS_COLWIDTHS + ROW.columns() * 4;
POS_TXTPROPS = POS_HANDLES + FHandles * 4;
POS_NODES = POS_TXTPROPS + txtProps * txtPropWidth;
@ -410,7 +410,7 @@ public class kelondroRecords {
// assign remaining values that are only present at run-time
this.overhead = OHBYTEC + 4 * OHHANDLEC;
this.recordsize = this.overhead;
this.recordsize = this.overhead + ROW.size();
this.recordsize = this.overhead + ROW.objectsize();
this.headchunksize = this.overhead + this.ROW.width(0);
this.tailchunksize = this.recordsize - this.headchunksize;
}
@ -751,13 +751,13 @@ public class kelondroRecords {
*/
public byte[] setValueRow(byte[] row) throws IOException {
// if the index is defined, then write values directly to the file, else only to the object
assert row.length == ROW.size();
assert row.length == ROW.objectsize();
byte[] result = getValueRow(); // previous value (this loads the values if not already happened)
// set values
if (this.handle.index != NUL) {
setValue(row, 0, ROW.width(0), headChunk, overhead);
if (ROW.columns() > 0) setValue(row, ROW.width(0), ROW.size() - ROW.width(0), tailChunk, 0);
if (ROW.columns() > 1) setValue(row, ROW.width(0), ROW.objectsize() - ROW.width(0), tailChunk, 0);
}
this.headChanged = true;
this.tailChanged = true;
@ -805,7 +805,7 @@ public class kelondroRecords {
}
// create return value
byte[] row = new byte[ROW.size()];
byte[] row = new byte[ROW.objectsize()];
// read key
System.arraycopy(headChunk, overhead, row, 0, ROW.width(0));
@ -1051,15 +1051,6 @@ public class kelondroRecords {
public final kelondroRow row() {
return this.ROW;
}
public final int columns() {
return this.ROW.columns();
}
public final int columnSize(int column) {
if ((column < 0) || (column >= this.ROW.columns())) return -1;
return ROW.width(column);
}
private final long seekpos(Handle handle) {
assert (handle.index >= 0): "handle index too low: " + handle.index;
@ -1263,8 +1254,8 @@ public class kelondroRecords {
System.out.println(" Data Offset: 0x" + Long.toHexString(POS_NODES));
System.out.println("--");
System.out.println("RECORDS");
System.out.print(" Columns : " + columns() + " columns {" + ROW.width(0));
for (int i = 1; i < columns(); i++) System.out.print(", " + ROW.width(i));
System.out.print(" Columns : " + row().columns() + " columns {" + ROW.width(0));
for (int i = 1; i < row().columns(); i++) System.out.print(", " + ROW.width(i));
System.out.println("}");
System.out.println(" Overhead : " + this.overhead + " bytes (" + OHBYTEC + " OH bytes, " + OHHANDLEC + " OH Handles)");
System.out.println(" Recordsize : " + this.recordsize + " bytes");

15
source/de/anomic/kelondro/kelondroRow.java
Unescape View File

@ -73,7 +73,7 @@ public class kelondroRow {
return this.row.length;
}
public int size() {
public int objectsize() {
return this.objectsize;
}
@ -164,6 +164,10 @@ public class kelondroRow {
return rowinstance;
}
public int columns() {
return row.length;
}
public boolean empty(int column) {
return rowinstance[colstart[column]] == 0;
}
@ -181,10 +185,15 @@ public class kelondroRow {
}
}
public void setCol(int column, long cell) {
public void setColLongB256(int column, long cell) {
kelondroNaturalOrder.encodeLong(cell, rowinstance, colstart[column], row[column].cellwidth());
}
public void setColLongB64E(int column, long cell) {
kelondroBase64Order.enhancedCoder.encodeLong(cell, rowinstance, colstart[column], row[column].cellwidth());
}
/*
public byte[][] getCols() {
byte[][] values = new byte[row.length][];
@ -203,7 +212,7 @@ public class kelondroRow {
return values;
}
*/
public String getColString(int column, String encoding) {
int length = row[column].cellwidth();
int offset = colstart[column];

30
source/de/anomic/kelondro/kelondroSplittedTree.java
Unescape View File

@ -75,25 +75,25 @@ public class kelondroSplittedTree implements kelondroIndex {
public kelondroSplittedTree(File pathToFiles, String filenameStub, kelondroOrder objectOrder,
long buffersize,
int forkfactor,
int[] columns,
kelondroRow rowdef,
int txtProps, int txtPropsWidth,
boolean exitOnFail) {
ktfs = new kelondroTree[forkfactor];
File f;
for (int i = 0; i < forkfactor; i++) {
f = dbFile(pathToFiles, filenameStub, forkfactor, columns.length, i);
f = dbFile(pathToFiles, filenameStub, forkfactor, rowdef.columns(), i);
if (f.exists()) {
try {
ktfs[i] = new kelondroTree(f, buffersize/forkfactor, kelondroTree.defaultObjectCachePercent);
this.order = ktfs[i].order();
} catch (IOException e) {
ktfs[i] = new kelondroTree(f, buffersize/forkfactor, kelondroTree.defaultObjectCachePercent,
columns, objectOrder, txtProps, txtPropsWidth, exitOnFail);
rowdef, objectOrder, txtProps, txtPropsWidth, exitOnFail);
this.order = objectOrder;
}
} else {
ktfs[i] = new kelondroTree(f, buffersize/forkfactor, kelondroTree.defaultObjectCachePercent,
columns, objectOrder, txtProps, txtPropsWidth, exitOnFail);
rowdef, objectOrder, txtProps, txtPropsWidth, exitOnFail);
this.order = objectOrder;
}
}
@ -113,27 +113,23 @@ public class kelondroSplittedTree implements kelondroIndex {
public static kelondroSplittedTree open(File pathToFiles, String filenameStub, kelondroOrder objectOrder,
long buffersize,
int forkfactor,
int[] columns, int txtProps, int txtPropsWidth,
kelondroRow rowdef, int txtProps, int txtPropsWidth,
boolean exitOnFail) throws IOException {
// generated a new splittet tree if it not exists or
// opens an existing one
if (existsAll(pathToFiles, filenameStub, forkfactor, columns.length)) {
return new kelondroSplittedTree(pathToFiles, filenameStub, objectOrder, buffersize, forkfactor, columns.length);
if (existsAll(pathToFiles, filenameStub, forkfactor, rowdef.columns())) {
return new kelondroSplittedTree(pathToFiles, filenameStub, objectOrder, buffersize, forkfactor, rowdef.columns());
} else {
return new kelondroSplittedTree(pathToFiles, filenameStub, objectOrder,
buffersize,
forkfactor,
columns, txtProps, txtPropsWidth,
rowdef, txtProps, txtPropsWidth,
exitOnFail);
}
}
public int columns() {
return ktfs[0].columns();
}
public int columnSize(int column) {
return ktfs[0].columnSize(column);
public kelondroRow row() {
return ktfs[0].row();
}
private int partition(byte[] key) {
@ -145,11 +141,11 @@ public class kelondroSplittedTree implements kelondroIndex {
return ktfs[partition(key)].get(key);
}
public byte[][] put(byte[][] row) throws IOException {
return ktfs[partition(row[0])].put(row);
public kelondroRow.Entry put(kelondroRow.Entry row) throws IOException {
return ktfs[partition(row.getColBytes(0))].put(row);
}
public byte[][] remove(byte[] key) throws IOException {
public kelondroRow.Entry remove(byte[] key) throws IOException {
return ktfs[partition(key)].remove(key);
}

18
source/de/anomic/kelondro/kelondroStack.java
Unescape View File

@ -68,12 +68,12 @@ public final class kelondroStack extends kelondroRecords {
private static int toor = 1; // pointer for FHandles-array: pointer to root node
public kelondroStack(File file, long buffersize, int key, int value, boolean exitOnFail) {
this(file, buffersize, new int[] { key, value }, exitOnFail);
this(file, buffersize, new kelondroRow(new int[] { key, value }), exitOnFail);
}
public kelondroStack(File file, long buffersize, int[] columns, boolean exitOnFail) {
public kelondroStack(File file, long buffersize, kelondroRow rowdef, boolean exitOnFail) {
// this creates a new stack
super(file, buffersize, thisOHBytes, thisOHHandles, columns, thisFHandles, columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
super(file, buffersize, thisOHBytes, thisOHHandles, rowdef, thisFHandles, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
try {
setHandle(root, null); // define the root value
setHandle(toor, null); // define the toor value
@ -107,7 +107,7 @@ public final class kelondroStack extends kelondroRecords {
if (f.exists()) f.delete();
// re-open a database with same settings as before
return new kelondroStack(f, bz, row.widths(), true);
return new kelondroStack(f, bz, row, true);
}
public class Counter implements Iterator {
@ -307,10 +307,10 @@ public final class kelondroStack extends kelondroRecords {
st = new StringTokenizer(s, separator);
// buffer the entry
c = 0;
while ((c < columns()) && (st.hasMoreTokens())) {
while ((c < row().columns()) && (st.hasMoreTokens())) {
buffer.setCol(c++, st.nextToken().trim().getBytes());
}
if ((st.hasMoreTokens()) || (c != columns())) {
if ((st.hasMoreTokens()) || (c != row().columns())) {
System.err.println("inapropriate number of entries in line " + line);
} else {
push(buffer);
@ -346,7 +346,7 @@ public final class kelondroStack extends kelondroRecords {
+ hp(n.getOHHandle(left)) + ", right "
+ hp(n.getOHHandle(right)));
System.out.print(" KEY:'" + r.getColString(0, null) + "'");
for (int j = 1; j < columns(); j++)
for (int j = 1; j < row().columns(); j++)
System.out.print(", V[" + j + "]:'" + r.getColString(j, null) + "'");
System.out.println();
}
@ -414,9 +414,7 @@ public final class kelondroStack extends kelondroRecords {
// create <keylen> <valuelen> <filename>
File f = new File(args[3]);
if (f.exists()) f.delete();
int[] lens = new int[2];
lens[0] = Integer.parseInt(args[1]);
lens[1] = Integer.parseInt(args[2]);
kelondroRow lens = new kelondroRow(new int[]{Integer.parseInt(args[1]), Integer.parseInt(args[2])});
kelondroStack fm = new kelondroStack(f, 0x100000, lens, true);
fm.close();
} else if (args[0].equals("-p")) {

65
source/de/anomic/kelondro/kelondroTree.java
Unescape View File

@ -93,19 +93,19 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
public kelondroTree(File file, long buffersize, int objectCachePercent, int key, int value, boolean exitOnFail) {
this(file, buffersize, objectCachePercent, new int[] { key, value }, new kelondroNaturalOrder(true), 1, 8, exitOnFail);
this(file, buffersize, objectCachePercent, new kelondroRow(new int[] { key, value }), new kelondroNaturalOrder(true), 1, 8, exitOnFail);
}
public kelondroTree(File file, long buffersize, int objectCachePercent, int[] columns, boolean exitOnFail) {
public kelondroTree(File file, long buffersize, int objectCachePercent, kelondroRow rowdef, boolean exitOnFail) {
// this creates a new tree file
this(file, buffersize, objectCachePercent, columns, new kelondroNaturalOrder(true), columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
this(file, buffersize, objectCachePercent, rowdef, new kelondroNaturalOrder(true), rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
}
public kelondroTree(File file, long buffersize, int objectCachePercent, int[] columns, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) {
public kelondroTree(File file, long buffersize, int objectCachePercent, kelondroRow rowdef, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) {
// this creates a new tree file
super(file,
(100 - objectCachePercent) * buffersize / 100,
thisOHBytes, thisOHHandles, columns,
thisOHBytes, thisOHHandles, rowdef,
thisFHandles, txtProps, txtPropsWidth, exitOnFail);
try {
setHandle(root, null); // define the root value
@ -120,16 +120,16 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
initObjectCache(buffersize, objectCachePercent);
}
public kelondroTree(kelondroRA ra, long buffersize, int objectCachePercent, int[] columns, boolean exitOnFail) {
public kelondroTree(kelondroRA ra, long buffersize, int objectCachePercent, kelondroRow rowdef, boolean exitOnFail) {
// this creates a new tree within a kelondroRA
this(ra, buffersize, objectCachePercent, columns, new kelondroNaturalOrder(true), columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
this(ra, buffersize, objectCachePercent, rowdef, new kelondroNaturalOrder(true), rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
}
public kelondroTree(kelondroRA ra, long buffersize, int objectCachePercent, int[] columns, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) {
public kelondroTree(kelondroRA ra, long buffersize, int objectCachePercent, kelondroRow rowdef, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) {
// this creates a new tree within a kelondroRA
super(ra,
(100 - objectCachePercent) * buffersize / 100,
thisOHBytes, thisOHHandles, columns,
thisOHBytes, thisOHHandles, rowdef,
thisFHandles, txtProps, txtPropsWidth, exitOnFail);
try {
setHandle(root, null); // define the root value
@ -171,7 +171,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
}
public final int cacheObjectChunkSize() {
return row().size() + 8 * super.columns();
return row().objectsize() + /* overhead */ 8 * super.row().columns();
}
public String[] cacheObjectStatus() {
@ -394,19 +394,18 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
}
// Associates the specified value with the specified key in this map
public byte[][] put(byte[][] newrow) throws IOException {
public kelondroRow.Entry put(kelondroRow.Entry newrow) throws IOException {
kelondroRow.Entry result = null;
//writeLock.stay(2000, 1000);
if (newrow.length != columns()) throw new IllegalArgumentException("put: wrong row length " + newrow.length + "; must be " + columns());
if (newrow.columns() != row().columns()) throw new IllegalArgumentException("put: wrong row length " + newrow.columns() + "; must be " + row().columns());
// first try to find the key element in the database
synchronized(writeSearchObj) {
kelondroRow.Entry newentry = row().newEntry(newrow);
if (objectCache != null) objectCache.put(newentry.getColBytes(0), newentry);
writeSearchObj.process(newrow[0]);
if (objectCache != null) objectCache.put(newrow.getColBytes(0), newrow);
writeSearchObj.process(newrow.getColBytes(0));
if (writeSearchObj.found()) {
// a node with this key exist. simply overwrite the content and return old content
Node e = writeSearchObj.getMatcher();
result = row().newEntry(e.setValueRow(newentry.bytes()));
result = row().newEntry(e.setValueRow(newrow.bytes()));
commitNode(e);
} else if (writeSearchObj.isRoot()) {
// a node with this key does not exist and there is no node at all
@ -415,7 +414,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
throw new kelondroException(filename, "tried to create root node twice");
// we dont have any Nodes in the file, so start here to create one
Node e = newNode();
e.setValueRow(newentry.bytes());
e.setValueRow(newrow.bytes());
// write the propetries
e.setOHByte(magic, (byte) 1);
e.setOHByte(balance, (byte) 0);
@ -437,7 +436,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
// create new node and assign values
Node parentNode = writeSearchObj.getParent();
Node theNode = newNode();
theNode.setValueRow(newentry.bytes());
theNode.setValueRow(newrow.bytes());
theNode.setOHByte(0, (byte) 1); // fresh magic
theNode.setOHByte(1, (byte) 0); // fresh balance
theNode.setOHHandle(parent, parentNode.handle());
@ -537,7 +536,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
}
}
//writeLock.release();
return (result == null) ? null : result.getCols();
return result;
}
private void assignChild(Node parentNode, Node childNode, int childType) throws IOException {
@ -656,15 +655,13 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
// Associates the specified value with the specified key in this map
public byte[] put(byte[] key, byte[] value) throws IOException {
byte[][] row = new byte[2][];
row[0] = key;
row[1] = value;
byte[][] ret = put(row);
if (ret == null) return null; else return ret[1];
kelondroRow.Entry row = row().newEntry(new byte[][]{key, value});
kelondroRow.Entry ret = put(row);
if (ret == null) return null; else return ret.getColBytes(0);
}
// Removes the mapping for this key from this map if present (optional operation).
public byte[][] remove(byte[] key) throws IOException {
public kelondroRow.Entry remove(byte[] key) throws IOException {
synchronized(writeSearchObj) {
if (objectCache != null) objectCache.remove(key);
writeSearchObj.process(key);
@ -672,7 +669,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
Node result = writeSearchObj.getMatcher();
kelondroRow.Entry values = row().newEntry(result.getValueRow());
remove(result, writeSearchObj.getParent());
return values.getCols();
return values;
} else {
return null;
}
@ -1029,7 +1026,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
Iterator i = (firstKey == null) ? new nodeIterator(up, rotating) : new nodeIterator(up, rotating, firstKey, including);
while ((rows.size() < count) && (i.hasNext())) {
n = (Node) i.next();
if (n != null) rows.put(new String(n.getKey()), row().newEntry(n.getValueRow()).getCols());
if (n != null) rows.put(new String(n.getKey()), row().newEntry(n.getValueRow()));
}
}
return rows;
@ -1196,7 +1193,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
String s;
StringTokenizer st;
int recs = 0;
byte[][] buffer = new byte[columns()][];
kelondroRow.Entry buffer = row().newEntry();
int c;
int line = 0;
while ((s = f.readLine()) != null) {
@ -1206,10 +1203,10 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
st = new StringTokenizer(s, separator);
// buffer the entry
c = 0;
while ((c < columns()) && (st.hasMoreTokens())) {
buffer[c++] = st.nextToken().trim().getBytes();
while ((c < row().columns()) && (st.hasMoreTokens())) {
buffer.setCol(c++, st.nextToken().trim().getBytes());
}
if ((st.hasMoreTokens()) || (c != columns())) {
if ((st.hasMoreTokens()) || (c != row().columns())) {
System.err.println("inapropriate number of entries in line " + line);
} else {
put(buffer);
@ -1253,7 +1250,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
Vector nextline;
Handle handle;
Node node;
int linelength, width = (1 << (height - 1)) * (columnSize(0) + 1);
int linelength, width = (1 << (height - 1)) * (row().width(0) + 1);
String key;
for (int h = 1; h < height; h++) {
linelength = width / (thisline.size() * 2);
@ -1399,9 +1396,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
// create <keylen> <valuelen> <filename>
File f = new File(args[3]);
if (f.exists()) f.delete();
int[] lens = new int[2];
lens[0] = Integer.parseInt(args[1]);
lens[1] = Integer.parseInt(args[2]);
kelondroRow lens = new kelondroRow(new int[]{Integer.parseInt(args[1]), Integer.parseInt(args[2])});
kelondroTree fm = new kelondroTree(f, 0x100000, 10, lens, true);
fm.close();
} else if (args[0].equals("-u")) {

5
source/de/anomic/plasma/plasmaCrawlBalancer.java
Unescape View File

@ -50,6 +50,7 @@ import java.util.ArrayList;
import de.anomic.index.indexURL;
import de.anomic.kelondro.kelondroRecords;
import de.anomic.kelondro.kelondroRow;
import de.anomic.kelondro.kelondroStack;
public class plasmaCrawlBalancer {
@ -62,10 +63,10 @@ public class plasmaCrawlBalancer {
try {
stack = new kelondroStack(stackFile, buffersize);
} catch (IOException e) {
stack = new kelondroStack(stackFile, buffersize, new int[] {indexURL.urlHashLength}, true);
stack = new kelondroStack(stackFile, buffersize, new kelondroRow(new int[] {indexURL.urlHashLength}), true);
}
} else {
stack = new kelondroStack(stackFile, buffersize, new int[] {indexURL.urlHashLength}, true);
stack = new kelondroStack(stackFile, buffersize, new kelondroRow(new int[] {indexURL.urlHashLength}), true);
}
domainStacks = new HashMap();
}

10
source/de/anomic/plasma/plasmaCrawlEURL.java
Unescape View File

@ -73,8 +73,8 @@ public class plasmaCrawlEURL extends indexURL {
urlNameLength, // the name of the url, from anchor tag <a>name</a>
urlDateLength, // the time when the url was first time appeared
urlDateLength, // the time when the url was last time tried to load
urlRetryLength, // number of load retries
urlErrorLength, // string describing load failure
urlRetryLength, // number of load retries
urlErrorLength, // string describing load failure
urlFlagLength // extra space
};
if (cachePath.exists()) try {
@ -82,11 +82,11 @@ public class plasmaCrawlEURL extends indexURL {
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) {
cachePath.delete();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true);
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true);
} else {
// create new cache
cachePath.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true);
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true);
}
}
@ -216,7 +216,7 @@ public class plasmaCrawlEURL extends indexURL {
this.failreason.getBytes(),
this.flags.getBytes()
};
urlHashCache.put(entry);
urlHashCache.put(urlHashCache.row().newEntry(entry));
} catch (IOException e) {
System.out.println("INTERNAL ERROR AT plasmaEURL:url2hash:" + e.toString());
}

8
source/de/anomic/plasma/plasmaCrawlLURL.java
Unescape View File

@ -119,12 +119,12 @@ public final class plasmaCrawlLURL extends indexURL {
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) {
cachePath.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true);
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true);
}
} else {
// create new cache
cachePath.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true);
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true);
}
// init result stacks
@ -575,7 +575,7 @@ public final class plasmaCrawlLURL extends indexURL {
kelondroBase64Order.enhancedCoder.encodeLong(size, urlSizeLength).getBytes(),
kelondroBase64Order.enhancedCoder.encodeLong(wordCount, urlWordCountLength).getBytes(),
};
urlHashCache.put(entry);
urlHashCache.put(urlHashCache.row().newEntry(entry));
serverLog.logFine("PLASMA","STORED new LURL " + url.toString());
this.stored = true;
} catch (Exception e) {
@ -846,7 +846,7 @@ public final class plasmaCrawlLURL extends indexURL {
if (res.statusCode == 200) {
entry.setCol(1, newUrl.toString().getBytes());
urlHashCache.put(entry.getCols());
urlHashCache.put(entry);
log.logInfo("UrlDB-Entry with urlHash '" + urlHash + "' corrected\n\tURL: " + oldUrlStr + " -> " + newUrlStr);
} else {
remove(urlHash);

19
source/de/anomic/plasma/plasmaCrawlNURL.java
Unescape View File

@ -124,26 +124,27 @@ public class plasmaCrawlNURL extends indexURL {
limitStack = new plasmaCrawlBalancer(limitStackFile, 0);
overhangStack = new plasmaCrawlBalancer(overhangStackFile, 0);
remoteStack = new plasmaCrawlBalancer(remoteStackFile, 0);
kelondroRow rowdef = new kelondroRow(new int[] {indexURL.urlHashLength});
if (imageStackFile.exists()) try {
imageStack = new kelondroStack(imageStackFile, 0);
} catch (IOException e) {
imageStack = new kelondroStack(imageStackFile, 0, new int[] {indexURL.urlHashLength}, true);
imageStack = new kelondroStack(imageStackFile, 0, rowdef, true);
} else {
imageStack = new kelondroStack(imageStackFile, 0, new int[] {indexURL.urlHashLength}, true);
imageStack = new kelondroStack(imageStackFile, 0, rowdef, true);
}
if (movieStackFile.exists()) try {
movieStack = new kelondroStack(movieStackFile, 0);
} catch (IOException e) {
movieStack = new kelondroStack(movieStackFile, 0, new int[] {indexURL.urlHashLength}, true);
movieStack = new kelondroStack(movieStackFile, 0, rowdef, true);
} else {
movieStack = new kelondroStack(movieStackFile, 0, new int[] {indexURL.urlHashLength}, true);
movieStack = new kelondroStack(movieStackFile, 0, rowdef, true);
}
if (musicStackFile.exists()) try {
musicStack = new kelondroStack(musicStackFile, 0);
} catch (IOException e) {
musicStack = new kelondroStack(musicStackFile, 0, new int[] {indexURL.urlHashLength}, true);
musicStack = new kelondroStack(musicStackFile, 0, rowdef, true);
} else {
musicStack = new kelondroStack(musicStackFile, 0, new int[] {indexURL.urlHashLength}, true);
musicStack = new kelondroStack(musicStackFile, 0, rowdef, true);
}
// init stack Index
@ -168,11 +169,11 @@ public class plasmaCrawlNURL extends indexURL {
urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) {
cacheFile.delete();
urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true);
urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true);
} else {
// create new cache
cacheFile.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true);
urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true);
}
}
@ -553,7 +554,7 @@ public class plasmaCrawlNURL extends indexURL {
this.flags.getBytes(),
normalizeHandle(this.handle).getBytes()
};
urlHashCache.put(entry);
urlHashCache.put(urlHashCache.row().newEntry(entry));
} catch (IOException e) {
serverLog.logSevere("PLASMA", "INTERNAL ERROR AT plasmaNURL:store:" + e.toString() + ", resetting NURL-DB");
e.printStackTrace();

41
source/de/anomic/plasma/plasmaCrawlStacker.java
Unescape View File

@ -62,6 +62,7 @@ import de.anomic.http.httpc;
import de.anomic.index.indexURL;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroRow;
import de.anomic.kelondro.kelondroTree;
import de.anomic.server.serverSemaphore;
import de.anomic.server.logging.serverLog;
@ -452,23 +453,23 @@ public final class plasmaCrawlStacker {
}
}
public stackCrawlMessage(String urlHash, byte[][] entryBytes) {
public stackCrawlMessage(String urlHash, kelondroRow.Entry entry) {
if (urlHash == null) throw new NullPointerException();
if (entryBytes == null) throw new NullPointerException();
if (entry == null) throw new NullPointerException();
try {
this.urlHash = urlHash;
this.initiator = new String(entryBytes[1], "UTF-8");
this.url = new String(entryBytes[2], "UTF-8").trim();
this.referrerHash = (entryBytes[3]==null) ? indexURL.dummyHash : new String(entryBytes[3], "UTF-8");
this.name = (entryBytes[4] == null) ? "" : new String(entryBytes[4], "UTF-8").trim();
this.loaddate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[5], "UTF-8")));
this.profileHandle = (entryBytes[6] == null) ? null : new String(entryBytes[6], "UTF-8").trim();
this.depth = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[7], "UTF-8"));
this.anchors = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[8], "UTF-8"));
this.forkfactor = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[9], "UTF-8"));
this.flags = new bitfield(entryBytes[10]);
this.handle = Integer.parseInt(new String(entryBytes[11], "UTF-8"));
this.initiator = entry.getColString(1, "UTF-8");
this.url = entry.getColString(2, "UTF-8").trim();
this.referrerHash = (entry.empty(3)) ? indexURL.dummyHash : entry.getColString(3, "UTF-8");
this.name = (entry.empty(4)) ? "" : entry.getColString(4, "UTF-8").trim();
this.loaddate = new Date(86400000 * entry.getColLongB64E(5));
this.profileHandle = (entry.empty(6)) ? null : entry.getColString(6, "UTF-8").trim();
this.depth = (int) entry.getColLongB64E(7);
this.anchors = (int) entry.getColLongB64E(8);
this.forkfactor = (int) entry.getColLongB64E(9);
this.flags = new bitfield(entry.getColBytes(10));
this.handle = Integer.parseInt(new String(entry.getColBytes(11), "UTF-8"));
} catch (Exception e) {
e.printStackTrace();
throw new IllegalStateException(e.toString());
@ -578,7 +579,7 @@ public final class plasmaCrawlStacker {
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) {
cacheFile.delete();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.ce, true);
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(plasmaCrawlNURL.ce), true);
}
try {
// loop through the list and fill the messageList with url hashs
@ -600,7 +601,7 @@ public final class plasmaCrawlStacker {
// deleting old db and creating a new db
try {this.urlEntryCache.close();}catch(Exception ex){}
cacheFile.delete();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.ce, true);
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(plasmaCrawlNURL.ce), true);
} catch (IOException e) {
/* if we have an error, we start with a fresh database */
plasmaCrawlStacker.this.log.logSevere("Unable to initialize crawl stacker queue, IOException:" + e.getMessage() + ". Reseting DB.\n",e);
@ -608,12 +609,12 @@ public final class plasmaCrawlStacker {
// deleting old db and creating a new db
try {this.urlEntryCache.close();}catch(Exception ex){}
cacheFile.delete();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.ce, true);
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(plasmaCrawlNURL.ce), true);
}
} else {
// create new cache
cacheFile.getParentFile().mkdirs();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.ce, true);
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(plasmaCrawlNURL.ce), true);
}
}
@ -634,7 +635,7 @@ public final class plasmaCrawlStacker {
boolean insertionDoneSuccessfully = false;
synchronized(this.urlEntryHashCache) {
byte[][] oldValue = this.urlEntryCache.put(newMessage.getBytes());
kelondroRow.Entry oldValue = this.urlEntryCache.put(this.urlEntryCache.row().newEntry(newMessage.getBytes()));
if (oldValue == null) {
insertionDoneSuccessfully = this.urlEntryHashCache.add(newMessage.urlHash);
}
@ -659,7 +660,7 @@ public final class plasmaCrawlStacker {
this.writeSync.P();
String urlHash = null;
byte[][] entryBytes = null;
kelondroRow.Entry entryBytes = null;
stackCrawlMessage newMessage = null;
try {
synchronized(this.urlEntryHashCache) {
@ -670,7 +671,7 @@ public final class plasmaCrawlStacker {
this.writeSync.V();
}
newMessage = new stackCrawlMessage(urlHash,entryBytes);
newMessage = new stackCrawlMessage(urlHash, entryBytes);
return newMessage;
}
}

26
source/de/anomic/plasma/plasmaSwitchboardQueue.java
Unescape View File

@ -80,22 +80,7 @@ public class plasmaSwitchboardQueue {
}
private void initQueueStack() {
if (sbQueueStackPath.exists()) try {
sbQueueStack = new kelondroStack(sbQueueStackPath, 0);
} catch (IOException e) {
sbQueueStackPath.delete();
sbQueueStack = new kelondroStack(sbQueueStackPath, 0, new int[] {
indexURL.urlStringLength,
indexURL.urlHashLength,
11,
1,
yacySeedDB.commonHashLength,
indexURL.urlCrawlDepthLength,
indexURL.urlCrawlProfileHandleLength,
indexURL.urlDescrLength
}, true);
} else {
sbQueueStack = new kelondroStack(sbQueueStackPath, 0, new int[] {
kelondroRow rowdef = new kelondroRow(new int[] {
indexURL.urlStringLength,
indexURL.urlHashLength,
11,
@ -104,7 +89,14 @@ public class plasmaSwitchboardQueue {
indexURL.urlCrawlDepthLength,
indexURL.urlCrawlProfileHandleLength,
indexURL.urlDescrLength
}, true);
});
if (sbQueueStackPath.exists()) try {
sbQueueStack = new kelondroStack(sbQueueStackPath, 0);
} catch (IOException e) {
sbQueueStackPath.delete();
sbQueueStack = new kelondroStack(sbQueueStackPath, 0, rowdef, true);
} else {
sbQueueStack = new kelondroStack(sbQueueStackPath, 0, rowdef, true);
}
}

12
source/de/anomic/plasma/plasmaWordConnotation.java
Unescape View File

@ -48,8 +48,8 @@ package de.anomic.plasma;
import java.io.File;
import java.io.IOException;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroDynTree;
import de.anomic.kelondro.kelondroRow;
public class plasmaWordConnotation {
@ -62,19 +62,19 @@ public class plasmaWordConnotation {
if (refDBfile.exists()) try {
refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, fillChar);
} catch (IOException e) {
refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, wordlength, nodesize, new int[] {wordlength, countlength}, fillChar, true);
refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, wordlength, nodesize, new kelondroRow(new int[] {wordlength, countlength}), fillChar, true);
} else {
refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, wordlength, nodesize, new int[] {wordlength, countlength}, fillChar, true);
refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, wordlength, nodesize, new kelondroRow(new int[] {wordlength, countlength}), fillChar, true);
}
}
private void addSingleRef(String word, String reference) throws IOException {
//word = word.toLowerCase();
//reference = reference.toLowerCase();
byte[][] record = refDB.get(word, reference.getBytes());
kelondroRow.Entry record = refDB.get(word, reference.getBytes());
long c;
if (record == null) c = 0; else c = kelondroBase64Order.enhancedCoder.decodeLong(new String(record[1], "UTF-8"));
record[1] = kelondroBase64Order.enhancedCoder.encodeLong(c++, countlength).getBytes();
if (record == null) c = 0; else c = record.getColLongB64E(1);
record.setColLongB64E(1, c++);
refDB.put(word, record);
}

48
source/de/anomic/plasma/plasmaWordIndexAssortment.java
Unescape View File

@ -61,7 +61,6 @@ import de.anomic.index.indexEntryAttribute;
import de.anomic.index.indexTreeMapContainer;
import de.anomic.index.indexURLEntry;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.kelondro.kelondroTree;
import de.anomic.kelondro.kelondroRow;
import de.anomic.server.logging.serverLog;
@ -84,7 +83,6 @@ public final class plasmaWordIndexAssortment {
private serverLog log;
private kelondroTree assortments;
private long bufferSize;
private int bufferStructureLength;
private static String intx(int x) {
String s = Integer.toString(x);
@ -93,22 +91,22 @@ public final class plasmaWordIndexAssortment {
}
private static int[] bufferStructure(int assortmentCapacity) {
int[] structure = new int[3 + 2 * assortmentCapacity];
structure[0] = bufferStructureBasis[0];
structure[1] = bufferStructureBasis[1];
structure[2] = bufferStructureBasis[2];
for (int i = 0; i < assortmentCapacity; i++) {
structure[3 + 2 * i] = bufferStructureBasis[3];
structure[4 + 2 * i] = bufferStructureBasis[4];
}
return structure;
int[] structure = new int[3 + 2 * assortmentCapacity];
structure[0] = bufferStructureBasis[0];
structure[1] = bufferStructureBasis[1];
structure[2] = bufferStructureBasis[2];
for (int i = 0; i < assortmentCapacity; i++) {
structure[3 + 2 * i] = bufferStructureBasis[3];
structure[4 + 2 * i] = bufferStructureBasis[4];
}
return structure;
}
public plasmaWordIndexAssortment(File storagePath, int assortmentLength, int bufferkb, serverLog log) {
if (!(storagePath.exists())) storagePath.mkdirs();
this.assortmentFile = new File(storagePath, assortmentFileName + intx(assortmentLength) + ".db");
this.assortmentLength = assortmentLength;
this.bufferStructureLength = 3 + 2 * assortmentLength;
if (!(storagePath.exists())) storagePath.mkdirs();
this.assortmentFile = new File(storagePath, assortmentFileName + intx(assortmentLength) + ".db");
this.assortmentLength = assortmentLength;
//this.bufferStructureLength = 3 + 2 * assortmentLength;
this.bufferSize = bufferkb * 1024;
this.log = log;
if (assortmentFile.exists()) {
@ -125,7 +123,7 @@ public final class plasmaWordIndexAssortment {
assortmentFile.delete(); // make space for new one
}
// create new assortment tree file
assortments = new kelondroTree(assortmentFile, bufferSize, kelondroTree.defaultObjectCachePercent, bufferStructure(assortmentLength), true);
assortments = new kelondroTree(assortmentFile, bufferSize, kelondroTree.defaultObjectCachePercent, new kelondroRow(bufferStructure(assortmentLength)), true);
if (log != null) log.logConfig("Created new Assortment Database, width " + assortmentLength + ", " + bufferkb + "kb buffer");
}
@ -134,18 +132,18 @@ public final class plasmaWordIndexAssortment {
// this throws an exception if the word hash already existed
//log.logDebug("storeAssortment: wordHash=" + wordHash + ", urlHash=" + entry.getUrlHash() + ", time=" + creationTime);
if (newContainer.size() != assortmentLength) throw new RuntimeException("plasmaWordIndexAssortment.store: wrong container size");
byte[][] row = new byte[this.bufferStructureLength][];
row[0] = newContainer.wordHash().getBytes();
row[1] = kelondroNaturalOrder.encodeLong(1, 4);
row[2] = kelondroNaturalOrder.encodeLong(newContainer.updated(), 8);
kelondroRow.Entry row = assortments.row().newEntry();
row.setCol(0, newContainer.wordHash().getBytes());
row.setColLongB256(1, 1);
row.setColLongB256(2, newContainer.updated());
Iterator entries = newContainer.entries();
indexURLEntry entry;
for (int i = 0; i < assortmentLength; i++) {
entry = (indexURLEntry) entries.next();
row[3 + 2 * i] = entry.getUrlHash().getBytes();
row[4 + 2 * i] = entry.toEncodedStringForm().getBytes();
row.setCol(3 + 2 * i, entry.getUrlHash().getBytes());
row.setCol(4 + 2 * i, entry.toEncodedStringForm().getBytes());
}
byte[][] oldrow = null;
kelondroRow.Entry oldrow = null;
try {
oldrow = assortments.put(row);
} catch (IOException e) {
@ -163,7 +161,7 @@ public final class plasmaWordIndexAssortment {
// and returns the content record
kelondroRow.Entry row = null;
try {
row = assortments.row().newEntry(assortments.remove(wordHash.getBytes()));
row = assortments.remove(wordHash.getBytes());
} catch (IOException e) {
log.logSevere("removeAssortment/IO-error: " + e.getMessage()
+ " - reset assortment-DB " + assortments.file(), e);
@ -245,7 +243,7 @@ public final class plasmaWordIndexAssortment {
if (!(assortmentFile.delete())) throw new RuntimeException("cannot delete assortment database");
}
if (assortmentFile.exists()) assortmentFile.delete();
assortments = new kelondroTree(assortmentFile, bufferSize, kelondroTree.defaultObjectCachePercent, bufferStructure(assortmentLength), true);
assortments = new kelondroTree(assortmentFile, bufferSize, kelondroTree.defaultObjectCachePercent, new kelondroRow(bufferStructure(assortmentLength)), true);
}
public Iterator hashes(String startWordHash, boolean up, boolean rot) throws IOException {

38
source/de/anomic/yacy/yacyNewsDB.java
Unescape View File

@ -50,6 +50,7 @@ import java.util.Iterator;
import de.anomic.yacy.yacyCore;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroColumn;
import de.anomic.kelondro.kelondroTree;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroRow;
@ -80,15 +81,17 @@ public class yacyNewsDB {
news = createDB(path, bufferkb);
}
}
public static final kelondroRow rowdef = new kelondroRow(new kelondroColumn[]{
new kelondroColumn(kelondroColumn.celltype_string, yacyNewsRecord.idLength(), "newsid", "id = created + originator"),
new kelondroColumn(kelondroColumn.celltype_string, yacyNewsRecord.categoryStringLength, "category", ""),
new kelondroColumn(kelondroColumn.celltype_string, yacyCore.universalDateShortPattern.length(), "received", ""),
new kelondroColumn(kelondroColumn.celltype_string, 2, "", ""),
new kelondroColumn(kelondroColumn.celltype_string, attributesMaxLength, "", ""),
});
private static kelondroTree createDB(File path, int bufferkb) {
return new kelondroTree(path, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new int[] {
yacyNewsRecord.idLength(), // id = created + originator
yacyNewsRecord.categoryStringLength, // category
yacyCore.universalDateShortPattern.length(), // received
2,
attributesMaxLength
}, true);
return new kelondroTree(path, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, rowdef, true);
}
private void resetDB() {
@ -153,7 +156,7 @@ public class yacyNewsDB {
}
public Object next() {
return b2r((byte[][]) rowIterator.next());
return b2r((kelondroRow.Entry) rowIterator.next());
}
public void remove() {
@ -181,6 +184,7 @@ public class yacyNewsDB {
);
}
/*
private static yacyNewsRecord b2r(byte[][] b) {
if (b == null) return null;
return new yacyNewsRecord(
@ -191,18 +195,18 @@ public class yacyNewsDB {
serverCodings.string2map(new String(b[4]))
);
}
private static byte[][] r2b(yacyNewsRecord r) {
*/
private kelondroRow.Entry r2b(yacyNewsRecord r) {
if (r == null) return null;
String attributes = r.attributes().toString();
if (attributes.length() > attributesMaxLength) throw new IllegalArgumentException("attribute length=" + attributes.length() + " exceeds maximum size=" + attributesMaxLength);
byte[][] b = new byte[5][];
b[0] = r.id().getBytes();
b[1] = r.category().getBytes();
b[2] = (r.received() == null) ? null : yacyCore.universalDateShortString(r.received()).getBytes();
b[3] = kelondroBase64Order.enhancedCoder.encodeLong(r.distributed(), 2).getBytes();
b[4] = attributes.getBytes();
return b;
kelondroRow.Entry entry = news.row().newEntry();
entry.setCol(0, r.id().getBytes());
entry.setCol(1, r.category().getBytes());
entry.setCol(2, (r.received() == null) ? null : yacyCore.universalDateShortString(r.received()).getBytes());
entry.setCol(3, kelondroBase64Order.enhancedCoder.encodeLong(r.distributed(), 2).getBytes());
entry.setCol(4, attributes.getBytes());
return entry;
}
}

11
source/de/anomic/yacy/yacyNewsQueue.java
Unescape View File

@ -48,6 +48,7 @@ import java.io.File;
import java.io.IOException;
import java.util.Date;
import de.anomic.kelondro.kelondroColumn;
import de.anomic.kelondro.kelondroRow;
import de.anomic.kelondro.kelondroStack;
import de.anomic.kelondro.kelondroException;
@ -74,12 +75,14 @@ public class yacyNewsQueue {
queueStack = createStack(path);
}
}
public static final kelondroRow rowdef = new kelondroRow(new kelondroColumn[]{
new kelondroColumn(kelondroColumn.celltype_string, yacyNewsRecord.idLength(), "newsid", "id = created + originator"),
new kelondroColumn(kelondroColumn.celltype_string, yacyCore.universalDateShortPattern.length(), "last touched", "")
});
private static kelondroStack createStack(File path) {
return new kelondroStack(path, 0, new int[] {
yacyNewsRecord.idLength(), // id = created + originator
yacyCore.universalDateShortPattern.length() // last touched
}, true);
return new kelondroStack(path, 0, rowdef, true);
}
private void resetDB() {

Write Preview
Loading…
Cancel
Save