boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

full integration of kelondroRow

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2167 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent 6e49721762
commit c36e9fc8d3
31 changed files with 343 additions and 370 deletions
Show Stats Download Patch File Download Diff File

28
source/dbtest.java
Unescape View File

@ -116,7 +116,7 @@ public class dbtest {
public void run() { public void run() {
final STEntry entry = new STEntry(this.getSource()); final STEntry entry = new STEntry(this.getSource());
try { try {
getTable().put(new byte[][] { entry.getKey(), entry.getValue() , entry.getValue() }); getTable().put(getTable().row().newEntry(new byte[][] { entry.getKey(), entry.getValue() , entry.getValue() }));
} catch (IOException e) { } catch (IOException e) {
System.err.println(e); System.err.println(e);
e.printStackTrace(); e.printStackTrace();
@ -166,12 +166,13 @@ public class dbtest {
profiler.start(); profiler.start();
// create the database access // create the database access
kelondroRow testRow = new kelondroRow(new int[]{keylength, valuelength, valuelength});
if (dbe.equals("kelondroTree")) { if (dbe.equals("kelondroTree")) {
File tablefile = new File(tablename + ".kelondro.db"); File tablefile = new File(tablename + ".kelondro.db");
if (tablefile.exists()) { if (tablefile.exists()) {
table = new kelondroTree(tablefile, buffer, kelondroTree.defaultObjectCachePercent); table = new kelondroTree(tablefile, buffer, kelondroTree.defaultObjectCachePercent);
} else { } else {
table = new kelondroTree(tablefile, buffer, kelondroTree.defaultObjectCachePercent, new int[]{keylength, valuelength, valuelength}, true); table = new kelondroTree(tablefile, buffer, kelondroTree.defaultObjectCachePercent, testRow, true);
} }
} }
if (dbe.equals("kelondroSplittedTree")) { if (dbe.equals("kelondroSplittedTree")) {
@ -179,7 +180,7 @@ public class dbtest {
table = kelondroSplittedTree.open(tablepath, tablename, kelondroBase64Order.enhancedCoder, table = kelondroSplittedTree.open(tablepath, tablename, kelondroBase64Order.enhancedCoder,
buffer, buffer,
8, 8,
new int[]{keylength, valuelength, valuelength}, 1, 80, testRow, 1, 80,
true); true);
} }
if (dbe.equals("kelondroFlexTable")) { if (dbe.equals("kelondroFlexTable")) {
@ -187,11 +188,11 @@ public class dbtest {
table = new kelondroFlexTable(tablepath, new File(tablename).getName(), new kelondroRow(new int[]{keylength, valuelength, valuelength}), true); table = new kelondroFlexTable(tablepath, new File(tablename).getName(), new kelondroRow(new int[]{keylength, valuelength, valuelength}), true);
} }
if (dbe.equals("mysql")) { if (dbe.equals("mysql")) {
table = new dbTable("mysql"); table = new dbTable("mysql", testRow);
} }
if (dbe.equals("pgsql")) { if (dbe.equals("pgsql")) {
table = new dbTable("pgsql"); table = new dbTable("pgsql", testRow);
} }
long afterinit = System.currentTimeMillis(); long afterinit = System.currentTimeMillis();
@ -210,7 +211,7 @@ public class dbtest {
long randomstart = Long.parseLong(args[4]); long randomstart = Long.parseLong(args[4]);
Random random = new Random(randomstart); Random random = new Random(randomstart);
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
table.put(new byte[][]{randomHash(random), dummyvalue1, dummyvalue2}); table.put(table.row().newEntry(new byte[][]{randomHash(random), dummyvalue1, dummyvalue2}));
if (i % 500 == 0) { if (i % 500 == 0) {
System.out.println(i + " entries processed so far."); System.out.println(i + " entries processed so far.");
} }
@ -305,8 +306,10 @@ final class dbTable implements kelondroIndex {
private PreparedStatement sqlStatement; private PreparedStatement sqlStatement;
private int commandCount = 0; private int commandCount = 0;
private int batchlimit = 1; private int batchlimit = 1;
private kelondroRow rowdef;
public dbTable(String dbType) throws Exception { public dbTable(String dbType, kelondroRow rowdef) throws Exception {
this.rowdef = rowdef;
openDatabaseConnection(dbType); openDatabaseConnection(dbType);
} }
@ -357,15 +360,18 @@ final class dbTable implements kelondroIndex {
} }
} }
public kelondroRow row() {
return this.rowdef;
}
public kelondroRow.Entry get(byte[] key) throws IOException { public kelondroRow.Entry get(byte[] key) throws IOException {
return null; return null;
} }
public byte[][] put(byte[][] row) throws IOException { public kelondroRow.Entry put(kelondroRow.Entry row) throws IOException {
try { try {
this.sqlStatement.setString(1,new String(row[0])); this.sqlStatement.setString(1, new String(row.getColString(0, null)));
sqlStatement.setBytes(2,row[1]); sqlStatement.setBytes(2, row.getColBytes(1));
sqlStatement.addBatch(); sqlStatement.addBatch();
commandCount++; commandCount++;
@ -380,7 +386,7 @@ final class dbTable implements kelondroIndex {
} }
} }
public byte[][] remove(byte[] key) throws IOException { public kelondroRow.Entry remove(byte[] key) throws IOException {
// TODO Auto-generated method stub // TODO Auto-generated method stub
return null; return null;
} }

2
source/de/anomic/index/indexRAMCacheRI.java
Unescape View File

@ -94,7 +94,7 @@ public final class indexRAMCacheRI extends indexAbstractRI implements indexRI {
File indexDumpFile = new File(databaseRoot, indexArrayFileName); File indexDumpFile = new File(databaseRoot, indexArrayFileName);
if (indexDumpFile.exists()) indexDumpFile.delete(); if (indexDumpFile.exists()) indexDumpFile.delete();
kelondroFixedWidthArray dumpArray = null; kelondroFixedWidthArray dumpArray = null;
dumpArray = new kelondroFixedWidthArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0, false); dumpArray = new kelondroFixedWidthArray(indexDumpFile, new kelondroRow(plasmaWordIndexAssortment.bufferStructureBasis), 0, false);
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
long messageTime = System.currentTimeMillis() + 5000; long messageTime = System.currentTimeMillis() + 5000;
long wordsPerSecond = 0, wordcount = 0, urlcount = 0; long wordsPerSecond = 0, wordcount = 0, urlcount = 0;

2
source/de/anomic/kelondro/kelondroArray.java
Unescape View File

@ -30,7 +30,7 @@ public interface kelondroArray {
public int size(); public int size();
public int columns(); public kelondroRow row();
public kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException; public kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException;

42
source/de/anomic/kelondro/kelondroBase64Order.java
Unescape View File

@ -88,7 +88,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
return o; return o;
} }
public static kelondroOrder bySignature(String signature) { public final static kelondroOrder bySignature(String signature) {
if (signature.equals("Bd")) return new kelondroBase64Order(false, false); if (signature.equals("Bd")) return new kelondroBase64Order(false, false);
if (signature.equals("bd")) return new kelondroBase64Order(false, true); if (signature.equals("bd")) return new kelondroBase64Order(false, true);
if (signature.equals("Bu")) return new kelondroBase64Order(true, false); if (signature.equals("Bu")) return new kelondroBase64Order(true, false);
@ -96,7 +96,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
return null; return null;
} }
public String signature() { public final String signature() {
if ((!asc) && (!rfc1113compliant)) return "Bd"; if ((!asc) && (!rfc1113compliant)) return "Bd";
if ((!asc) && ( rfc1113compliant)) return "bd"; if ((!asc) && ( rfc1113compliant)) return "bd";
if (( asc) && (!rfc1113compliant)) return "Bu"; if (( asc) && (!rfc1113compliant)) return "Bu";
@ -104,15 +104,15 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
return null; return null;
} }
public char encodeByte(byte b) { public final char encodeByte(byte b) {
return (char) alpha[b]; return (char) alpha[b];
} }
public byte decodeByte(char b) { public final byte decodeByte(char b) {
return ahpla[b]; return ahpla[b];
} }
public String encodeLongSmart(long c, int length) { public final String encodeLongSmart(long c, int length) {
if (c >= max(length)) { if (c >= max(length)) {
StringBuffer s = new StringBuffer(length); StringBuffer s = new StringBuffer(length);
s.setLength(length); s.setLength(length);
@ -123,7 +123,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
} }
} }
public String encodeLong(long c, int length) { public final String encodeLong(long c, int length) {
StringBuffer s = new StringBuffer(length); StringBuffer s = new StringBuffer(length);
s.setLength(length); s.setLength(length);
while (length > 0) { while (length > 0) {
@ -133,14 +133,22 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
return s.toString(); return s.toString();
} }
public long decodeLong(String s) { public final void encodeLong(long c, byte[] b, int offset, int length) {
assert offset + length <= b.length;
while (length > 0) {
b[--length + offset] = (byte) alpha[(byte) (c & 0x3F)];
c >>= 6;
}
}
public final long decodeLong(String s) {
while (s.endsWith("=")) s = s.substring(0, s.length() - 1); while (s.endsWith("=")) s = s.substring(0, s.length() - 1);
long c = 0; long c = 0;
for (int i = 0; i < s.length(); i++) c = (c << 6) | ahpla[s.charAt(i)]; for (int i = 0; i < s.length(); i++) c = (c << 6) | ahpla[s.charAt(i)];
return c; return c;
} }
public long decodeLong(byte[] s, int offset, int length) { public final long decodeLong(byte[] s, int offset, int length) {
while ((length > 0) && (s[offset + length - 1] == '=')) length--; while ((length > 0) && (s[offset + length - 1] == '=')) length--;
long c = 0; long c = 0;
for (int i = 0; i < length; i++) c = (c << 6) | ahpla[s[offset + i]]; for (int i = 0; i < length; i++) c = (c << 6) | ahpla[s[offset + i]];
@ -155,14 +163,14 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
return c; return c;
} }
public String encodeString(String in) { public final String encodeString(String in) {
return encode(in.getBytes()); return encode(in.getBytes());
} }
// we will use this encoding to encode strings with 2^8 values to // we will use this encoding to encode strings with 2^8 values to
// b64-Strings // b64-Strings
// we will do that by grouping each three input bytes to four output bytes. // we will do that by grouping each three input bytes to four output bytes.
public String encode(byte[] in) { public final String encode(byte[] in) {
StringBuffer out = new StringBuffer(in.length / 3 * 4 + 3); StringBuffer out = new StringBuffer(in.length / 3 * 4 + 3);
int pos = 0; int pos = 0;
long l; long l;
@ -181,7 +189,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
return out.toString(); return out.toString();
} }
public String decodeString(String in) { public final String decodeString(String in) {
try { try {
//return new String(decode(in), "ISO-8859-1"); //return new String(decode(in), "ISO-8859-1");
return new String(decode(in), "UTF-8"); return new String(decode(in), "UTF-8");
@ -191,7 +199,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
} }
} }
public byte[] decode(String in) { public final byte[] decode(String in) {
try { try {
int posIn = 0; int posIn = 0;
int posOut = 0; int posOut = 0;
@ -233,7 +241,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
} }
} }
private long cardinalI(byte[] key) { private final long cardinalI(byte[] key) {
// returns a cardinal number in the range of 0 .. Long.MAX_VALUE // returns a cardinal number in the range of 0 .. Long.MAX_VALUE
long c = 0; long c = 0;
int p = 0; int p = 0;
@ -243,7 +251,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
return c; return c;
} }
public long cardinal(byte[] key) { public final long cardinal(byte[] key) {
if (this.zero == null) return cardinalI(key); if (this.zero == null) return cardinalI(key);
long zeroCardinal = cardinalI(this.zero); long zeroCardinal = cardinalI(this.zero);
long keyCardinal = cardinalI(key); long keyCardinal = cardinalI(key);
@ -251,11 +259,11 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
return Long.MAX_VALUE - keyCardinal + zeroCardinal + 1; return Long.MAX_VALUE - keyCardinal + zeroCardinal + 1;
} }
public int compare(byte[] a, byte[] b) { public final int compare(byte[] a, byte[] b) {
return (asc) ? compare0(a, b) : compare0(b, a); return (asc) ? compare0(a, b) : compare0(b, a);
} }
public int compare0(byte[] a, byte[] b) { public final int compare0(byte[] a, byte[] b) {
if (zero == null) return compares(a, b); if (zero == null) return compares(a, b);
// we have an artificial start point. check all combinations // we have an artificial start point. check all combinations
int az = compares(a, zero); // -1 if a < z; 0 if a == z; 1 if a > z int az = compares(a, zero); // -1 if a < z; 0 if a == z; 1 if a > z
@ -267,7 +275,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
return bz; return bz;
} }
public int compares(byte[] a, byte[] b) { public final int compares(byte[] a, byte[] b) {
int i = 0; int i = 0;
final int al = a.length; final int al = a.length;
final int bl = b.length; final int bl = b.length;

45
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -72,7 +72,7 @@ public class kelondroCollectionIndex {
columns[2] = 4; // chunkcount (number of chunks in this collection) columns[2] = 4; // chunkcount (number of chunks in this collection)
columns[3] = 4; // index (position in index file) columns[3] = 4; // index (position in index file)
columns[4] = 2; // update time in days since 1.1.2000 columns[4] = 2; // update time in days since 1.1.2000
index = new kelondroSplittedTree(path, filenameStub, indexOrder, buffersize, 8, columns, 1, 80, true); index = new kelondroSplittedTree(path, filenameStub, indexOrder, buffersize, 8, new kelondroRow(columns), 1, 80, true);
// create array files // create array files
this.array = new kelondroFixedWidthArray[partitions]; this.array = new kelondroFixedWidthArray[partitions];
@ -97,13 +97,13 @@ public class kelondroCollectionIndex {
} else { } else {
int load = 1; for (int i = 0; i < partitionNumber; i++) load = load * loadfactor; int load = 1; for (int i = 0; i < partitionNumber; i++) load = load * loadfactor;
int[] columns = new int[4]; int[] columns = new int[4];
columns[0] = index.columnSize(0); // add always the key columns[0] = index.row().width(0); // add always the key
columns[1] = 4; // chunkcount (raw format) columns[1] = 4; // chunkcount (raw format)
columns[2] = 2; // last time read columns[2] = 2; // last time read
columns[3] = 2; // last time wrote columns[3] = 2; // last time wrote
columns[4] = 2; // flag string, assigns collection order as currently stored in table columns[4] = 2; // flag string, assigns collection order as currently stored in table
columns[5] = load * genericChunkSize; columns[5] = load * genericChunkSize;
return new kelondroFixedWidthArray(f, columns, 0, true); return new kelondroFixedWidthArray(f, new kelondroRow(columns), 0, true);
} }
} }
@ -134,12 +134,13 @@ public class kelondroCollectionIndex {
// write a new entry in this array // write a new entry in this array
int newRowNumber = array[part].add(array[part].row().newEntry(newarrayrow)); int newRowNumber = array[part].add(array[part].row().newEntry(newarrayrow));
// store the new row number in the index // store the new row number in the index
index.put(new byte[][]{key, kelondroRow.Entry e = index.row().newEntry();
kelondroNaturalOrder.encodeLong(this.chunksize, 4), e.setCol(0, key);
kelondroNaturalOrder.encodeLong(collection.size(), 4), e.setColLongB256(1, this.chunksize);
kelondroNaturalOrder.encodeLong((long) newRowNumber, 4), e.setColLongB256(2, collection.size());
kelondroNaturalOrder.encodeLong(daysSince2000(System.currentTimeMillis()), 2) e.setColLongB256(3, (long) newRowNumber);
}); e.setColLongB256(4, daysSince2000(System.currentTimeMillis()));
index.put(e);
} else { } else {
// overwrite the old collection // overwrite the old collection
// read old information // read old information
@ -154,24 +155,26 @@ public class kelondroCollectionIndex {
// we don't need a new slot, just write in the old one // we don't need a new slot, just write in the old one
array[oldPartitionNumber].set(rownumber, array[oldPartitionNumber].row().newEntry(newarrayrow)); array[oldPartitionNumber].set(rownumber, array[oldPartitionNumber].row().newEntry(newarrayrow));
// update the index entry // update the index entry
index.put(new byte[][]{key, kelondroRow.Entry e = index.row().newEntry();
kelondroNaturalOrder.encodeLong(this.chunksize, 4), e.setCol(0, key);
kelondroNaturalOrder.encodeLong(collection.size(), 4), e.setColLongB256(1, this.chunksize);
kelondroNaturalOrder.encodeLong((long) rownumber, 4), e.setColLongB256(2, collection.size());
kelondroNaturalOrder.encodeLong(daysSince2000(System.currentTimeMillis()), 2) e.setColLongB256(3, (long) rownumber);
}); e.setColLongB256(4, daysSince2000(System.currentTimeMillis()));
index.put(e);
} else { } else {
// we need a new slot, that means we must first delete the old entry // we need a new slot, that means we must first delete the old entry
array[oldPartitionNumber].remove(rownumber); array[oldPartitionNumber].remove(rownumber);
// write a new entry in the other array // write a new entry in the other array
int newRowNumber = array[newPartitionNumber].add(array[newPartitionNumber].row().newEntry(newarrayrow)); int newRowNumber = array[newPartitionNumber].add(array[newPartitionNumber].row().newEntry(newarrayrow));
// store the new row number in the index // store the new row number in the index
index.put(new byte[][]{key, kelondroRow.Entry e = index.row().newEntry();
kelondroNaturalOrder.encodeLong(this.chunksize, 4), e.setCol(0, key);
kelondroNaturalOrder.encodeLong(collection.size(), 4), e.setColLongB256(1, this.chunksize);
kelondroNaturalOrder.encodeLong((long) newRowNumber, 4), e.setColLongB256(2, collection.size());
kelondroNaturalOrder.encodeLong(daysSince2000(System.currentTimeMillis()), 2) e.setColLongB256(3, (long) newRowNumber);
}); e.setColLongB256(4, daysSince2000(System.currentTimeMillis()));
index.put(e);
} }
} }
} }

16
source/de/anomic/kelondro/kelondroDyn.java
Unescape View File

@ -76,9 +76,9 @@ public class kelondroDyn extends kelondroTree {
int nodesize, char fillChar, kelondroOrder objectOrder, int nodesize, char fillChar, kelondroOrder objectOrder,
boolean exitOnFail) { boolean exitOnFail) {
// creates a new dynamic tree // creates a new dynamic tree
super(file, buffersize, kelondroTree.defaultObjectCachePercent, new int[] { key + counterlen, nodesize }, objectOrder, 1, 8, exitOnFail); super(file, buffersize, kelondroTree.defaultObjectCachePercent, new kelondroRow(new int[] { key + counterlen, nodesize }), objectOrder, 1, 8, exitOnFail);
this.keylen = columnSize(0) - counterlen; this.keylen = row().width(0) - counterlen;
this.reclen = columnSize(1); this.reclen = row().width(1);
this.fillChar = fillChar; this.fillChar = fillChar;
this.segmentCount = 0; this.segmentCount = 0;
writeSegmentCount(); writeSegmentCount();
@ -88,8 +88,8 @@ public class kelondroDyn extends kelondroTree {
public kelondroDyn(File file, long buffersize, char fillChar) throws IOException { public kelondroDyn(File file, long buffersize, char fillChar) throws IOException {
// this opens a file with an existing dynamic tree // this opens a file with an existing dynamic tree
super(file, buffersize, kelondroTree.defaultObjectCachePercent); super(file, buffersize, kelondroTree.defaultObjectCachePercent);
this.keylen = columnSize(0) - counterlen; this.keylen = row().width(0) - counterlen;
this.reclen = columnSize(1); this.reclen = row().width(1);
this.fillChar = fillChar; this.fillChar = fillChar;
this.segmentCount = 0; this.segmentCount = 0;
buffer = new kelondroObjectBuffer(file.toString()); buffer = new kelondroObjectBuffer(file.toString());
@ -159,11 +159,11 @@ public class kelondroDyn extends kelondroTree {
String k; String k;
String v; String v;
int c; int c;
byte[][] nt; kelondroRow.Entry nt;
while (ri.hasNext()) { while (ri.hasNext()) {
nt = (byte[][]) ri.next(); nt = (kelondroRow.Entry) ri.next();
if (nt == null) throw new kelondroException(filename, "no more elements available"); if (nt == null) throw new kelondroException(filename, "no more elements available");
g = nt[0]; g = nt.getColBytes(0);
if (g == null) return null; if (g == null) return null;
k = new String(g, 0, keylen); k = new String(g, 0, keylen);
v = new String(g, keylen, counterlen); v = new String(g, keylen, counterlen);

75
source/de/anomic/kelondro/kelondroDynTree.java
Unescape View File

@ -53,7 +53,7 @@ import java.util.Iterator;
public class kelondroDynTree { public class kelondroDynTree {
// basic data structures // basic data structures
private int[] columns; private kelondroRow rowdef;
private kelondroDyn table; private kelondroDyn table;
private Hashtable treeRAHandles; private Hashtable treeRAHandles;
private File file; private File file;
@ -69,10 +69,10 @@ public class kelondroDynTree {
private Hashtable buffer, cache; private Hashtable buffer, cache;
private long cycleBuffer; private long cycleBuffer;
public kelondroDynTree(File file, long buffersize, int keylength, int nodesize, int[] columns, char fillChar, boolean exitOnFail) { public kelondroDynTree(File file, long buffersize, int keylength, int nodesize, kelondroRow rowdef, char fillChar, boolean exitOnFail) {
// creates a new DynTree // creates a new DynTree
this.file = file; this.file = file;
this.columns = columns; this.rowdef = rowdef;
this.buffer = new Hashtable(); this.buffer = new Hashtable();
this.cache = new Hashtable(); this.cache = new Hashtable();
//this.cycleCache = Long.MIN_VALUE; //this.cycleCache = Long.MIN_VALUE;
@ -97,8 +97,9 @@ public class kelondroDynTree {
Iterator i = table.dynKeys(true, false); Iterator i = table.dynKeys(true, false);
String onekey = (String) i.next(); String onekey = (String) i.next();
kelondroTree onetree = getTree(onekey); kelondroTree onetree = getTree(onekey);
this.columns = new int[onetree.columns()]; int[] columns = new int[onetree.row().columns()];
for (int j = 0; j < columns.length; j++) columns[j] = onetree.columnSize(j); for (int j = 0; j < columns.length; j++) columns[j] = onetree.row().width(j);
this.rowdef = new kelondroRow(columns);
closeTree(onekey); closeTree(onekey);
} }
@ -109,15 +110,6 @@ public class kelondroDynTree {
table.close(); table.close();
if (size == 0) this.file.delete(); if (size == 0) this.file.delete();
} }
/*
public void setReadCacheAttr(int maxcount, int maxsize, long maxage, long cycletime) {
maxcountCache = maxcount;
maxsizeCache = maxsize;
maxageCache = maxage;
cycletimeCache = cycletime;
}
*/
public void setWriteBufferAttr(int maxcount, int maxsize, long maxage, long cycletime) { public void setWriteBufferAttr(int maxcount, int maxsize, long maxage, long cycletime) {
maxcountBuffer = maxcount; maxcountBuffer = maxcount;
@ -135,7 +127,7 @@ public class kelondroDynTree {
kelondroRA ra = table.getRA(key); // works always, even with no-existing entry kelondroRA ra = table.getRA(key); // works always, even with no-existing entry
treeRAHandles.put(key, ra); treeRAHandles.put(key, ra);
try { try {
return new kelondroTree(ra, buffersize, kelondroTree.defaultObjectCachePercent, columns, false); return new kelondroTree(ra, buffersize, kelondroTree.defaultObjectCachePercent, rowdef, false);
} catch (RuntimeException e) { } catch (RuntimeException e) {
throw new IOException(e.getMessage()); throw new IOException(e.getMessage());
} }
@ -183,11 +175,11 @@ public class kelondroDynTree {
this.timestamp = Long.MAX_VALUE; // to flag no-update this.timestamp = Long.MAX_VALUE; // to flag no-update
} }
public byte[][] get(byte[] key) throws IOException { public kelondroRow.Entry get(byte[] key) throws IOException {
byte[][] entry = (byte[][]) tcache.get(key); kelondroRow.Entry entry = (kelondroRow.Entry) tcache.get(key);
if (entry == null) { if (entry == null) {
kelondroTree t = getTree(this.tablename); kelondroTree t = getTree(this.tablename);
entry = t.get(key).getCols(); entry = t.get(key);
t.close(); t.close();
this.tcache.put(key, entry); this.tcache.put(key, entry);
this.timestamp = System.currentTimeMillis(); this.timestamp = System.currentTimeMillis();
@ -195,8 +187,8 @@ public class kelondroDynTree {
return entry; return entry;
} }
protected void put(byte[][] entry) { // this is only used internal protected void put(kelondroRow.Entry entry) { // this is only used internal
this.tcache.put(entry[0], entry); this.tcache.put(entry.getColBytes(0), entry);
this.timestamp = System.currentTimeMillis(); this.timestamp = System.currentTimeMillis();
} }
@ -218,8 +210,8 @@ public class kelondroDynTree {
this.timestamp = Long.MAX_VALUE; // to flag no-update this.timestamp = Long.MAX_VALUE; // to flag no-update
} }
public void put(byte[][] entry) { public void put(kelondroRow.Entry entry) {
this.tbuffer.put(entry[0], entry); this.tbuffer.put(entry.getColBytes(0), entry);
this.timestamp = System.currentTimeMillis(); this.timestamp = System.currentTimeMillis();
} }
@ -233,11 +225,11 @@ public class kelondroDynTree {
if (this.tbuffer.size() == 0) return; if (this.tbuffer.size() == 0) return;
Enumeration e = this.tbuffer.keys(); Enumeration e = this.tbuffer.keys();
kelondroTree t = getTree(this.tablename); kelondroTree t = getTree(this.tablename);
byte[][] entry; kelondroRow.Entry entry;
byte[] key; byte[] key;
while (e.hasMoreElements()) { while (e.hasMoreElements()) {
key = (byte[]) e.nextElement(); key = (byte[]) e.nextElement();
entry = (byte[][]) this.tbuffer.get(key); entry = (kelondroRow.Entry) this.tbuffer.get(key);
t.put(entry); t.put(entry);
} }
t.close(); t.close();
@ -248,7 +240,7 @@ public class kelondroDynTree {
// read cached // read cached
public synchronized byte[][] get(String tablename, byte[] key) throws IOException { public synchronized kelondroRow.Entry get(String tablename, byte[] key) throws IOException {
treeCache tc = (treeCache) cache.get(table); treeCache tc = (treeCache) cache.get(table);
if (tc == null) { if (tc == null) {
tc = new treeCache(tablename); tc = new treeCache(tablename);
@ -256,31 +248,9 @@ public class kelondroDynTree {
} }
return tc.get(key); return tc.get(key);
} }
/*
// clean-up method for cache:
private void flushCache() {
if ((System.currentTimeMillis() - this.cycleCache < this.cycletimeCache) &&
(cache.size() < this.maxcountCache)) return;
this.cycleCache = System.currentTimeMillis();
// collect all caches which have a time > maxagecache
Enumeration e = cache.keys();
String tablename;
treeCache tc;
while (e.hasMoreElements()) {
tablename = (String) e.nextElement();
tc = (treeCache) cache.get(tablename);
if ((System.currentTimeMillis() - tc.timestamp > this.maxageCache) ||
(tc.cache.size() > this.maxsizeCache) ||
(cache.size() > this.maxcountCache)) {
cache.remove(tablename);
}
}
}
*/
// write buffered // write buffered
public synchronized void put(String tablename, byte[][] newrow) { public synchronized void put(String tablename, kelondroRow.Entry newrow) {
treeBuffer tb = (treeBuffer) buffer.get(tablename); treeBuffer tb = (treeBuffer) buffer.get(tablename);
if (tb == null) { if (tb == null) {
tb = new treeBuffer(tablename); tb = new treeBuffer(tablename);
@ -350,17 +320,18 @@ public class kelondroDynTree {
File file = new File("D:\\bin\\testDyn.db"); File file = new File("D:\\bin\\testDyn.db");
if (file.exists()) { if (file.exists()) {
kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, '_'); kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, '_');
System.out.println("opened: table keylength=" + dt.table.columnSize(0) + ", sectorsize=" + dt.table.columnSize(1) + ", " + dt.table.size() + " entries."); System.out.println("opened: table keylength=" + dt.table.row().width(0) + ", sectorsize=" + dt.table.row().width(1) + ", " + dt.table.size() + " entries.");
} else { } else {
kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, 16, 512, new int[] {10,20,30}, '_', true); kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, 16, 512, new kelondroRow(new int[] {10,20,30}), '_', true);
String name; String name;
kelondroTree t; kelondroTree t;
byte[][] line = new byte[][] {"".getBytes(), "abc".getBytes(), "def".getBytes()}; kelondroRow.Entry line;
for (int i = 1; i < 100; i++) { for (int i = 1; i < 100; i++) {
name = "test" + i; name = "test" + i;
t = dt.newTree(name); t = dt.newTree(name);
line = t.row().newEntry(new byte[][] {"".getBytes(), "abc".getBytes(), "def".getBytes()});
for (int j = 1; j < 10; j++) { for (int j = 1; j < 10; j++) {
line[0] = ("entry" + j).getBytes(); line.setCol(0, ("entry" + j).getBytes());
t.put(line); t.put(line);
} }
dt.closeTree(name); dt.closeTree(name);

8
source/de/anomic/kelondro/kelondroFScoreCluster.java
Unescape View File

@ -70,11 +70,11 @@ public class kelondroFScoreCluster {
} catch (IOException e) { } catch (IOException e) {
refcountDBfile.delete(); refcountDBfile.delete();
countrefDBfile.delete(); countrefDBfile.delete();
refcountDB = new kelondroTree(refcountDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new int[] {wordlength, countlength}, objectOrder, 1, countlength, exitOnFail); refcountDB = new kelondroTree(refcountDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new kelondroRow(new int[] {wordlength, countlength}), objectOrder, 1, countlength, exitOnFail);
countrefDB = new kelondroTree(countrefDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new int[] {countlength + wordlength, 4}, objectOrder, 1, countlength, exitOnFail); countrefDB = new kelondroTree(countrefDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new kelondroRow(new int[] {countlength + wordlength, 4}), objectOrder, 1, countlength, exitOnFail);
} else if ((!(refcountDBfile.exists())) && (!(countrefDBfile.exists()))) { } else if ((!(refcountDBfile.exists())) && (!(countrefDBfile.exists()))) {
refcountDB = new kelondroTree(refcountDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new int[] {wordlength, countlength}, objectOrder, 1, countlength, exitOnFail); refcountDB = new kelondroTree(refcountDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new kelondroRow(new int[] {wordlength, countlength}), objectOrder, 1, countlength, exitOnFail);
countrefDB = new kelondroTree(countrefDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new int[] {countlength + wordlength, 4}, objectOrder, 1, countlength, exitOnFail); countrefDB = new kelondroTree(countrefDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new kelondroRow(new int[] {countlength + wordlength, 4}), objectOrder, 1, countlength, exitOnFail);
} else { } else {
if (exitOnFail) { if (exitOnFail) {
System.exit(-1); System.exit(-1);

12
source/de/anomic/kelondro/kelondroFixedWidthArray.java
Unescape View File

@ -55,9 +55,9 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
private static short thisOHBytes = 0; // our record definition does not need extra bytes private static short thisOHBytes = 0; // our record definition does not need extra bytes
private static short thisOHHandles = 0; // and no handles private static short thisOHHandles = 0; // and no handles
public kelondroFixedWidthArray(File file, int[] columns, int intprops, boolean exitOnFail) { public kelondroFixedWidthArray(File file, kelondroRow rowdef, int intprops, boolean exitOnFail) {
// this creates a new array // this creates a new array
super(file, 0, thisOHBytes, thisOHHandles, columns, intprops, columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail); super(file, 0, thisOHBytes, thisOHHandles, rowdef, intprops, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
for (int i = 0; i < intprops; i++) try { for (int i = 0; i < intprops; i++) try {
setHandle(i, new Handle(0)); setHandle(i, new Handle(0));
} catch (IOException e) { } catch (IOException e) {
@ -124,7 +124,7 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
for (int i = 0; i < size(); i++) { for (int i = 0; i < size(); i++) {
System.out.print("row " + i + ": "); System.out.print("row " + i + ": ");
row = get(i); row = get(i);
for (int j = 0; j < columns(); j++) System.out.print(((row.empty(j)) ? "NULL" : row.getColString(j, "UTF-8")) + ", "); for (int j = 0; j < row.columns(); j++) System.out.print(((row.empty(j)) ? "NULL" : row.getColString(j, "UTF-8")) + ", ");
System.out.println(); System.out.println();
} }
System.out.println("EndOfTable"); System.out.println("EndOfTable");
@ -144,7 +144,7 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
// create <filename> <valuelen> // create <filename> <valuelen>
File f = new File(args[1]); File f = new File(args[1]);
if (f.exists()) f.delete(); if (f.exists()) f.delete();
kelondroFixedWidthArray fm = new kelondroFixedWidthArray(f, new int[]{Integer.parseInt(args[2])}, 2, true); kelondroFixedWidthArray fm = new kelondroFixedWidthArray(f, new kelondroRow(new int[]{Integer.parseInt(args[2])}), 2, true);
fm.close(); fm.close();
} else } else
if ((args.length == 2) && (args[0].equals("-v"))) { if ((args.length == 2) && (args[0].equals("-v"))) {
@ -158,7 +158,7 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
// get <filename> <index> // get <filename> <index>
kelondroFixedWidthArray fm = new kelondroFixedWidthArray(new File(args[1])); kelondroFixedWidthArray fm = new kelondroFixedWidthArray(new File(args[1]));
kelondroRow.Entry row = fm.get(Integer.parseInt(args[2])); kelondroRow.Entry row = fm.get(Integer.parseInt(args[2]));
for (int j = 0; j < fm.columns(); j++) System.out.print(row.getColString(j, null) + " "); for (int j = 0; j < fm.row().columns(); j++) System.out.print(row.getColString(j, null) + " ");
System.out.println(); System.out.println();
fm.close(); fm.close();
} else } else
@ -186,7 +186,7 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
if ((args.length == 1) && (args[0].equals("-test"))) { if ((args.length == 1) && (args[0].equals("-test"))) {
File testfile = new File("test.array"); File testfile = new File("test.array");
if (testfile.exists()) testfile.delete(); if (testfile.exists()) testfile.delete();
kelondroFixedWidthArray fm = new kelondroFixedWidthArray(testfile, new int[]{30, 50}, 9, true); kelondroFixedWidthArray fm = new kelondroFixedWidthArray(testfile, new kelondroRow(new int[]{30, 50}), 9, true);
for (int i = 0; i < 100; i++) { for (int i = 0; i < 100; i++) {
fm.set(i, fm.row().newEntry(new byte[][]{("name" + i).getBytes(), ("value" + i).getBytes()})); fm.set(i, fm.row().newEntry(new byte[][]{("name" + i).getBytes(), ("value" + i).getBytes()}));
} }

18
source/de/anomic/kelondro/kelondroFlexTable.java
Unescape View File

@ -61,31 +61,27 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
} }
*/ */
public int columnSize(int column) {
return rowdef.width(column);
}
public kelondroRow.Entry get(byte[] key) throws IOException { public kelondroRow.Entry get(byte[] key) throws IOException {
Integer i = (Integer) this.index.get(key); Integer i = (Integer) this.index.get(key);
if (i == null) return null; if (i == null) return null;
return super.get(i.intValue()); return super.get(i.intValue());
} }
public byte[][] put(byte[][] row) throws IOException { public kelondroRow.Entry put(kelondroRow.Entry row) throws IOException {
Integer i = (Integer) this.index.get(row[0]); Integer i = (Integer) this.index.get(row.getColBytes(0));
if (i == null) { if (i == null) {
i = new Integer(super.add(super.rowdef.newEntry(row))); i = new Integer(super.add(row));
this.index.put(row[0], i); this.index.put(row.getColBytes(0), i);
return null; return null;
} else { } else {
return super.set(i.intValue(), super.rowdef.newEntry(row)).getCols(); return super.set(i.intValue(), row);
} }
} }
public byte[][] remove(byte[] key) throws IOException { public kelondroRow.Entry remove(byte[] key) throws IOException {
Integer i = (Integer) this.index.get(key); Integer i = (Integer) this.index.get(key);
if (i == null) return null; if (i == null) return null;
byte[][] r = super.get(i.intValue()).getCols(); kelondroRow.Entry r = super.get(i.intValue());
super.remove(i.intValue()); super.remove(i.intValue());
return r; return r;
} }

33
source/de/anomic/kelondro/kelondroFlexWidthArray.java
Unescape View File

@ -81,7 +81,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
columns[j - p] = rowdef.width(j); columns[j - p] = rowdef.width(j);
check = check.substring(0, j) + "X" + check.substring(j + 1); check = check.substring(0, j) + "X" + check.substring(j + 1);
} }
col[p] = new kelondroFixedWidthArray(new File(tabledir, colfilename(p, q)), columns, 16, true); col[p] = new kelondroFixedWidthArray(new File(tabledir, colfilename(p, q)), new kelondroRow(columns), 16, true);
} }
} }
@ -94,12 +94,13 @@ public class kelondroFlexWidthArray implements kelondroArray {
return "col." + f + ".list"; return "col." + f + ".list";
} }
public int size() {
return col[0].size(); public kelondroRow row() {
return rowdef;
} }
public int columns() { public int size() {
return rowdef.columns(); return col[0].size();
} }
public kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException { public kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException {
@ -112,12 +113,12 @@ public class kelondroFlexWidthArray implements kelondroArray {
rowentry.bytes(), rowentry.bytes(),
rowdef.colstart[r], rowdef.colstart[r],
rowdef.colstart[r] rowdef.colstart[r]
- rowdef.colstart[r + col[r].columns() - 1] - rowdef.colstart[r + col[r].row().columns() - 1]
+ rowdef.width(r)); + rowdef.width(r));
e1 = col[r].set(index, e0); e1 = col[r].set(index, e0);
for (int i = 0; i < col[r].columns(); i++) for (int i = 0; i < col[r].row().columns(); i++)
p.setCol(r + i, e1.getColBytes(i)); p.setCol(r + i, e1.getColBytes(i));
r = r + col[r].columns(); r = r + col[r].row().columns();
} }
} }
return p; return p;
@ -130,9 +131,9 @@ public class kelondroFlexWidthArray implements kelondroArray {
synchronized (col) { synchronized (col) {
while (r < rowdef.columns()) { while (r < rowdef.columns()) {
e = col[r].get(index); e = col[r].get(index);
for (int i = 0; i < col[r].columns(); i++) for (int i = 0; i < col[r].row().columns(); i++)
p.setCol(r + i, e.getColBytes(i)); p.setCol(r + i, e.getColBytes(i));
r = r + col[r].columns(); r = r + col[r].row().columns();
} }
} }
return p; return p;
@ -144,17 +145,17 @@ public class kelondroFlexWidthArray implements kelondroArray {
synchronized (col) { synchronized (col) {
e = col[0].row().newEntry(rowentry.bytes(), 0, rowdef.width(0)); e = col[0].row().newEntry(rowentry.bytes(), 0, rowdef.width(0));
index = col[0].add(e); index = col[0].add(e);
int r = col[0].columns(); int r = col[0].row().columns();
while (r < rowdef.columns()) { while (r < rowdef.columns()) {
e = col[r].row().newEntry( e = col[r].row().newEntry(
rowentry.bytes(), rowentry.bytes(),
rowdef.colstart[r], rowdef.colstart[r],
rowdef.colstart[r + col[r].columns() - 1] rowdef.colstart[r + col[r].row().columns() - 1]
+ rowdef.width(r + col[r].columns() - 1) + rowdef.width(r + col[r].row().columns() - 1)
- rowdef.colstart[r]); - rowdef.colstart[r]);
col[r].set(index, e); col[r].set(index, e);
r = r + col[r].columns(); r = r + col[r].row().columns();
} }
} }
return index; return index;
@ -165,7 +166,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
synchronized (col) { synchronized (col) {
while (r < rowdef.columns()) { while (r < rowdef.columns()) {
col[r].remove(index); col[r].remove(index);
r = r + col[r].columns(); r = r + col[r].row().columns();
} }
} }
} }
@ -176,7 +177,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
for (int i = 0; i < size(); i++) { for (int i = 0; i < size(); i++) {
System.out.print("row " + i + ": "); System.out.print("row " + i + ": ");
row = get(i); row = get(i);
for (int j = 0; j < columns(); j++) System.out.print(((row.empty(j)) ? "NULL" : row.getColString(j, "UTF-8")) + ", "); for (int j = 0; j < row().columns(); j++) System.out.print(((row.empty(j)) ? "NULL" : row.getColString(j, "UTF-8")) + ", ");
System.out.println(); System.out.println();
} }
System.out.println("EndOfTable"); System.out.println("EndOfTable");

16
source/de/anomic/kelondro/kelondroHashtable.java
Unescape View File

@ -142,7 +142,7 @@ public class kelondroHashtable {
private static final byte[] dummyKey = kelondroBase64Order.enhancedCoder.encodeLong(0, 5).getBytes(); private static final byte[] dummyKey = kelondroBase64Order.enhancedCoder.encodeLong(0, 5).getBytes();
public kelondroHashtable(File file, int[] columns, int offset, int maxsize, int maxrehash, boolean exitOnFail) { public kelondroHashtable(File file, kelondroRow rowdef, int offset, int maxsize, int maxrehash, boolean exitOnFail) {
// this creates a new hashtable // this creates a new hashtable
// the key element is not part of the columns array // the key element is not part of the columns array
// this is unlike the kelondroTree, where the key is part of a row // this is unlike the kelondroTree, where the key is part of a row
@ -152,14 +152,14 @@ public class kelondroHashtable {
// this number is needed to omit grow of the table in case of re-hashing // this number is needed to omit grow of the table in case of re-hashing
// the maxsize is re-computed to a virtual folding height and will result in a tablesize // the maxsize is re-computed to a virtual folding height and will result in a tablesize
// less than the given maxsize. The actual maxsize can be retrieved by maxsize() // less than the given maxsize. The actual maxsize can be retrieved by maxsize()
this.hashArray = new kelondroFixedWidthArray(file, extCol(columns), 6, exitOnFail); this.hashArray = new kelondroFixedWidthArray(file, extCol(rowdef), 6, exitOnFail);
this.offset = offset; this.offset = offset;
this.maxk = kelondroMSetTools.log2a(maxsize); // equal to |log2(maxsize)| + 1 this.maxk = kelondroMSetTools.log2a(maxsize); // equal to |log2(maxsize)| + 1
if (this.maxk >= kelondroMSetTools.log2a(maxsize + power2(offset + 1) + 1) - 1) this.maxk--; if (this.maxk >= kelondroMSetTools.log2a(maxsize + power2(offset + 1) + 1) - 1) this.maxk--;
this.maxrehash = maxrehash; this.maxrehash = maxrehash;
dummyRow = this.hashArray.row().newEntry(); dummyRow = this.hashArray.row().newEntry();
dummyRow.setCol(0, dummyKey); dummyRow.setCol(0, dummyKey);
for (int i = 0; i < hashArray.columns(); i++) for (int i = 0; i < hashArray.row().columns(); i++)
try { try {
hashArray.seti(0, this.offset); hashArray.seti(0, this.offset);
hashArray.seti(1, this.maxk); hashArray.seti(1, this.maxk);
@ -179,11 +179,11 @@ public class kelondroHashtable {
this.maxrehash = hashArray.geti(2); this.maxrehash = hashArray.geti(2);
} }
private int[] extCol(int[] columns) { private kelondroRow extCol(kelondroRow rowdef) {
int[] newCol = new int[columns.length + 1]; int[] newCol = new int[rowdef.columns() + 1];
newCol[0] = 4; newCol[0] = 4;
System.arraycopy(columns, 0, newCol, 1, columns.length); for (int i = 0; i < rowdef.columns(); i++) newCol[i + 1] = rowdef.width(i);
return newCol; return new kelondroRow(newCol);
} }
public static int power2(int x) { public static int power2(int x) {
@ -219,7 +219,7 @@ public class kelondroHashtable {
// write row // write row
kelondroRow.Entry newhkrow = hashArray.row().newEntry(); kelondroRow.Entry newhkrow = hashArray.row().newEntry();
newhkrow.setCol(0, hash.key()); newhkrow.setColLongB256(0, hash.key());
newhkrow.setCol(1, rowentry.bytes()); newhkrow.setCol(1, rowentry.bytes());
hashArray.set(rowNumber, newhkrow); hashArray.set(rowNumber, newhkrow);
return hashArray.row().newEntry(oldhkrow.getColBytes(1)); return hashArray.row().newEntry(oldhkrow.getColBytes(1));

7
source/de/anomic/kelondro/kelondroIndex.java
Unescape View File

@ -54,12 +54,11 @@ import java.io.IOException;
public interface kelondroIndex { public interface kelondroIndex {
public int columns(); public kelondroRow row();
public int columnSize(int column);
public kelondroRow.Entry get(byte[] key) throws IOException; public kelondroRow.Entry get(byte[] key) throws IOException;
public byte[][] put(byte[][] row) throws IOException; public kelondroRow.Entry put(kelondroRow.Entry row) throws IOException;
public byte[][] remove(byte[] key) throws IOException; public kelondroRow.Entry remove(byte[] key) throws IOException;
//public Iterator rows(boolean up, boolean rotating, byte[] startKey) throws IOException; // Objects are of type byte[][] //public Iterator rows(boolean up, boolean rotating, byte[] startKey) throws IOException; // Objects are of type byte[][]
//public Iterator keys(boolean up, boolean rotating, byte[] startKey) throws IOException; // Objects are of type String //public Iterator keys(boolean up, boolean rotating, byte[] startKey) throws IOException; // Objects are of type String
//public TreeMap rowMap(boolean up, boolean rotating, byte[] firstKey, boolean including, int count) throws IOException; //public TreeMap rowMap(boolean up, boolean rotating, byte[] firstKey, boolean including, int count) throws IOException;

2
source/de/anomic/kelondro/kelondroMap.java
Unescape View File

@ -130,7 +130,7 @@ public class kelondroMap {
} }
public int keySize() { public int keySize() {
return dyn.columnSize(0); return dyn.row().width(0);
} }
public int[] cacheNodeChunkSize() { public int[] cacheNodeChunkSize() {

8
source/de/anomic/kelondro/kelondroMapTable.java
Unescape View File

@ -87,7 +87,7 @@ public class kelondroMapTable {
mTables.put(tablename, map); mTables.put(tablename, map);
} }
public void declareTree(String tablename, int[] columns, long buffersize /*bytes*/, boolean exitOnFail) { public void declareTree(String tablename, kelondroRow rowdef, long buffersize /*bytes*/, boolean exitOnFail) {
if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared already in other context."); if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared already in other context.");
if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared twice."); if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared twice.");
File tablefile = new File(tablesPath, "table." + tablename + ".tdb"); File tablefile = new File(tablesPath, "table." + tablename + ".tdb");
@ -96,10 +96,10 @@ public class kelondroMapTable {
Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent); Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) { } catch (IOException e) {
tablefile.getParentFile().mkdirs(); tablefile.getParentFile().mkdirs();
Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent, columns, exitOnFail); Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent, rowdef, exitOnFail);
} else { } else {
tablefile.getParentFile().mkdirs(); tablefile.getParentFile().mkdirs();
Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent, columns, exitOnFail); Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent, rowdef, exitOnFail);
} }
tTables.put(tablename, Tree); tTables.put(tablename, Tree);
} }
@ -112,7 +112,7 @@ public class kelondroMapTable {
mTables.put(tablename, table); mTables.put(tablename, table);
} }
public synchronized void update(String tablename, byte[][] row /* first element is the unique key = index */) throws IOException { public synchronized void update(String tablename, kelondroRow.Entry row /* first element is the unique key = index */) throws IOException {
kelondroTree tree = (kelondroTree) tTables.get(tablename); kelondroTree tree = (kelondroTree) tTables.get(tablename);
if (tree == null) throw new RuntimeException("kelondroTables.update: tree table '" + tablename + "' does not exist."); if (tree == null) throw new RuntimeException("kelondroTables.update: tree table '" + tablename + "' does not exist.");
tree.put(row); tree.put(row);

14
source/de/anomic/kelondro/kelondroNaturalOrder.java
Unescape View File

@ -56,25 +56,25 @@ public class kelondroNaturalOrder extends kelondroAbstractOrder implements kelon
this.zero = null; this.zero = null;
} }
public Object clone() { public final Object clone() {
kelondroNaturalOrder o = new kelondroNaturalOrder(this.asc); kelondroNaturalOrder o = new kelondroNaturalOrder(this.asc);
o.rotate(this.zero); o.rotate(this.zero);
return o; return o;
} }
public static kelondroOrder bySignature(String signature) { public final static kelondroOrder bySignature(String signature) {
if (signature.equals("nd")) return new kelondroNaturalOrder(false); if (signature.equals("nd")) return new kelondroNaturalOrder(false);
if (signature.equals("nu")) return new kelondroNaturalOrder(true); if (signature.equals("nu")) return new kelondroNaturalOrder(true);
return null; return null;
} }
public String signature() { public final String signature() {
if (!asc) return "nd"; if (!asc) return "nd";
if ( asc) return "nu"; if ( asc) return "nu";
return null; return null;
} }
private static long cardinalI(byte[] key) { private final static long cardinalI(byte[] key) {
// returns a cardinal number in the range of 0 .. Long.MAX_VALUE // returns a cardinal number in the range of 0 .. Long.MAX_VALUE
long c = 0; long c = 0;
int p = 0; int p = 0;
@ -84,7 +84,7 @@ public class kelondroNaturalOrder extends kelondroAbstractOrder implements kelon
return c; return c;
} }
public long cardinal(byte[] key) { public final long cardinal(byte[] key) {
if (this.zero == null) return cardinalI(key); if (this.zero == null) return cardinalI(key);
long zeroCardinal = cardinalI(this.zero); long zeroCardinal = cardinalI(this.zero);
long keyCardinal = cardinalI(key); long keyCardinal = cardinalI(key);
@ -131,11 +131,11 @@ public class kelondroNaturalOrder extends kelondroAbstractOrder implements kelon
// is less than, equal to, or greater than the second. // is less than, equal to, or greater than the second.
// two arrays are also equal if one array is a subset of the other's array // two arrays are also equal if one array is a subset of the other's array
// with filled-up char(0)-values // with filled-up char(0)-values
public int compare(byte[] a, byte[] b) { public final int compare(byte[] a, byte[] b) {
return (asc) ? compare0(a, b) : compare0(b, a); return (asc) ? compare0(a, b) : compare0(b, a);
} }
public int compare0(byte[] a, byte[] b) { public final int compare0(byte[] a, byte[] b) {
if (zero == null) return compares(a, b); if (zero == null) return compares(a, b);
// we have an artificial start point. check all combinations // we have an artificial start point. check all combinations
int az = compares(a, zero); // -1 if a < z; 0 if a == z; 1 if a > z int az = compares(a, zero); // -1 if a < z; 0 if a == z; 1 if a > z

39
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -193,7 +193,7 @@ public class kelondroRecords {
public kelondroRecords(File file, long buffersize /* bytes */, public kelondroRecords(File file, long buffersize /* bytes */,
short ohbytec, short ohhandlec, short ohbytec, short ohhandlec,
int[] columns, int FHandles, int txtProps, int txtPropWidth, kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth,
boolean exitOnFail) { boolean exitOnFail) {
// creates a new file // creates a new file
// file: the file that shall be created // file: the file that shall be created
@ -208,7 +208,7 @@ public class kelondroRecords {
kelondroRA raf = new kelondroFileRA(this.filename); kelondroRA raf = new kelondroFileRA(this.filename);
// kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100); // kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100);
// kelondroRA raf = new kelondroNIOFileRA(this.filename, false, 10000); // kelondroRA raf = new kelondroNIOFileRA(this.filename, false, 10000);
init(raf, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth, buffersize / 10); init(raf, ohbytec, ohhandlec, rowdef, FHandles, txtProps, txtPropWidth, buffersize / 10);
} catch (IOException e) { } catch (IOException e) {
logFailure("cannot create / " + e.getMessage()); logFailure("cannot create / " + e.getMessage());
if (exitOnFail) if (exitOnFail)
@ -219,11 +219,11 @@ public class kelondroRecords {
public kelondroRecords(kelondroRA ra, long buffersize /* bytes */, public kelondroRecords(kelondroRA ra, long buffersize /* bytes */,
short ohbytec, short ohhandlec, short ohbytec, short ohhandlec,
int[] columns, int FHandles, int txtProps, int txtPropWidth, kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth,
boolean exitOnFail) { boolean exitOnFail) {
this.filename = null; this.filename = null;
try { try {
init(ra, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth, buffersize / 10); init(ra, ohbytec, ohhandlec, rowdef, FHandles, txtProps, txtPropWidth, buffersize / 10);
} catch (IOException e) { } catch (IOException e) {
logFailure("cannot create / " + e.getMessage()); logFailure("cannot create / " + e.getMessage());
if (exitOnFail) System.exit(-1); if (exitOnFail) System.exit(-1);
@ -232,7 +232,7 @@ public class kelondroRecords {
} }
private void init(kelondroRA ra, short ohbytec, short ohhandlec, private void init(kelondroRA ra, short ohbytec, short ohhandlec,
int[] columns, int FHandles, int txtProps, int txtPropWidth, long writeBufferSize) throws IOException { kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth, long writeBufferSize) throws IOException {
// create new Chunked IO // create new Chunked IO
if (useWriteBuffer) { if (useWriteBuffer) {
@ -242,16 +242,16 @@ public class kelondroRecords {
} }
// create row // create row
ROW = new kelondroRow(columns); ROW = rowdef;
// store dynamic run-time data // store dynamic run-time data
this.overhead = ohbytec + 4 * ohhandlec; this.overhead = ohbytec + 4 * ohhandlec;
this.recordsize = this.overhead + ROW.size(); this.recordsize = this.overhead + ROW.objectsize();
this.headchunksize = overhead + columns[0]; this.headchunksize = overhead + ROW.width(0);
this.tailchunksize = this.recordsize - this.headchunksize; this.tailchunksize = this.recordsize - this.headchunksize;
// store dynamic run-time seek pointers // store dynamic run-time seek pointers
POS_HANDLES = POS_COLWIDTHS + columns.length * 4; POS_HANDLES = POS_COLWIDTHS + ROW.columns() * 4;
POS_TXTPROPS = POS_HANDLES + FHandles * 4; POS_TXTPROPS = POS_HANDLES + FHandles * 4;
POS_NODES = POS_TXTPROPS + txtProps * txtPropWidth; POS_NODES = POS_TXTPROPS + txtProps * txtPropWidth;
@ -410,7 +410,7 @@ public class kelondroRecords {
// assign remaining values that are only present at run-time // assign remaining values that are only present at run-time
this.overhead = OHBYTEC + 4 * OHHANDLEC; this.overhead = OHBYTEC + 4 * OHHANDLEC;
this.recordsize = this.overhead; this.recordsize = this.overhead;
this.recordsize = this.overhead + ROW.size(); this.recordsize = this.overhead + ROW.objectsize();
this.headchunksize = this.overhead + this.ROW.width(0); this.headchunksize = this.overhead + this.ROW.width(0);
this.tailchunksize = this.recordsize - this.headchunksize; this.tailchunksize = this.recordsize - this.headchunksize;
} }
@ -751,13 +751,13 @@ public class kelondroRecords {
*/ */
public byte[] setValueRow(byte[] row) throws IOException { public byte[] setValueRow(byte[] row) throws IOException {
// if the index is defined, then write values directly to the file, else only to the object // if the index is defined, then write values directly to the file, else only to the object
assert row.length == ROW.size(); assert row.length == ROW.objectsize();
byte[] result = getValueRow(); // previous value (this loads the values if not already happened) byte[] result = getValueRow(); // previous value (this loads the values if not already happened)
// set values // set values
if (this.handle.index != NUL) { if (this.handle.index != NUL) {
setValue(row, 0, ROW.width(0), headChunk, overhead); setValue(row, 0, ROW.width(0), headChunk, overhead);
if (ROW.columns() > 0) setValue(row, ROW.width(0), ROW.size() - ROW.width(0), tailChunk, 0); if (ROW.columns() > 1) setValue(row, ROW.width(0), ROW.objectsize() - ROW.width(0), tailChunk, 0);
} }
this.headChanged = true; this.headChanged = true;
this.tailChanged = true; this.tailChanged = true;
@ -805,7 +805,7 @@ public class kelondroRecords {
} }
// create return value // create return value
byte[] row = new byte[ROW.size()]; byte[] row = new byte[ROW.objectsize()];
// read key // read key
System.arraycopy(headChunk, overhead, row, 0, ROW.width(0)); System.arraycopy(headChunk, overhead, row, 0, ROW.width(0));
@ -1051,15 +1051,6 @@ public class kelondroRecords {
public final kelondroRow row() { public final kelondroRow row() {
return this.ROW; return this.ROW;
} }
public final int columns() {
return this.ROW.columns();
}
public final int columnSize(int column) {
if ((column < 0) || (column >= this.ROW.columns())) return -1;
return ROW.width(column);
}
private final long seekpos(Handle handle) { private final long seekpos(Handle handle) {
assert (handle.index >= 0): "handle index too low: " + handle.index; assert (handle.index >= 0): "handle index too low: " + handle.index;
@ -1263,8 +1254,8 @@ public class kelondroRecords {
System.out.println(" Data Offset: 0x" + Long.toHexString(POS_NODES)); System.out.println(" Data Offset: 0x" + Long.toHexString(POS_NODES));
System.out.println("--"); System.out.println("--");
System.out.println("RECORDS"); System.out.println("RECORDS");
System.out.print(" Columns : " + columns() + " columns {" + ROW.width(0)); System.out.print(" Columns : " + row().columns() + " columns {" + ROW.width(0));
for (int i = 1; i < columns(); i++) System.out.print(", " + ROW.width(i)); for (int i = 1; i < row().columns(); i++) System.out.print(", " + ROW.width(i));
System.out.println("}"); System.out.println("}");
System.out.println(" Overhead : " + this.overhead + " bytes (" + OHBYTEC + " OH bytes, " + OHHANDLEC + " OH Handles)"); System.out.println(" Overhead : " + this.overhead + " bytes (" + OHBYTEC + " OH bytes, " + OHHANDLEC + " OH Handles)");
System.out.println(" Recordsize : " + this.recordsize + " bytes"); System.out.println(" Recordsize : " + this.recordsize + " bytes");

15
source/de/anomic/kelondro/kelondroRow.java
Unescape View File

@ -73,7 +73,7 @@ public class kelondroRow {
return this.row.length; return this.row.length;
} }
public int size() { public int objectsize() {
return this.objectsize; return this.objectsize;
} }
@ -164,6 +164,10 @@ public class kelondroRow {
return rowinstance; return rowinstance;
} }
public int columns() {
return row.length;
}
public boolean empty(int column) { public boolean empty(int column) {
return rowinstance[colstart[column]] == 0; return rowinstance[colstart[column]] == 0;
} }
@ -181,10 +185,15 @@ public class kelondroRow {
} }
} }
public void setCol(int column, long cell) { public void setColLongB256(int column, long cell) {
kelondroNaturalOrder.encodeLong(cell, rowinstance, colstart[column], row[column].cellwidth()); kelondroNaturalOrder.encodeLong(cell, rowinstance, colstart[column], row[column].cellwidth());
} }
public void setColLongB64E(int column, long cell) {
kelondroBase64Order.enhancedCoder.encodeLong(cell, rowinstance, colstart[column], row[column].cellwidth());
}
/*
public byte[][] getCols() { public byte[][] getCols() {
byte[][] values = new byte[row.length][]; byte[][] values = new byte[row.length][];
@ -203,7 +212,7 @@ public class kelondroRow {
return values; return values;
} }
*/
public String getColString(int column, String encoding) { public String getColString(int column, String encoding) {
int length = row[column].cellwidth(); int length = row[column].cellwidth();
int offset = colstart[column]; int offset = colstart[column];

30
source/de/anomic/kelondro/kelondroSplittedTree.java
Unescape View File

@ -75,25 +75,25 @@ public class kelondroSplittedTree implements kelondroIndex {
public kelondroSplittedTree(File pathToFiles, String filenameStub, kelondroOrder objectOrder, public kelondroSplittedTree(File pathToFiles, String filenameStub, kelondroOrder objectOrder,
long buffersize, long buffersize,
int forkfactor, int forkfactor,
int[] columns, kelondroRow rowdef,
int txtProps, int txtPropsWidth, int txtProps, int txtPropsWidth,
boolean exitOnFail) { boolean exitOnFail) {
ktfs = new kelondroTree[forkfactor]; ktfs = new kelondroTree[forkfactor];
File f; File f;
for (int i = 0; i < forkfactor; i++) { for (int i = 0; i < forkfactor; i++) {
f = dbFile(pathToFiles, filenameStub, forkfactor, columns.length, i); f = dbFile(pathToFiles, filenameStub, forkfactor, rowdef.columns(), i);
if (f.exists()) { if (f.exists()) {
try { try {
ktfs[i] = new kelondroTree(f, buffersize/forkfactor, kelondroTree.defaultObjectCachePercent); ktfs[i] = new kelondroTree(f, buffersize/forkfactor, kelondroTree.defaultObjectCachePercent);
this.order = ktfs[i].order(); this.order = ktfs[i].order();
} catch (IOException e) { } catch (IOException e) {
ktfs[i] = new kelondroTree(f, buffersize/forkfactor, kelondroTree.defaultObjectCachePercent, ktfs[i] = new kelondroTree(f, buffersize/forkfactor, kelondroTree.defaultObjectCachePercent,
columns, objectOrder, txtProps, txtPropsWidth, exitOnFail); rowdef, objectOrder, txtProps, txtPropsWidth, exitOnFail);
this.order = objectOrder; this.order = objectOrder;
} }
} else { } else {
ktfs[i] = new kelondroTree(f, buffersize/forkfactor, kelondroTree.defaultObjectCachePercent, ktfs[i] = new kelondroTree(f, buffersize/forkfactor, kelondroTree.defaultObjectCachePercent,
columns, objectOrder, txtProps, txtPropsWidth, exitOnFail); rowdef, objectOrder, txtProps, txtPropsWidth, exitOnFail);
this.order = objectOrder; this.order = objectOrder;
} }
} }
@ -113,27 +113,23 @@ public class kelondroSplittedTree implements kelondroIndex {
public static kelondroSplittedTree open(File pathToFiles, String filenameStub, kelondroOrder objectOrder, public static kelondroSplittedTree open(File pathToFiles, String filenameStub, kelondroOrder objectOrder,
long buffersize, long buffersize,
int forkfactor, int forkfactor,
int[] columns, int txtProps, int txtPropsWidth, kelondroRow rowdef, int txtProps, int txtPropsWidth,
boolean exitOnFail) throws IOException { boolean exitOnFail) throws IOException {
// generated a new splittet tree if it not exists or // generated a new splittet tree if it not exists or
// opens an existing one // opens an existing one
if (existsAll(pathToFiles, filenameStub, forkfactor, columns.length)) { if (existsAll(pathToFiles, filenameStub, forkfactor, rowdef.columns())) {
return new kelondroSplittedTree(pathToFiles, filenameStub, objectOrder, buffersize, forkfactor, columns.length); return new kelondroSplittedTree(pathToFiles, filenameStub, objectOrder, buffersize, forkfactor, rowdef.columns());
} else { } else {
return new kelondroSplittedTree(pathToFiles, filenameStub, objectOrder, return new kelondroSplittedTree(pathToFiles, filenameStub, objectOrder,
buffersize, buffersize,
forkfactor, forkfactor,
columns, txtProps, txtPropsWidth, rowdef, txtProps, txtPropsWidth,
exitOnFail); exitOnFail);
} }
} }
public int columns() { public kelondroRow row() {
return ktfs[0].columns(); return ktfs[0].row();
}
public int columnSize(int column) {
return ktfs[0].columnSize(column);
} }
private int partition(byte[] key) { private int partition(byte[] key) {
@ -145,11 +141,11 @@ public class kelondroSplittedTree implements kelondroIndex {
return ktfs[partition(key)].get(key); return ktfs[partition(key)].get(key);
} }
public byte[][] put(byte[][] row) throws IOException { public kelondroRow.Entry put(kelondroRow.Entry row) throws IOException {
return ktfs[partition(row[0])].put(row); return ktfs[partition(row.getColBytes(0))].put(row);
} }
public byte[][] remove(byte[] key) throws IOException { public kelondroRow.Entry remove(byte[] key) throws IOException {
return ktfs[partition(key)].remove(key); return ktfs[partition(key)].remove(key);
} }

18
source/de/anomic/kelondro/kelondroStack.java
Unescape View File

@ -68,12 +68,12 @@ public final class kelondroStack extends kelondroRecords {
private static int toor = 1; // pointer for FHandles-array: pointer to root node private static int toor = 1; // pointer for FHandles-array: pointer to root node
public kelondroStack(File file, long buffersize, int key, int value, boolean exitOnFail) { public kelondroStack(File file, long buffersize, int key, int value, boolean exitOnFail) {
this(file, buffersize, new int[] { key, value }, exitOnFail); this(file, buffersize, new kelondroRow(new int[] { key, value }), exitOnFail);
} }
public kelondroStack(File file, long buffersize, int[] columns, boolean exitOnFail) { public kelondroStack(File file, long buffersize, kelondroRow rowdef, boolean exitOnFail) {
// this creates a new stack // this creates a new stack
super(file, buffersize, thisOHBytes, thisOHHandles, columns, thisFHandles, columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail); super(file, buffersize, thisOHBytes, thisOHHandles, rowdef, thisFHandles, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
try { try {
setHandle(root, null); // define the root value setHandle(root, null); // define the root value
setHandle(toor, null); // define the toor value setHandle(toor, null); // define the toor value
@ -107,7 +107,7 @@ public final class kelondroStack extends kelondroRecords {
if (f.exists()) f.delete(); if (f.exists()) f.delete();
// re-open a database with same settings as before // re-open a database with same settings as before
return new kelondroStack(f, bz, row.widths(), true); return new kelondroStack(f, bz, row, true);
} }
public class Counter implements Iterator { public class Counter implements Iterator {
@ -307,10 +307,10 @@ public final class kelondroStack extends kelondroRecords {
st = new StringTokenizer(s, separator); st = new StringTokenizer(s, separator);
// buffer the entry // buffer the entry
c = 0; c = 0;
while ((c < columns()) && (st.hasMoreTokens())) { while ((c < row().columns()) && (st.hasMoreTokens())) {
buffer.setCol(c++, st.nextToken().trim().getBytes()); buffer.setCol(c++, st.nextToken().trim().getBytes());
} }
if ((st.hasMoreTokens()) || (c != columns())) { if ((st.hasMoreTokens()) || (c != row().columns())) {
System.err.println("inapropriate number of entries in line " + line); System.err.println("inapropriate number of entries in line " + line);
} else { } else {
push(buffer); push(buffer);
@ -346,7 +346,7 @@ public final class kelondroStack extends kelondroRecords {
+ hp(n.getOHHandle(left)) + ", right " + hp(n.getOHHandle(left)) + ", right "
+ hp(n.getOHHandle(right))); + hp(n.getOHHandle(right)));
System.out.print(" KEY:'" + r.getColString(0, null) + "'"); System.out.print(" KEY:'" + r.getColString(0, null) + "'");
for (int j = 1; j < columns(); j++) for (int j = 1; j < row().columns(); j++)
System.out.print(", V[" + j + "]:'" + r.getColString(j, null) + "'"); System.out.print(", V[" + j + "]:'" + r.getColString(j, null) + "'");
System.out.println(); System.out.println();
} }
@ -414,9 +414,7 @@ public final class kelondroStack extends kelondroRecords {
// create <keylen> <valuelen> <filename> // create <keylen> <valuelen> <filename>
File f = new File(args[3]); File f = new File(args[3]);
if (f.exists()) f.delete(); if (f.exists()) f.delete();
int[] lens = new int[2]; kelondroRow lens = new kelondroRow(new int[]{Integer.parseInt(args[1]), Integer.parseInt(args[2])});
lens[0] = Integer.parseInt(args[1]);
lens[1] = Integer.parseInt(args[2]);
kelondroStack fm = new kelondroStack(f, 0x100000, lens, true); kelondroStack fm = new kelondroStack(f, 0x100000, lens, true);
fm.close(); fm.close();
} else if (args[0].equals("-p")) { } else if (args[0].equals("-p")) {

65
source/de/anomic/kelondro/kelondroTree.java
Unescape View File

@ -93,19 +93,19 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
public kelondroTree(File file, long buffersize, int objectCachePercent, int key, int value, boolean exitOnFail) { public kelondroTree(File file, long buffersize, int objectCachePercent, int key, int value, boolean exitOnFail) {
this(file, buffersize, objectCachePercent, new int[] { key, value }, new kelondroNaturalOrder(true), 1, 8, exitOnFail); this(file, buffersize, objectCachePercent, new kelondroRow(new int[] { key, value }), new kelondroNaturalOrder(true), 1, 8, exitOnFail);
} }
public kelondroTree(File file, long buffersize, int objectCachePercent, int[] columns, boolean exitOnFail) { public kelondroTree(File file, long buffersize, int objectCachePercent, kelondroRow rowdef, boolean exitOnFail) {
// this creates a new tree file // this creates a new tree file
this(file, buffersize, objectCachePercent, columns, new kelondroNaturalOrder(true), columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail); this(file, buffersize, objectCachePercent, rowdef, new kelondroNaturalOrder(true), rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
} }
public kelondroTree(File file, long buffersize, int objectCachePercent, int[] columns, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) { public kelondroTree(File file, long buffersize, int objectCachePercent, kelondroRow rowdef, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) {
// this creates a new tree file // this creates a new tree file
super(file, super(file,
(100 - objectCachePercent) * buffersize / 100, (100 - objectCachePercent) * buffersize / 100,
thisOHBytes, thisOHHandles, columns, thisOHBytes, thisOHHandles, rowdef,
thisFHandles, txtProps, txtPropsWidth, exitOnFail); thisFHandles, txtProps, txtPropsWidth, exitOnFail);
try { try {
setHandle(root, null); // define the root value setHandle(root, null); // define the root value
@ -120,16 +120,16 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
initObjectCache(buffersize, objectCachePercent); initObjectCache(buffersize, objectCachePercent);
} }
public kelondroTree(kelondroRA ra, long buffersize, int objectCachePercent, int[] columns, boolean exitOnFail) { public kelondroTree(kelondroRA ra, long buffersize, int objectCachePercent, kelondroRow rowdef, boolean exitOnFail) {
// this creates a new tree within a kelondroRA // this creates a new tree within a kelondroRA
this(ra, buffersize, objectCachePercent, columns, new kelondroNaturalOrder(true), columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail); this(ra, buffersize, objectCachePercent, rowdef, new kelondroNaturalOrder(true), rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
} }
public kelondroTree(kelondroRA ra, long buffersize, int objectCachePercent, int[] columns, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) { public kelondroTree(kelondroRA ra, long buffersize, int objectCachePercent, kelondroRow rowdef, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) {
// this creates a new tree within a kelondroRA // this creates a new tree within a kelondroRA
super(ra, super(ra,
(100 - objectCachePercent) * buffersize / 100, (100 - objectCachePercent) * buffersize / 100,
thisOHBytes, thisOHHandles, columns, thisOHBytes, thisOHHandles, rowdef,
thisFHandles, txtProps, txtPropsWidth, exitOnFail); thisFHandles, txtProps, txtPropsWidth, exitOnFail);
try { try {
setHandle(root, null); // define the root value setHandle(root, null); // define the root value
@ -171,7 +171,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
} }
public final int cacheObjectChunkSize() { public final int cacheObjectChunkSize() {
return row().size() + 8 * super.columns(); return row().objectsize() + /* overhead */ 8 * super.row().columns();
} }
public String[] cacheObjectStatus() { public String[] cacheObjectStatus() {
@ -394,19 +394,18 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
} }
// Associates the specified value with the specified key in this map // Associates the specified value with the specified key in this map
public byte[][] put(byte[][] newrow) throws IOException { public kelondroRow.Entry put(kelondroRow.Entry newrow) throws IOException {
kelondroRow.Entry result = null; kelondroRow.Entry result = null;
//writeLock.stay(2000, 1000); //writeLock.stay(2000, 1000);
if (newrow.length != columns()) throw new IllegalArgumentException("put: wrong row length " + newrow.length + "; must be " + columns()); if (newrow.columns() != row().columns()) throw new IllegalArgumentException("put: wrong row length " + newrow.columns() + "; must be " + row().columns());
// first try to find the key element in the database // first try to find the key element in the database
synchronized(writeSearchObj) { synchronized(writeSearchObj) {
kelondroRow.Entry newentry = row().newEntry(newrow); if (objectCache != null) objectCache.put(newrow.getColBytes(0), newrow);
if (objectCache != null) objectCache.put(newentry.getColBytes(0), newentry); writeSearchObj.process(newrow.getColBytes(0));
writeSearchObj.process(newrow[0]);
if (writeSearchObj.found()) { if (writeSearchObj.found()) {
// a node with this key exist. simply overwrite the content and return old content // a node with this key exist. simply overwrite the content and return old content
Node e = writeSearchObj.getMatcher(); Node e = writeSearchObj.getMatcher();
result = row().newEntry(e.setValueRow(newentry.bytes())); result = row().newEntry(e.setValueRow(newrow.bytes()));
commitNode(e); commitNode(e);
} else if (writeSearchObj.isRoot()) { } else if (writeSearchObj.isRoot()) {
// a node with this key does not exist and there is no node at all // a node with this key does not exist and there is no node at all
@ -415,7 +414,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
throw new kelondroException(filename, "tried to create root node twice"); throw new kelondroException(filename, "tried to create root node twice");
// we dont have any Nodes in the file, so start here to create one // we dont have any Nodes in the file, so start here to create one
Node e = newNode(); Node e = newNode();
e.setValueRow(newentry.bytes()); e.setValueRow(newrow.bytes());
// write the propetries // write the propetries
e.setOHByte(magic, (byte) 1); e.setOHByte(magic, (byte) 1);
e.setOHByte(balance, (byte) 0); e.setOHByte(balance, (byte) 0);
@ -437,7 +436,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
// create new node and assign values // create new node and assign values
Node parentNode = writeSearchObj.getParent(); Node parentNode = writeSearchObj.getParent();
Node theNode = newNode(); Node theNode = newNode();
theNode.setValueRow(newentry.bytes()); theNode.setValueRow(newrow.bytes());
theNode.setOHByte(0, (byte) 1); // fresh magic theNode.setOHByte(0, (byte) 1); // fresh magic
theNode.setOHByte(1, (byte) 0); // fresh balance theNode.setOHByte(1, (byte) 0); // fresh balance
theNode.setOHHandle(parent, parentNode.handle()); theNode.setOHHandle(parent, parentNode.handle());
@ -537,7 +536,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
} }
} }
//writeLock.release(); //writeLock.release();
return (result == null) ? null : result.getCols(); return result;
} }
private void assignChild(Node parentNode, Node childNode, int childType) throws IOException { private void assignChild(Node parentNode, Node childNode, int childType) throws IOException {
@ -656,15 +655,13 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
// Associates the specified value with the specified key in this map // Associates the specified value with the specified key in this map
public byte[] put(byte[] key, byte[] value) throws IOException { public byte[] put(byte[] key, byte[] value) throws IOException {
byte[][] row = new byte[2][]; kelondroRow.Entry row = row().newEntry(new byte[][]{key, value});
row[0] = key; kelondroRow.Entry ret = put(row);
row[1] = value; if (ret == null) return null; else return ret.getColBytes(0);
byte[][] ret = put(row);
if (ret == null) return null; else return ret[1];
} }
// Removes the mapping for this key from this map if present (optional operation). // Removes the mapping for this key from this map if present (optional operation).
public byte[][] remove(byte[] key) throws IOException { public kelondroRow.Entry remove(byte[] key) throws IOException {
synchronized(writeSearchObj) { synchronized(writeSearchObj) {
if (objectCache != null) objectCache.remove(key); if (objectCache != null) objectCache.remove(key);
writeSearchObj.process(key); writeSearchObj.process(key);
@ -672,7 +669,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
Node result = writeSearchObj.getMatcher(); Node result = writeSearchObj.getMatcher();
kelondroRow.Entry values = row().newEntry(result.getValueRow()); kelondroRow.Entry values = row().newEntry(result.getValueRow());
remove(result, writeSearchObj.getParent()); remove(result, writeSearchObj.getParent());
return values.getCols(); return values;
} else { } else {
return null; return null;
} }
@ -1029,7 +1026,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
Iterator i = (firstKey == null) ? new nodeIterator(up, rotating) : new nodeIterator(up, rotating, firstKey, including); Iterator i = (firstKey == null) ? new nodeIterator(up, rotating) : new nodeIterator(up, rotating, firstKey, including);
while ((rows.size() < count) && (i.hasNext())) { while ((rows.size() < count) && (i.hasNext())) {
n = (Node) i.next(); n = (Node) i.next();
if (n != null) rows.put(new String(n.getKey()), row().newEntry(n.getValueRow()).getCols()); if (n != null) rows.put(new String(n.getKey()), row().newEntry(n.getValueRow()));
} }
} }
return rows; return rows;
@ -1196,7 +1193,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
String s; String s;
StringTokenizer st; StringTokenizer st;
int recs = 0; int recs = 0;
byte[][] buffer = new byte[columns()][]; kelondroRow.Entry buffer = row().newEntry();
int c; int c;
int line = 0; int line = 0;
while ((s = f.readLine()) != null) { while ((s = f.readLine()) != null) {
@ -1206,10 +1203,10 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
st = new StringTokenizer(s, separator); st = new StringTokenizer(s, separator);
// buffer the entry // buffer the entry
c = 0; c = 0;
while ((c < columns()) && (st.hasMoreTokens())) { while ((c < row().columns()) && (st.hasMoreTokens())) {
buffer[c++] = st.nextToken().trim().getBytes(); buffer.setCol(c++, st.nextToken().trim().getBytes());
} }
if ((st.hasMoreTokens()) || (c != columns())) { if ((st.hasMoreTokens()) || (c != row().columns())) {
System.err.println("inapropriate number of entries in line " + line); System.err.println("inapropriate number of entries in line " + line);
} else { } else {
put(buffer); put(buffer);
@ -1253,7 +1250,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
Vector nextline; Vector nextline;
Handle handle; Handle handle;
Node node; Node node;
int linelength, width = (1 << (height - 1)) * (columnSize(0) + 1); int linelength, width = (1 << (height - 1)) * (row().width(0) + 1);
String key; String key;
for (int h = 1; h < height; h++) { for (int h = 1; h < height; h++) {
linelength = width / (thisline.size() * 2); linelength = width / (thisline.size() * 2);
@ -1399,9 +1396,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
// create <keylen> <valuelen> <filename> // create <keylen> <valuelen> <filename>
File f = new File(args[3]); File f = new File(args[3]);
if (f.exists()) f.delete(); if (f.exists()) f.delete();
int[] lens = new int[2]; kelondroRow lens = new kelondroRow(new int[]{Integer.parseInt(args[1]), Integer.parseInt(args[2])});
lens[0] = Integer.parseInt(args[1]);
lens[1] = Integer.parseInt(args[2]);
kelondroTree fm = new kelondroTree(f, 0x100000, 10, lens, true); kelondroTree fm = new kelondroTree(f, 0x100000, 10, lens, true);
fm.close(); fm.close();
} else if (args[0].equals("-u")) { } else if (args[0].equals("-u")) {

5
source/de/anomic/plasma/plasmaCrawlBalancer.java
Unescape View File

@ -50,6 +50,7 @@ import java.util.ArrayList;
import de.anomic.index.indexURL; import de.anomic.index.indexURL;
import de.anomic.kelondro.kelondroRecords; import de.anomic.kelondro.kelondroRecords;
import de.anomic.kelondro.kelondroRow;
import de.anomic.kelondro.kelondroStack; import de.anomic.kelondro.kelondroStack;
public class plasmaCrawlBalancer { public class plasmaCrawlBalancer {
@ -62,10 +63,10 @@ public class plasmaCrawlBalancer {
try { try {
stack = new kelondroStack(stackFile, buffersize); stack = new kelondroStack(stackFile, buffersize);
} catch (IOException e) { } catch (IOException e) {
stack = new kelondroStack(stackFile, buffersize, new int[] {indexURL.urlHashLength}, true); stack = new kelondroStack(stackFile, buffersize, new kelondroRow(new int[] {indexURL.urlHashLength}), true);
} }
} else { } else {
stack = new kelondroStack(stackFile, buffersize, new int[] {indexURL.urlHashLength}, true); stack = new kelondroStack(stackFile, buffersize, new kelondroRow(new int[] {indexURL.urlHashLength}), true);
} }
domainStacks = new HashMap(); domainStacks = new HashMap();
} }

10
source/de/anomic/plasma/plasmaCrawlEURL.java
Unescape View File

@ -73,8 +73,8 @@ public class plasmaCrawlEURL extends indexURL {
urlNameLength, // the name of the url, from anchor tag <a>name</a> urlNameLength, // the name of the url, from anchor tag <a>name</a>
urlDateLength, // the time when the url was first time appeared urlDateLength, // the time when the url was first time appeared
urlDateLength, // the time when the url was last time tried to load urlDateLength, // the time when the url was last time tried to load
urlRetryLength, // number of load retries urlRetryLength, // number of load retries
urlErrorLength, // string describing load failure urlErrorLength, // string describing load failure
urlFlagLength // extra space urlFlagLength // extra space
}; };
if (cachePath.exists()) try { if (cachePath.exists()) try {
@ -82,11 +82,11 @@ public class plasmaCrawlEURL extends indexURL {
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent); urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) { } catch (IOException e) {
cachePath.delete(); cachePath.delete();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true); urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true);
} else { } else {
// create new cache // create new cache
cachePath.getParentFile().mkdirs(); cachePath.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true); urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true);
} }
} }
@ -216,7 +216,7 @@ public class plasmaCrawlEURL extends indexURL {
this.failreason.getBytes(), this.failreason.getBytes(),
this.flags.getBytes() this.flags.getBytes()
}; };
urlHashCache.put(entry); urlHashCache.put(urlHashCache.row().newEntry(entry));
} catch (IOException e) { } catch (IOException e) {
System.out.println("INTERNAL ERROR AT plasmaEURL:url2hash:" + e.toString()); System.out.println("INTERNAL ERROR AT plasmaEURL:url2hash:" + e.toString());
} }

8
source/de/anomic/plasma/plasmaCrawlLURL.java
Unescape View File

@ -119,12 +119,12 @@ public final class plasmaCrawlLURL extends indexURL {
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent); urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) { } catch (IOException e) {
cachePath.getParentFile().mkdirs(); cachePath.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true); urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true);
} }
} else { } else {
// create new cache // create new cache
cachePath.getParentFile().mkdirs(); cachePath.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true); urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true);
} }
// init result stacks // init result stacks
@ -575,7 +575,7 @@ public final class plasmaCrawlLURL extends indexURL {
kelondroBase64Order.enhancedCoder.encodeLong(size, urlSizeLength).getBytes(), kelondroBase64Order.enhancedCoder.encodeLong(size, urlSizeLength).getBytes(),
kelondroBase64Order.enhancedCoder.encodeLong(wordCount, urlWordCountLength).getBytes(), kelondroBase64Order.enhancedCoder.encodeLong(wordCount, urlWordCountLength).getBytes(),
}; };
urlHashCache.put(entry); urlHashCache.put(urlHashCache.row().newEntry(entry));
serverLog.logFine("PLASMA","STORED new LURL " + url.toString()); serverLog.logFine("PLASMA","STORED new LURL " + url.toString());
this.stored = true; this.stored = true;
} catch (Exception e) { } catch (Exception e) {
@ -846,7 +846,7 @@ public final class plasmaCrawlLURL extends indexURL {
if (res.statusCode == 200) { if (res.statusCode == 200) {
entry.setCol(1, newUrl.toString().getBytes()); entry.setCol(1, newUrl.toString().getBytes());
urlHashCache.put(entry.getCols()); urlHashCache.put(entry);
log.logInfo("UrlDB-Entry with urlHash '" + urlHash + "' corrected\n\tURL: " + oldUrlStr + " -> " + newUrlStr); log.logInfo("UrlDB-Entry with urlHash '" + urlHash + "' corrected\n\tURL: " + oldUrlStr + " -> " + newUrlStr);
} else { } else {
remove(urlHash); remove(urlHash);

19
source/de/anomic/plasma/plasmaCrawlNURL.java
Unescape View File

@ -124,26 +124,27 @@ public class plasmaCrawlNURL extends indexURL {
limitStack = new plasmaCrawlBalancer(limitStackFile, 0); limitStack = new plasmaCrawlBalancer(limitStackFile, 0);
overhangStack = new plasmaCrawlBalancer(overhangStackFile, 0); overhangStack = new plasmaCrawlBalancer(overhangStackFile, 0);
remoteStack = new plasmaCrawlBalancer(remoteStackFile, 0); remoteStack = new plasmaCrawlBalancer(remoteStackFile, 0);
kelondroRow rowdef = new kelondroRow(new int[] {indexURL.urlHashLength});
if (imageStackFile.exists()) try { if (imageStackFile.exists()) try {
imageStack = new kelondroStack(imageStackFile, 0); imageStack = new kelondroStack(imageStackFile, 0);
} catch (IOException e) { } catch (IOException e) {
imageStack = new kelondroStack(imageStackFile, 0, new int[] {indexURL.urlHashLength}, true); imageStack = new kelondroStack(imageStackFile, 0, rowdef, true);
} else { } else {
imageStack = new kelondroStack(imageStackFile, 0, new int[] {indexURL.urlHashLength}, true); imageStack = new kelondroStack(imageStackFile, 0, rowdef, true);
} }
if (movieStackFile.exists()) try { if (movieStackFile.exists()) try {
movieStack = new kelondroStack(movieStackFile, 0); movieStack = new kelondroStack(movieStackFile, 0);
} catch (IOException e) { } catch (IOException e) {
movieStack = new kelondroStack(movieStackFile, 0, new int[] {indexURL.urlHashLength}, true); movieStack = new kelondroStack(movieStackFile, 0, rowdef, true);
} else { } else {
movieStack = new kelondroStack(movieStackFile, 0, new int[] {indexURL.urlHashLength}, true); movieStack = new kelondroStack(movieStackFile, 0, rowdef, true);
} }
if (musicStackFile.exists()) try { if (musicStackFile.exists()) try {
musicStack = new kelondroStack(musicStackFile, 0); musicStack = new kelondroStack(musicStackFile, 0);
} catch (IOException e) { } catch (IOException e) {
musicStack = new kelondroStack(musicStackFile, 0, new int[] {indexURL.urlHashLength}, true); musicStack = new kelondroStack(musicStackFile, 0, rowdef, true);
} else { } else {
musicStack = new kelondroStack(musicStackFile, 0, new int[] {indexURL.urlHashLength}, true); musicStack = new kelondroStack(musicStackFile, 0, rowdef, true);
} }
// init stack Index // init stack Index
@ -168,11 +169,11 @@ public class plasmaCrawlNURL extends indexURL {
urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent); urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) { } catch (IOException e) {
cacheFile.delete(); cacheFile.delete();
urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true); urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true);
} else { } else {
// create new cache // create new cache
cacheFile.getParentFile().mkdirs(); cacheFile.getParentFile().mkdirs();
urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true); urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true);
} }
} }
@ -553,7 +554,7 @@ public class plasmaCrawlNURL extends indexURL {
this.flags.getBytes(), this.flags.getBytes(),
normalizeHandle(this.handle).getBytes() normalizeHandle(this.handle).getBytes()
}; };
urlHashCache.put(entry); urlHashCache.put(urlHashCache.row().newEntry(entry));
} catch (IOException e) { } catch (IOException e) {
serverLog.logSevere("PLASMA", "INTERNAL ERROR AT plasmaNURL:store:" + e.toString() + ", resetting NURL-DB"); serverLog.logSevere("PLASMA", "INTERNAL ERROR AT plasmaNURL:store:" + e.toString() + ", resetting NURL-DB");
e.printStackTrace(); e.printStackTrace();

41
source/de/anomic/plasma/plasmaCrawlStacker.java
Unescape View File

@ -62,6 +62,7 @@ import de.anomic.http.httpc;
import de.anomic.index.indexURL; import de.anomic.index.indexURL;
import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroRow;
import de.anomic.kelondro.kelondroTree; import de.anomic.kelondro.kelondroTree;
import de.anomic.server.serverSemaphore; import de.anomic.server.serverSemaphore;
import de.anomic.server.logging.serverLog; import de.anomic.server.logging.serverLog;
@ -452,23 +453,23 @@ public final class plasmaCrawlStacker {
} }
} }
public stackCrawlMessage(String urlHash, byte[][] entryBytes) { public stackCrawlMessage(String urlHash, kelondroRow.Entry entry) {
if (urlHash == null) throw new NullPointerException(); if (urlHash == null) throw new NullPointerException();
if (entryBytes == null) throw new NullPointerException(); if (entry == null) throw new NullPointerException();
try { try {
this.urlHash = urlHash; this.urlHash = urlHash;
this.initiator = new String(entryBytes[1], "UTF-8"); this.initiator = entry.getColString(1, "UTF-8");
this.url = new String(entryBytes[2], "UTF-8").trim(); this.url = entry.getColString(2, "UTF-8").trim();
this.referrerHash = (entryBytes[3]==null) ? indexURL.dummyHash : new String(entryBytes[3], "UTF-8"); this.referrerHash = (entry.empty(3)) ? indexURL.dummyHash : entry.getColString(3, "UTF-8");
this.name = (entryBytes[4] == null) ? "" : new String(entryBytes[4], "UTF-8").trim(); this.name = (entry.empty(4)) ? "" : entry.getColString(4, "UTF-8").trim();
this.loaddate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[5], "UTF-8"))); this.loaddate = new Date(86400000 * entry.getColLongB64E(5));
this.profileHandle = (entryBytes[6] == null) ? null : new String(entryBytes[6], "UTF-8").trim(); this.profileHandle = (entry.empty(6)) ? null : entry.getColString(6, "UTF-8").trim();
this.depth = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[7], "UTF-8")); this.depth = (int) entry.getColLongB64E(7);
this.anchors = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[8], "UTF-8")); this.anchors = (int) entry.getColLongB64E(8);
this.forkfactor = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[9], "UTF-8")); this.forkfactor = (int) entry.getColLongB64E(9);
this.flags = new bitfield(entryBytes[10]); this.flags = new bitfield(entry.getColBytes(10));
this.handle = Integer.parseInt(new String(entryBytes[11], "UTF-8")); this.handle = Integer.parseInt(new String(entry.getColBytes(11), "UTF-8"));
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
throw new IllegalStateException(e.toString()); throw new IllegalStateException(e.toString());
@ -578,7 +579,7 @@ public final class plasmaCrawlStacker {
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent); this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent);
} catch (IOException e) { } catch (IOException e) {
cacheFile.delete(); cacheFile.delete();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.ce, true); this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(plasmaCrawlNURL.ce), true);
} }
try { try {
// loop through the list and fill the messageList with url hashs // loop through the list and fill the messageList with url hashs
@ -600,7 +601,7 @@ public final class plasmaCrawlStacker {
// deleting old db and creating a new db // deleting old db and creating a new db
try {this.urlEntryCache.close();}catch(Exception ex){} try {this.urlEntryCache.close();}catch(Exception ex){}
cacheFile.delete(); cacheFile.delete();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.ce, true); this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(plasmaCrawlNURL.ce), true);
} catch (IOException e) { } catch (IOException e) {
/* if we have an error, we start with a fresh database */ /* if we have an error, we start with a fresh database */
plasmaCrawlStacker.this.log.logSevere("Unable to initialize crawl stacker queue, IOException:" + e.getMessage() + ". Reseting DB.\n",e); plasmaCrawlStacker.this.log.logSevere("Unable to initialize crawl stacker queue, IOException:" + e.getMessage() + ". Reseting DB.\n",e);
@ -608,12 +609,12 @@ public final class plasmaCrawlStacker {
// deleting old db and creating a new db // deleting old db and creating a new db
try {this.urlEntryCache.close();}catch(Exception ex){} try {this.urlEntryCache.close();}catch(Exception ex){}
cacheFile.delete(); cacheFile.delete();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.ce, true); this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(plasmaCrawlNURL.ce), true);
} }
} else { } else {
// create new cache // create new cache
cacheFile.getParentFile().mkdirs(); cacheFile.getParentFile().mkdirs();
this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.ce, true); this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(plasmaCrawlNURL.ce), true);
} }
} }
@ -634,7 +635,7 @@ public final class plasmaCrawlStacker {
boolean insertionDoneSuccessfully = false; boolean insertionDoneSuccessfully = false;
synchronized(this.urlEntryHashCache) { synchronized(this.urlEntryHashCache) {
byte[][] oldValue = this.urlEntryCache.put(newMessage.getBytes()); kelondroRow.Entry oldValue = this.urlEntryCache.put(this.urlEntryCache.row().newEntry(newMessage.getBytes()));
if (oldValue == null) { if (oldValue == null) {
insertionDoneSuccessfully = this.urlEntryHashCache.add(newMessage.urlHash); insertionDoneSuccessfully = this.urlEntryHashCache.add(newMessage.urlHash);
} }
@ -659,7 +660,7 @@ public final class plasmaCrawlStacker {
this.writeSync.P(); this.writeSync.P();
String urlHash = null; String urlHash = null;
byte[][] entryBytes = null; kelondroRow.Entry entryBytes = null;
stackCrawlMessage newMessage = null; stackCrawlMessage newMessage = null;
try { try {
synchronized(this.urlEntryHashCache) { synchronized(this.urlEntryHashCache) {
@ -670,7 +671,7 @@ public final class plasmaCrawlStacker {
this.writeSync.V(); this.writeSync.V();
} }
newMessage = new stackCrawlMessage(urlHash,entryBytes); newMessage = new stackCrawlMessage(urlHash, entryBytes);
return newMessage; return newMessage;
} }
} }

26
source/de/anomic/plasma/plasmaSwitchboardQueue.java
Unescape View File

@ -80,22 +80,7 @@ public class plasmaSwitchboardQueue {
} }
private void initQueueStack() { private void initQueueStack() {
if (sbQueueStackPath.exists()) try { kelondroRow rowdef = new kelondroRow(new int[] {
sbQueueStack = new kelondroStack(sbQueueStackPath, 0);
} catch (IOException e) {
sbQueueStackPath.delete();
sbQueueStack = new kelondroStack(sbQueueStackPath, 0, new int[] {
indexURL.urlStringLength,
indexURL.urlHashLength,
11,
1,
yacySeedDB.commonHashLength,
indexURL.urlCrawlDepthLength,
indexURL.urlCrawlProfileHandleLength,
indexURL.urlDescrLength
}, true);
} else {
sbQueueStack = new kelondroStack(sbQueueStackPath, 0, new int[] {
indexURL.urlStringLength, indexURL.urlStringLength,
indexURL.urlHashLength, indexURL.urlHashLength,
11, 11,
@ -104,7 +89,14 @@ public class plasmaSwitchboardQueue {
indexURL.urlCrawlDepthLength, indexURL.urlCrawlDepthLength,
indexURL.urlCrawlProfileHandleLength, indexURL.urlCrawlProfileHandleLength,
indexURL.urlDescrLength indexURL.urlDescrLength
}, true); });
if (sbQueueStackPath.exists()) try {
sbQueueStack = new kelondroStack(sbQueueStackPath, 0);
} catch (IOException e) {
sbQueueStackPath.delete();
sbQueueStack = new kelondroStack(sbQueueStackPath, 0, rowdef, true);
} else {
sbQueueStack = new kelondroStack(sbQueueStackPath, 0, rowdef, true);
} }
} }

12
source/de/anomic/plasma/plasmaWordConnotation.java
Unescape View File

@ -48,8 +48,8 @@ package de.anomic.plasma;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroDynTree; import de.anomic.kelondro.kelondroDynTree;
import de.anomic.kelondro.kelondroRow;
public class plasmaWordConnotation { public class plasmaWordConnotation {
@ -62,19 +62,19 @@ public class plasmaWordConnotation {
if (refDBfile.exists()) try { if (refDBfile.exists()) try {
refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, fillChar); refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, fillChar);
} catch (IOException e) { } catch (IOException e) {
refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, wordlength, nodesize, new int[] {wordlength, countlength}, fillChar, true); refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, wordlength, nodesize, new kelondroRow(new int[] {wordlength, countlength}), fillChar, true);
} else { } else {
refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, wordlength, nodesize, new int[] {wordlength, countlength}, fillChar, true); refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, wordlength, nodesize, new kelondroRow(new int[] {wordlength, countlength}), fillChar, true);
} }
} }
private void addSingleRef(String word, String reference) throws IOException { private void addSingleRef(String word, String reference) throws IOException {
//word = word.toLowerCase(); //word = word.toLowerCase();
//reference = reference.toLowerCase(); //reference = reference.toLowerCase();
byte[][] record = refDB.get(word, reference.getBytes()); kelondroRow.Entry record = refDB.get(word, reference.getBytes());
long c; long c;
if (record == null) c = 0; else c = kelondroBase64Order.enhancedCoder.decodeLong(new String(record[1], "UTF-8")); if (record == null) c = 0; else c = record.getColLongB64E(1);
record[1] = kelondroBase64Order.enhancedCoder.encodeLong(c++, countlength).getBytes(); record.setColLongB64E(1, c++);
refDB.put(word, record); refDB.put(word, record);
} }

48
source/de/anomic/plasma/plasmaWordIndexAssortment.java
Unescape View File

@ -61,7 +61,6 @@ import de.anomic.index.indexEntryAttribute;
import de.anomic.index.indexTreeMapContainer; import de.anomic.index.indexTreeMapContainer;
import de.anomic.index.indexURLEntry; import de.anomic.index.indexURLEntry;
import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.kelondro.kelondroTree; import de.anomic.kelondro.kelondroTree;
import de.anomic.kelondro.kelondroRow; import de.anomic.kelondro.kelondroRow;
import de.anomic.server.logging.serverLog; import de.anomic.server.logging.serverLog;
@ -84,7 +83,6 @@ public final class plasmaWordIndexAssortment {
private serverLog log; private serverLog log;
private kelondroTree assortments; private kelondroTree assortments;
private long bufferSize; private long bufferSize;
private int bufferStructureLength;
private static String intx(int x) { private static String intx(int x) {
String s = Integer.toString(x); String s = Integer.toString(x);
@ -93,22 +91,22 @@ public final class plasmaWordIndexAssortment {
} }
private static int[] bufferStructure(int assortmentCapacity) { private static int[] bufferStructure(int assortmentCapacity) {
int[] structure = new int[3 + 2 * assortmentCapacity]; int[] structure = new int[3 + 2 * assortmentCapacity];
structure[0] = bufferStructureBasis[0]; structure[0] = bufferStructureBasis[0];
structure[1] = bufferStructureBasis[1]; structure[1] = bufferStructureBasis[1];
structure[2] = bufferStructureBasis[2]; structure[2] = bufferStructureBasis[2];
for (int i = 0; i < assortmentCapacity; i++) { for (int i = 0; i < assortmentCapacity; i++) {
structure[3 + 2 * i] = bufferStructureBasis[3]; structure[3 + 2 * i] = bufferStructureBasis[3];
structure[4 + 2 * i] = bufferStructureBasis[4]; structure[4 + 2 * i] = bufferStructureBasis[4];
} }
return structure; return structure;
} }
public plasmaWordIndexAssortment(File storagePath, int assortmentLength, int bufferkb, serverLog log) { public plasmaWordIndexAssortment(File storagePath, int assortmentLength, int bufferkb, serverLog log) {
if (!(storagePath.exists())) storagePath.mkdirs(); if (!(storagePath.exists())) storagePath.mkdirs();
this.assortmentFile = new File(storagePath, assortmentFileName + intx(assortmentLength) + ".db"); this.assortmentFile = new File(storagePath, assortmentFileName + intx(assortmentLength) + ".db");
this.assortmentLength = assortmentLength; this.assortmentLength = assortmentLength;
this.bufferStructureLength = 3 + 2 * assortmentLength; //this.bufferStructureLength = 3 + 2 * assortmentLength;
this.bufferSize = bufferkb * 1024; this.bufferSize = bufferkb * 1024;
this.log = log; this.log = log;
if (assortmentFile.exists()) { if (assortmentFile.exists()) {
@ -125,7 +123,7 @@ public final class plasmaWordIndexAssortment {
assortmentFile.delete(); // make space for new one assortmentFile.delete(); // make space for new one
} }
// create new assortment tree file // create new assortment tree file
assortments = new kelondroTree(assortmentFile, bufferSize, kelondroTree.defaultObjectCachePercent, bufferStructure(assortmentLength), true); assortments = new kelondroTree(assortmentFile, bufferSize, kelondroTree.defaultObjectCachePercent, new kelondroRow(bufferStructure(assortmentLength)), true);
if (log != null) log.logConfig("Created new Assortment Database, width " + assortmentLength + ", " + bufferkb + "kb buffer"); if (log != null) log.logConfig("Created new Assortment Database, width " + assortmentLength + ", " + bufferkb + "kb buffer");
} }
@ -134,18 +132,18 @@ public final class plasmaWordIndexAssortment {
// this throws an exception if the word hash already existed // this throws an exception if the word hash already existed
//log.logDebug("storeAssortment: wordHash=" + wordHash + ", urlHash=" + entry.getUrlHash() + ", time=" + creationTime); //log.logDebug("storeAssortment: wordHash=" + wordHash + ", urlHash=" + entry.getUrlHash() + ", time=" + creationTime);
if (newContainer.size() != assortmentLength) throw new RuntimeException("plasmaWordIndexAssortment.store: wrong container size"); if (newContainer.size() != assortmentLength) throw new RuntimeException("plasmaWordIndexAssortment.store: wrong container size");
byte[][] row = new byte[this.bufferStructureLength][]; kelondroRow.Entry row = assortments.row().newEntry();
row[0] = newContainer.wordHash().getBytes(); row.setCol(0, newContainer.wordHash().getBytes());
row[1] = kelondroNaturalOrder.encodeLong(1, 4); row.setColLongB256(1, 1);
row[2] = kelondroNaturalOrder.encodeLong(newContainer.updated(), 8); row.setColLongB256(2, newContainer.updated());
Iterator entries = newContainer.entries(); Iterator entries = newContainer.entries();
indexURLEntry entry; indexURLEntry entry;
for (int i = 0; i < assortmentLength; i++) { for (int i = 0; i < assortmentLength; i++) {
entry = (indexURLEntry) entries.next(); entry = (indexURLEntry) entries.next();
row[3 + 2 * i] = entry.getUrlHash().getBytes(); row.setCol(3 + 2 * i, entry.getUrlHash().getBytes());
row[4 + 2 * i] = entry.toEncodedStringForm().getBytes(); row.setCol(4 + 2 * i, entry.toEncodedStringForm().getBytes());
} }
byte[][] oldrow = null; kelondroRow.Entry oldrow = null;
try { try {
oldrow = assortments.put(row); oldrow = assortments.put(row);
} catch (IOException e) { } catch (IOException e) {
@ -163,7 +161,7 @@ public final class plasmaWordIndexAssortment {
// and returns the content record // and returns the content record
kelondroRow.Entry row = null; kelondroRow.Entry row = null;
try { try {
row = assortments.row().newEntry(assortments.remove(wordHash.getBytes())); row = assortments.remove(wordHash.getBytes());
} catch (IOException e) { } catch (IOException e) {
log.logSevere("removeAssortment/IO-error: " + e.getMessage() log.logSevere("removeAssortment/IO-error: " + e.getMessage()
+ " - reset assortment-DB " + assortments.file(), e); + " - reset assortment-DB " + assortments.file(), e);
@ -245,7 +243,7 @@ public final class plasmaWordIndexAssortment {
if (!(assortmentFile.delete())) throw new RuntimeException("cannot delete assortment database"); if (!(assortmentFile.delete())) throw new RuntimeException("cannot delete assortment database");
} }
if (assortmentFile.exists()) assortmentFile.delete(); if (assortmentFile.exists()) assortmentFile.delete();
assortments = new kelondroTree(assortmentFile, bufferSize, kelondroTree.defaultObjectCachePercent, bufferStructure(assortmentLength), true); assortments = new kelondroTree(assortmentFile, bufferSize, kelondroTree.defaultObjectCachePercent, new kelondroRow(bufferStructure(assortmentLength)), true);
} }
public Iterator hashes(String startWordHash, boolean up, boolean rot) throws IOException { public Iterator hashes(String startWordHash, boolean up, boolean rot) throws IOException {

38
source/de/anomic/yacy/yacyNewsDB.java
Unescape View File

@ -50,6 +50,7 @@ import java.util.Iterator;
import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacyCore;
import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroColumn;
import de.anomic.kelondro.kelondroTree; import de.anomic.kelondro.kelondroTree;
import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroRow; import de.anomic.kelondro.kelondroRow;
@ -80,15 +81,17 @@ public class yacyNewsDB {
news = createDB(path, bufferkb); news = createDB(path, bufferkb);
} }
} }
public static final kelondroRow rowdef = new kelondroRow(new kelondroColumn[]{
new kelondroColumn(kelondroColumn.celltype_string, yacyNewsRecord.idLength(), "newsid", "id = created + originator"),
new kelondroColumn(kelondroColumn.celltype_string, yacyNewsRecord.categoryStringLength, "category", ""),
new kelondroColumn(kelondroColumn.celltype_string, yacyCore.universalDateShortPattern.length(), "received", ""),
new kelondroColumn(kelondroColumn.celltype_string, 2, "", ""),
new kelondroColumn(kelondroColumn.celltype_string, attributesMaxLength, "", ""),
});
private static kelondroTree createDB(File path, int bufferkb) { private static kelondroTree createDB(File path, int bufferkb) {
return new kelondroTree(path, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new int[] { return new kelondroTree(path, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, rowdef, true);
yacyNewsRecord.idLength(), // id = created + originator
yacyNewsRecord.categoryStringLength, // category
yacyCore.universalDateShortPattern.length(), // received
2,
attributesMaxLength
}, true);
} }
private void resetDB() { private void resetDB() {
@ -153,7 +156,7 @@ public class yacyNewsDB {
} }
public Object next() { public Object next() {
return b2r((byte[][]) rowIterator.next()); return b2r((kelondroRow.Entry) rowIterator.next());
} }
public void remove() { public void remove() {
@ -181,6 +184,7 @@ public class yacyNewsDB {
); );
} }
/*
private static yacyNewsRecord b2r(byte[][] b) { private static yacyNewsRecord b2r(byte[][] b) {
if (b == null) return null; if (b == null) return null;
return new yacyNewsRecord( return new yacyNewsRecord(
@ -191,18 +195,18 @@ public class yacyNewsDB {
serverCodings.string2map(new String(b[4])) serverCodings.string2map(new String(b[4]))
); );
} }
*/
private static byte[][] r2b(yacyNewsRecord r) { private kelondroRow.Entry r2b(yacyNewsRecord r) {
if (r == null) return null; if (r == null) return null;
String attributes = r.attributes().toString(); String attributes = r.attributes().toString();
if (attributes.length() > attributesMaxLength) throw new IllegalArgumentException("attribute length=" + attributes.length() + " exceeds maximum size=" + attributesMaxLength); if (attributes.length() > attributesMaxLength) throw new IllegalArgumentException("attribute length=" + attributes.length() + " exceeds maximum size=" + attributesMaxLength);
byte[][] b = new byte[5][]; kelondroRow.Entry entry = news.row().newEntry();
b[0] = r.id().getBytes(); entry.setCol(0, r.id().getBytes());
b[1] = r.category().getBytes(); entry.setCol(1, r.category().getBytes());
b[2] = (r.received() == null) ? null : yacyCore.universalDateShortString(r.received()).getBytes(); entry.setCol(2, (r.received() == null) ? null : yacyCore.universalDateShortString(r.received()).getBytes());
b[3] = kelondroBase64Order.enhancedCoder.encodeLong(r.distributed(), 2).getBytes(); entry.setCol(3, kelondroBase64Order.enhancedCoder.encodeLong(r.distributed(), 2).getBytes());
b[4] = attributes.getBytes(); entry.setCol(4, attributes.getBytes());
return b; return entry;
} }
} }

11
source/de/anomic/yacy/yacyNewsQueue.java
Unescape View File

@ -48,6 +48,7 @@ import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.Date; import java.util.Date;
import de.anomic.kelondro.kelondroColumn;
import de.anomic.kelondro.kelondroRow; import de.anomic.kelondro.kelondroRow;
import de.anomic.kelondro.kelondroStack; import de.anomic.kelondro.kelondroStack;
import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroException;
@ -74,12 +75,14 @@ public class yacyNewsQueue {
queueStack = createStack(path); queueStack = createStack(path);
} }
} }
public static final kelondroRow rowdef = new kelondroRow(new kelondroColumn[]{
new kelondroColumn(kelondroColumn.celltype_string, yacyNewsRecord.idLength(), "newsid", "id = created + originator"),
new kelondroColumn(kelondroColumn.celltype_string, yacyCore.universalDateShortPattern.length(), "last touched", "")
});
private static kelondroStack createStack(File path) { private static kelondroStack createStack(File path) {
return new kelondroStack(path, 0, new int[] { return new kelondroStack(path, 0, rowdef, true);
yacyNewsRecord.idLength(), // id = created + originator
yacyCore.universalDateShortPattern.length() // last touched
}, true);
} }
private void resetDB() { private void resetDB() {

Write Preview
Loading…
Cancel
Save