From c36e9fc8d3043f632d7b29d02a0bb5728d3c26f1 Mon Sep 17 00:00:00 2001 From: orbiter Date: Fri, 2 Jun 2006 12:45:57 +0000 Subject: [PATCH] full integration of kelondroRow git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2167 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/dbtest.java | 28 ++++--- source/de/anomic/index/indexRAMCacheRI.java | 2 +- source/de/anomic/kelondro/kelondroArray.java | 2 +- .../anomic/kelondro/kelondroBase64Order.java | 42 ++++++----- .../kelondro/kelondroCollectionIndex.java | 45 +++++------ source/de/anomic/kelondro/kelondroDyn.java | 16 ++-- .../de/anomic/kelondro/kelondroDynTree.java | 75 ++++++------------- .../kelondro/kelondroFScoreCluster.java | 8 +- .../kelondro/kelondroFixedWidthArray.java | 12 +-- .../de/anomic/kelondro/kelondroFlexTable.java | 18 ++--- .../kelondro/kelondroFlexWidthArray.java | 33 ++++---- .../de/anomic/kelondro/kelondroHashtable.java | 16 ++-- source/de/anomic/kelondro/kelondroIndex.java | 7 +- source/de/anomic/kelondro/kelondroMap.java | 2 +- .../de/anomic/kelondro/kelondroMapTable.java | 8 +- .../anomic/kelondro/kelondroNaturalOrder.java | 14 ++-- .../de/anomic/kelondro/kelondroRecords.java | 39 ++++------ source/de/anomic/kelondro/kelondroRow.java | 15 +++- .../anomic/kelondro/kelondroSplittedTree.java | 30 ++++---- source/de/anomic/kelondro/kelondroStack.java | 18 ++--- source/de/anomic/kelondro/kelondroTree.java | 65 ++++++++-------- .../de/anomic/plasma/plasmaCrawlBalancer.java | 5 +- source/de/anomic/plasma/plasmaCrawlEURL.java | 10 +-- source/de/anomic/plasma/plasmaCrawlLURL.java | 8 +- source/de/anomic/plasma/plasmaCrawlNURL.java | 19 ++--- .../de/anomic/plasma/plasmaCrawlStacker.java | 41 +++++----- .../anomic/plasma/plasmaSwitchboardQueue.java | 26 +++---- .../anomic/plasma/plasmaWordConnotation.java | 12 +-- .../plasma/plasmaWordIndexAssortment.java | 48 ++++++------ source/de/anomic/yacy/yacyNewsDB.java | 38 +++++----- source/de/anomic/yacy/yacyNewsQueue.java | 11 ++- 31 files changed, 343 insertions(+), 370 deletions(-) diff --git a/source/dbtest.java b/source/dbtest.java index af7049e95..dc70e71f5 100644 --- a/source/dbtest.java +++ b/source/dbtest.java @@ -116,7 +116,7 @@ public class dbtest { public void run() { final STEntry entry = new STEntry(this.getSource()); try { - getTable().put(new byte[][] { entry.getKey(), entry.getValue() , entry.getValue() }); + getTable().put(getTable().row().newEntry(new byte[][] { entry.getKey(), entry.getValue() , entry.getValue() })); } catch (IOException e) { System.err.println(e); e.printStackTrace(); @@ -166,12 +166,13 @@ public class dbtest { profiler.start(); // create the database access + kelondroRow testRow = new kelondroRow(new int[]{keylength, valuelength, valuelength}); if (dbe.equals("kelondroTree")) { File tablefile = new File(tablename + ".kelondro.db"); if (tablefile.exists()) { table = new kelondroTree(tablefile, buffer, kelondroTree.defaultObjectCachePercent); } else { - table = new kelondroTree(tablefile, buffer, kelondroTree.defaultObjectCachePercent, new int[]{keylength, valuelength, valuelength}, true); + table = new kelondroTree(tablefile, buffer, kelondroTree.defaultObjectCachePercent, testRow, true); } } if (dbe.equals("kelondroSplittedTree")) { @@ -179,7 +180,7 @@ public class dbtest { table = kelondroSplittedTree.open(tablepath, tablename, kelondroBase64Order.enhancedCoder, buffer, 8, - new int[]{keylength, valuelength, valuelength}, 1, 80, + testRow, 1, 80, true); } if (dbe.equals("kelondroFlexTable")) { @@ -187,11 +188,11 @@ public class dbtest { table = new kelondroFlexTable(tablepath, new File(tablename).getName(), new kelondroRow(new int[]{keylength, valuelength, valuelength}), true); } if (dbe.equals("mysql")) { - table = new dbTable("mysql"); + table = new dbTable("mysql", testRow); } if (dbe.equals("pgsql")) { - table = new dbTable("pgsql"); + table = new dbTable("pgsql", testRow); } long afterinit = System.currentTimeMillis(); @@ -210,7 +211,7 @@ public class dbtest { long randomstart = Long.parseLong(args[4]); Random random = new Random(randomstart); for (int i = 0; i < count; i++) { - table.put(new byte[][]{randomHash(random), dummyvalue1, dummyvalue2}); + table.put(table.row().newEntry(new byte[][]{randomHash(random), dummyvalue1, dummyvalue2})); if (i % 500 == 0) { System.out.println(i + " entries processed so far."); } @@ -305,8 +306,10 @@ final class dbTable implements kelondroIndex { private PreparedStatement sqlStatement; private int commandCount = 0; private int batchlimit = 1; + private kelondroRow rowdef; - public dbTable(String dbType) throws Exception { + public dbTable(String dbType, kelondroRow rowdef) throws Exception { + this.rowdef = rowdef; openDatabaseConnection(dbType); } @@ -357,15 +360,18 @@ final class dbTable implements kelondroIndex { } } + public kelondroRow row() { + return this.rowdef; + } public kelondroRow.Entry get(byte[] key) throws IOException { return null; } - public byte[][] put(byte[][] row) throws IOException { + public kelondroRow.Entry put(kelondroRow.Entry row) throws IOException { try { - this.sqlStatement.setString(1,new String(row[0])); - sqlStatement.setBytes(2,row[1]); + this.sqlStatement.setString(1, new String(row.getColString(0, null))); + sqlStatement.setBytes(2, row.getColBytes(1)); sqlStatement.addBatch(); commandCount++; @@ -380,7 +386,7 @@ final class dbTable implements kelondroIndex { } } - public byte[][] remove(byte[] key) throws IOException { + public kelondroRow.Entry remove(byte[] key) throws IOException { // TODO Auto-generated method stub return null; } diff --git a/source/de/anomic/index/indexRAMCacheRI.java b/source/de/anomic/index/indexRAMCacheRI.java index 79af1e271..24ab9c85e 100644 --- a/source/de/anomic/index/indexRAMCacheRI.java +++ b/source/de/anomic/index/indexRAMCacheRI.java @@ -94,7 +94,7 @@ public final class indexRAMCacheRI extends indexAbstractRI implements indexRI { File indexDumpFile = new File(databaseRoot, indexArrayFileName); if (indexDumpFile.exists()) indexDumpFile.delete(); kelondroFixedWidthArray dumpArray = null; - dumpArray = new kelondroFixedWidthArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0, false); + dumpArray = new kelondroFixedWidthArray(indexDumpFile, new kelondroRow(plasmaWordIndexAssortment.bufferStructureBasis), 0, false); long startTime = System.currentTimeMillis(); long messageTime = System.currentTimeMillis() + 5000; long wordsPerSecond = 0, wordcount = 0, urlcount = 0; diff --git a/source/de/anomic/kelondro/kelondroArray.java b/source/de/anomic/kelondro/kelondroArray.java index 09632a751..de44c46fa 100644 --- a/source/de/anomic/kelondro/kelondroArray.java +++ b/source/de/anomic/kelondro/kelondroArray.java @@ -30,7 +30,7 @@ public interface kelondroArray { public int size(); - public int columns(); + public kelondroRow row(); public kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException; diff --git a/source/de/anomic/kelondro/kelondroBase64Order.java b/source/de/anomic/kelondro/kelondroBase64Order.java index dcfd20d09..d87c663fe 100644 --- a/source/de/anomic/kelondro/kelondroBase64Order.java +++ b/source/de/anomic/kelondro/kelondroBase64Order.java @@ -88,7 +88,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond return o; } - public static kelondroOrder bySignature(String signature) { + public final static kelondroOrder bySignature(String signature) { if (signature.equals("Bd")) return new kelondroBase64Order(false, false); if (signature.equals("bd")) return new kelondroBase64Order(false, true); if (signature.equals("Bu")) return new kelondroBase64Order(true, false); @@ -96,7 +96,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond return null; } - public String signature() { + public final String signature() { if ((!asc) && (!rfc1113compliant)) return "Bd"; if ((!asc) && ( rfc1113compliant)) return "bd"; if (( asc) && (!rfc1113compliant)) return "Bu"; @@ -104,15 +104,15 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond return null; } - public char encodeByte(byte b) { + public final char encodeByte(byte b) { return (char) alpha[b]; } - public byte decodeByte(char b) { + public final byte decodeByte(char b) { return ahpla[b]; } - public String encodeLongSmart(long c, int length) { + public final String encodeLongSmart(long c, int length) { if (c >= max(length)) { StringBuffer s = new StringBuffer(length); s.setLength(length); @@ -123,7 +123,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond } } - public String encodeLong(long c, int length) { + public final String encodeLong(long c, int length) { StringBuffer s = new StringBuffer(length); s.setLength(length); while (length > 0) { @@ -133,14 +133,22 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond return s.toString(); } - public long decodeLong(String s) { + public final void encodeLong(long c, byte[] b, int offset, int length) { + assert offset + length <= b.length; + while (length > 0) { + b[--length + offset] = (byte) alpha[(byte) (c & 0x3F)]; + c >>= 6; + } + } + + public final long decodeLong(String s) { while (s.endsWith("=")) s = s.substring(0, s.length() - 1); long c = 0; for (int i = 0; i < s.length(); i++) c = (c << 6) | ahpla[s.charAt(i)]; return c; } - public long decodeLong(byte[] s, int offset, int length) { + public final long decodeLong(byte[] s, int offset, int length) { while ((length > 0) && (s[offset + length - 1] == '=')) length--; long c = 0; for (int i = 0; i < length; i++) c = (c << 6) | ahpla[s[offset + i]]; @@ -155,14 +163,14 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond return c; } - public String encodeString(String in) { + public final String encodeString(String in) { return encode(in.getBytes()); } // we will use this encoding to encode strings with 2^8 values to // b64-Strings // we will do that by grouping each three input bytes to four output bytes. - public String encode(byte[] in) { + public final String encode(byte[] in) { StringBuffer out = new StringBuffer(in.length / 3 * 4 + 3); int pos = 0; long l; @@ -181,7 +189,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond return out.toString(); } - public String decodeString(String in) { + public final String decodeString(String in) { try { //return new String(decode(in), "ISO-8859-1"); return new String(decode(in), "UTF-8"); @@ -191,7 +199,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond } } - public byte[] decode(String in) { + public final byte[] decode(String in) { try { int posIn = 0; int posOut = 0; @@ -233,7 +241,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond } } - private long cardinalI(byte[] key) { + private final long cardinalI(byte[] key) { // returns a cardinal number in the range of 0 .. Long.MAX_VALUE long c = 0; int p = 0; @@ -243,7 +251,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond return c; } - public long cardinal(byte[] key) { + public final long cardinal(byte[] key) { if (this.zero == null) return cardinalI(key); long zeroCardinal = cardinalI(this.zero); long keyCardinal = cardinalI(key); @@ -251,11 +259,11 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond return Long.MAX_VALUE - keyCardinal + zeroCardinal + 1; } - public int compare(byte[] a, byte[] b) { + public final int compare(byte[] a, byte[] b) { return (asc) ? compare0(a, b) : compare0(b, a); } - public int compare0(byte[] a, byte[] b) { + public final int compare0(byte[] a, byte[] b) { if (zero == null) return compares(a, b); // we have an artificial start point. check all combinations int az = compares(a, zero); // -1 if a < z; 0 if a == z; 1 if a > z @@ -267,7 +275,7 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond return bz; } - public int compares(byte[] a, byte[] b) { + public final int compares(byte[] a, byte[] b) { int i = 0; final int al = a.length; final int bl = b.length; diff --git a/source/de/anomic/kelondro/kelondroCollectionIndex.java b/source/de/anomic/kelondro/kelondroCollectionIndex.java index 80570981a..0a3119b58 100644 --- a/source/de/anomic/kelondro/kelondroCollectionIndex.java +++ b/source/de/anomic/kelondro/kelondroCollectionIndex.java @@ -72,7 +72,7 @@ public class kelondroCollectionIndex { columns[2] = 4; // chunkcount (number of chunks in this collection) columns[3] = 4; // index (position in index file) columns[4] = 2; // update time in days since 1.1.2000 - index = new kelondroSplittedTree(path, filenameStub, indexOrder, buffersize, 8, columns, 1, 80, true); + index = new kelondroSplittedTree(path, filenameStub, indexOrder, buffersize, 8, new kelondroRow(columns), 1, 80, true); // create array files this.array = new kelondroFixedWidthArray[partitions]; @@ -97,13 +97,13 @@ public class kelondroCollectionIndex { } else { int load = 1; for (int i = 0; i < partitionNumber; i++) load = load * loadfactor; int[] columns = new int[4]; - columns[0] = index.columnSize(0); // add always the key + columns[0] = index.row().width(0); // add always the key columns[1] = 4; // chunkcount (raw format) columns[2] = 2; // last time read columns[3] = 2; // last time wrote columns[4] = 2; // flag string, assigns collection order as currently stored in table columns[5] = load * genericChunkSize; - return new kelondroFixedWidthArray(f, columns, 0, true); + return new kelondroFixedWidthArray(f, new kelondroRow(columns), 0, true); } } @@ -134,12 +134,13 @@ public class kelondroCollectionIndex { // write a new entry in this array int newRowNumber = array[part].add(array[part].row().newEntry(newarrayrow)); // store the new row number in the index - index.put(new byte[][]{key, - kelondroNaturalOrder.encodeLong(this.chunksize, 4), - kelondroNaturalOrder.encodeLong(collection.size(), 4), - kelondroNaturalOrder.encodeLong((long) newRowNumber, 4), - kelondroNaturalOrder.encodeLong(daysSince2000(System.currentTimeMillis()), 2) - }); + kelondroRow.Entry e = index.row().newEntry(); + e.setCol(0, key); + e.setColLongB256(1, this.chunksize); + e.setColLongB256(2, collection.size()); + e.setColLongB256(3, (long) newRowNumber); + e.setColLongB256(4, daysSince2000(System.currentTimeMillis())); + index.put(e); } else { // overwrite the old collection // read old information @@ -154,24 +155,26 @@ public class kelondroCollectionIndex { // we don't need a new slot, just write in the old one array[oldPartitionNumber].set(rownumber, array[oldPartitionNumber].row().newEntry(newarrayrow)); // update the index entry - index.put(new byte[][]{key, - kelondroNaturalOrder.encodeLong(this.chunksize, 4), - kelondroNaturalOrder.encodeLong(collection.size(), 4), - kelondroNaturalOrder.encodeLong((long) rownumber, 4), - kelondroNaturalOrder.encodeLong(daysSince2000(System.currentTimeMillis()), 2) - }); + kelondroRow.Entry e = index.row().newEntry(); + e.setCol(0, key); + e.setColLongB256(1, this.chunksize); + e.setColLongB256(2, collection.size()); + e.setColLongB256(3, (long) rownumber); + e.setColLongB256(4, daysSince2000(System.currentTimeMillis())); + index.put(e); } else { // we need a new slot, that means we must first delete the old entry array[oldPartitionNumber].remove(rownumber); // write a new entry in the other array int newRowNumber = array[newPartitionNumber].add(array[newPartitionNumber].row().newEntry(newarrayrow)); // store the new row number in the index - index.put(new byte[][]{key, - kelondroNaturalOrder.encodeLong(this.chunksize, 4), - kelondroNaturalOrder.encodeLong(collection.size(), 4), - kelondroNaturalOrder.encodeLong((long) newRowNumber, 4), - kelondroNaturalOrder.encodeLong(daysSince2000(System.currentTimeMillis()), 2) - }); + kelondroRow.Entry e = index.row().newEntry(); + e.setCol(0, key); + e.setColLongB256(1, this.chunksize); + e.setColLongB256(2, collection.size()); + e.setColLongB256(3, (long) newRowNumber); + e.setColLongB256(4, daysSince2000(System.currentTimeMillis())); + index.put(e); } } } diff --git a/source/de/anomic/kelondro/kelondroDyn.java b/source/de/anomic/kelondro/kelondroDyn.java index f7f8b6649..ba6a673a2 100644 --- a/source/de/anomic/kelondro/kelondroDyn.java +++ b/source/de/anomic/kelondro/kelondroDyn.java @@ -76,9 +76,9 @@ public class kelondroDyn extends kelondroTree { int nodesize, char fillChar, kelondroOrder objectOrder, boolean exitOnFail) { // creates a new dynamic tree - super(file, buffersize, kelondroTree.defaultObjectCachePercent, new int[] { key + counterlen, nodesize }, objectOrder, 1, 8, exitOnFail); - this.keylen = columnSize(0) - counterlen; - this.reclen = columnSize(1); + super(file, buffersize, kelondroTree.defaultObjectCachePercent, new kelondroRow(new int[] { key + counterlen, nodesize }), objectOrder, 1, 8, exitOnFail); + this.keylen = row().width(0) - counterlen; + this.reclen = row().width(1); this.fillChar = fillChar; this.segmentCount = 0; writeSegmentCount(); @@ -88,8 +88,8 @@ public class kelondroDyn extends kelondroTree { public kelondroDyn(File file, long buffersize, char fillChar) throws IOException { // this opens a file with an existing dynamic tree super(file, buffersize, kelondroTree.defaultObjectCachePercent); - this.keylen = columnSize(0) - counterlen; - this.reclen = columnSize(1); + this.keylen = row().width(0) - counterlen; + this.reclen = row().width(1); this.fillChar = fillChar; this.segmentCount = 0; buffer = new kelondroObjectBuffer(file.toString()); @@ -159,11 +159,11 @@ public class kelondroDyn extends kelondroTree { String k; String v; int c; - byte[][] nt; + kelondroRow.Entry nt; while (ri.hasNext()) { - nt = (byte[][]) ri.next(); + nt = (kelondroRow.Entry) ri.next(); if (nt == null) throw new kelondroException(filename, "no more elements available"); - g = nt[0]; + g = nt.getColBytes(0); if (g == null) return null; k = new String(g, 0, keylen); v = new String(g, keylen, counterlen); diff --git a/source/de/anomic/kelondro/kelondroDynTree.java b/source/de/anomic/kelondro/kelondroDynTree.java index 005e71d84..c0e63d773 100644 --- a/source/de/anomic/kelondro/kelondroDynTree.java +++ b/source/de/anomic/kelondro/kelondroDynTree.java @@ -53,7 +53,7 @@ import java.util.Iterator; public class kelondroDynTree { // basic data structures - private int[] columns; + private kelondroRow rowdef; private kelondroDyn table; private Hashtable treeRAHandles; private File file; @@ -69,10 +69,10 @@ public class kelondroDynTree { private Hashtable buffer, cache; private long cycleBuffer; - public kelondroDynTree(File file, long buffersize, int keylength, int nodesize, int[] columns, char fillChar, boolean exitOnFail) { + public kelondroDynTree(File file, long buffersize, int keylength, int nodesize, kelondroRow rowdef, char fillChar, boolean exitOnFail) { // creates a new DynTree this.file = file; - this.columns = columns; + this.rowdef = rowdef; this.buffer = new Hashtable(); this.cache = new Hashtable(); //this.cycleCache = Long.MIN_VALUE; @@ -97,8 +97,9 @@ public class kelondroDynTree { Iterator i = table.dynKeys(true, false); String onekey = (String) i.next(); kelondroTree onetree = getTree(onekey); - this.columns = new int[onetree.columns()]; - for (int j = 0; j < columns.length; j++) columns[j] = onetree.columnSize(j); + int[] columns = new int[onetree.row().columns()]; + for (int j = 0; j < columns.length; j++) columns[j] = onetree.row().width(j); + this.rowdef = new kelondroRow(columns); closeTree(onekey); } @@ -109,15 +110,6 @@ public class kelondroDynTree { table.close(); if (size == 0) this.file.delete(); } - - /* - public void setReadCacheAttr(int maxcount, int maxsize, long maxage, long cycletime) { - maxcountCache = maxcount; - maxsizeCache = maxsize; - maxageCache = maxage; - cycletimeCache = cycletime; - } - */ public void setWriteBufferAttr(int maxcount, int maxsize, long maxage, long cycletime) { maxcountBuffer = maxcount; @@ -135,7 +127,7 @@ public class kelondroDynTree { kelondroRA ra = table.getRA(key); // works always, even with no-existing entry treeRAHandles.put(key, ra); try { - return new kelondroTree(ra, buffersize, kelondroTree.defaultObjectCachePercent, columns, false); + return new kelondroTree(ra, buffersize, kelondroTree.defaultObjectCachePercent, rowdef, false); } catch (RuntimeException e) { throw new IOException(e.getMessage()); } @@ -183,11 +175,11 @@ public class kelondroDynTree { this.timestamp = Long.MAX_VALUE; // to flag no-update } - public byte[][] get(byte[] key) throws IOException { - byte[][] entry = (byte[][]) tcache.get(key); + public kelondroRow.Entry get(byte[] key) throws IOException { + kelondroRow.Entry entry = (kelondroRow.Entry) tcache.get(key); if (entry == null) { kelondroTree t = getTree(this.tablename); - entry = t.get(key).getCols(); + entry = t.get(key); t.close(); this.tcache.put(key, entry); this.timestamp = System.currentTimeMillis(); @@ -195,8 +187,8 @@ public class kelondroDynTree { return entry; } - protected void put(byte[][] entry) { // this is only used internal - this.tcache.put(entry[0], entry); + protected void put(kelondroRow.Entry entry) { // this is only used internal + this.tcache.put(entry.getColBytes(0), entry); this.timestamp = System.currentTimeMillis(); } @@ -218,8 +210,8 @@ public class kelondroDynTree { this.timestamp = Long.MAX_VALUE; // to flag no-update } - public void put(byte[][] entry) { - this.tbuffer.put(entry[0], entry); + public void put(kelondroRow.Entry entry) { + this.tbuffer.put(entry.getColBytes(0), entry); this.timestamp = System.currentTimeMillis(); } @@ -233,11 +225,11 @@ public class kelondroDynTree { if (this.tbuffer.size() == 0) return; Enumeration e = this.tbuffer.keys(); kelondroTree t = getTree(this.tablename); - byte[][] entry; + kelondroRow.Entry entry; byte[] key; while (e.hasMoreElements()) { key = (byte[]) e.nextElement(); - entry = (byte[][]) this.tbuffer.get(key); + entry = (kelondroRow.Entry) this.tbuffer.get(key); t.put(entry); } t.close(); @@ -248,7 +240,7 @@ public class kelondroDynTree { // read cached - public synchronized byte[][] get(String tablename, byte[] key) throws IOException { + public synchronized kelondroRow.Entry get(String tablename, byte[] key) throws IOException { treeCache tc = (treeCache) cache.get(table); if (tc == null) { tc = new treeCache(tablename); @@ -256,31 +248,9 @@ public class kelondroDynTree { } return tc.get(key); } - - /* - // clean-up method for cache: - private void flushCache() { - if ((System.currentTimeMillis() - this.cycleCache < this.cycletimeCache) && - (cache.size() < this.maxcountCache)) return; - this.cycleCache = System.currentTimeMillis(); - // collect all caches which have a time > maxagecache - Enumeration e = cache.keys(); - String tablename; - treeCache tc; - while (e.hasMoreElements()) { - tablename = (String) e.nextElement(); - tc = (treeCache) cache.get(tablename); - if ((System.currentTimeMillis() - tc.timestamp > this.maxageCache) || - (tc.cache.size() > this.maxsizeCache) || - (cache.size() > this.maxcountCache)) { - cache.remove(tablename); - } - } - } - */ // write buffered - public synchronized void put(String tablename, byte[][] newrow) { + public synchronized void put(String tablename, kelondroRow.Entry newrow) { treeBuffer tb = (treeBuffer) buffer.get(tablename); if (tb == null) { tb = new treeBuffer(tablename); @@ -350,17 +320,18 @@ public class kelondroDynTree { File file = new File("D:\\bin\\testDyn.db"); if (file.exists()) { kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, '_'); - System.out.println("opened: table keylength=" + dt.table.columnSize(0) + ", sectorsize=" + dt.table.columnSize(1) + ", " + dt.table.size() + " entries."); + System.out.println("opened: table keylength=" + dt.table.row().width(0) + ", sectorsize=" + dt.table.row().width(1) + ", " + dt.table.size() + " entries."); } else { - kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, 16, 512, new int[] {10,20,30}, '_', true); + kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, 16, 512, new kelondroRow(new int[] {10,20,30}), '_', true); String name; kelondroTree t; - byte[][] line = new byte[][] {"".getBytes(), "abc".getBytes(), "def".getBytes()}; + kelondroRow.Entry line; for (int i = 1; i < 100; i++) { name = "test" + i; t = dt.newTree(name); + line = t.row().newEntry(new byte[][] {"".getBytes(), "abc".getBytes(), "def".getBytes()}); for (int j = 1; j < 10; j++) { - line[0] = ("entry" + j).getBytes(); + line.setCol(0, ("entry" + j).getBytes()); t.put(line); } dt.closeTree(name); diff --git a/source/de/anomic/kelondro/kelondroFScoreCluster.java b/source/de/anomic/kelondro/kelondroFScoreCluster.java index 39e17fe15..7dc89323c 100644 --- a/source/de/anomic/kelondro/kelondroFScoreCluster.java +++ b/source/de/anomic/kelondro/kelondroFScoreCluster.java @@ -70,11 +70,11 @@ public class kelondroFScoreCluster { } catch (IOException e) { refcountDBfile.delete(); countrefDBfile.delete(); - refcountDB = new kelondroTree(refcountDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new int[] {wordlength, countlength}, objectOrder, 1, countlength, exitOnFail); - countrefDB = new kelondroTree(countrefDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new int[] {countlength + wordlength, 4}, objectOrder, 1, countlength, exitOnFail); + refcountDB = new kelondroTree(refcountDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new kelondroRow(new int[] {wordlength, countlength}), objectOrder, 1, countlength, exitOnFail); + countrefDB = new kelondroTree(countrefDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new kelondroRow(new int[] {countlength + wordlength, 4}), objectOrder, 1, countlength, exitOnFail); } else if ((!(refcountDBfile.exists())) && (!(countrefDBfile.exists()))) { - refcountDB = new kelondroTree(refcountDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new int[] {wordlength, countlength}, objectOrder, 1, countlength, exitOnFail); - countrefDB = new kelondroTree(countrefDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new int[] {countlength + wordlength, 4}, objectOrder, 1, countlength, exitOnFail); + refcountDB = new kelondroTree(refcountDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new kelondroRow(new int[] {wordlength, countlength}), objectOrder, 1, countlength, exitOnFail); + countrefDB = new kelondroTree(countrefDBfile, 0x100000L, kelondroTree.defaultObjectCachePercent, new kelondroRow(new int[] {countlength + wordlength, 4}), objectOrder, 1, countlength, exitOnFail); } else { if (exitOnFail) { System.exit(-1); diff --git a/source/de/anomic/kelondro/kelondroFixedWidthArray.java b/source/de/anomic/kelondro/kelondroFixedWidthArray.java index 10c49a41a..793bebc16 100644 --- a/source/de/anomic/kelondro/kelondroFixedWidthArray.java +++ b/source/de/anomic/kelondro/kelondroFixedWidthArray.java @@ -55,9 +55,9 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro private static short thisOHBytes = 0; // our record definition does not need extra bytes private static short thisOHHandles = 0; // and no handles - public kelondroFixedWidthArray(File file, int[] columns, int intprops, boolean exitOnFail) { + public kelondroFixedWidthArray(File file, kelondroRow rowdef, int intprops, boolean exitOnFail) { // this creates a new array - super(file, 0, thisOHBytes, thisOHHandles, columns, intprops, columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail); + super(file, 0, thisOHBytes, thisOHHandles, rowdef, intprops, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail); for (int i = 0; i < intprops; i++) try { setHandle(i, new Handle(0)); } catch (IOException e) { @@ -124,7 +124,7 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro for (int i = 0; i < size(); i++) { System.out.print("row " + i + ": "); row = get(i); - for (int j = 0; j < columns(); j++) System.out.print(((row.empty(j)) ? "NULL" : row.getColString(j, "UTF-8")) + ", "); + for (int j = 0; j < row.columns(); j++) System.out.print(((row.empty(j)) ? "NULL" : row.getColString(j, "UTF-8")) + ", "); System.out.println(); } System.out.println("EndOfTable"); @@ -144,7 +144,7 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro // create File f = new File(args[1]); if (f.exists()) f.delete(); - kelondroFixedWidthArray fm = new kelondroFixedWidthArray(f, new int[]{Integer.parseInt(args[2])}, 2, true); + kelondroFixedWidthArray fm = new kelondroFixedWidthArray(f, new kelondroRow(new int[]{Integer.parseInt(args[2])}), 2, true); fm.close(); } else if ((args.length == 2) && (args[0].equals("-v"))) { @@ -158,7 +158,7 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro // get kelondroFixedWidthArray fm = new kelondroFixedWidthArray(new File(args[1])); kelondroRow.Entry row = fm.get(Integer.parseInt(args[2])); - for (int j = 0; j < fm.columns(); j++) System.out.print(row.getColString(j, null) + " "); + for (int j = 0; j < fm.row().columns(); j++) System.out.print(row.getColString(j, null) + " "); System.out.println(); fm.close(); } else @@ -186,7 +186,7 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro if ((args.length == 1) && (args[0].equals("-test"))) { File testfile = new File("test.array"); if (testfile.exists()) testfile.delete(); - kelondroFixedWidthArray fm = new kelondroFixedWidthArray(testfile, new int[]{30, 50}, 9, true); + kelondroFixedWidthArray fm = new kelondroFixedWidthArray(testfile, new kelondroRow(new int[]{30, 50}), 9, true); for (int i = 0; i < 100; i++) { fm.set(i, fm.row().newEntry(new byte[][]{("name" + i).getBytes(), ("value" + i).getBytes()})); } diff --git a/source/de/anomic/kelondro/kelondroFlexTable.java b/source/de/anomic/kelondro/kelondroFlexTable.java index b88353f7f..4f52623e1 100644 --- a/source/de/anomic/kelondro/kelondroFlexTable.java +++ b/source/de/anomic/kelondro/kelondroFlexTable.java @@ -61,31 +61,27 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr } */ - public int columnSize(int column) { - return rowdef.width(column); - } - public kelondroRow.Entry get(byte[] key) throws IOException { Integer i = (Integer) this.index.get(key); if (i == null) return null; return super.get(i.intValue()); } - public byte[][] put(byte[][] row) throws IOException { - Integer i = (Integer) this.index.get(row[0]); + public kelondroRow.Entry put(kelondroRow.Entry row) throws IOException { + Integer i = (Integer) this.index.get(row.getColBytes(0)); if (i == null) { - i = new Integer(super.add(super.rowdef.newEntry(row))); - this.index.put(row[0], i); + i = new Integer(super.add(row)); + this.index.put(row.getColBytes(0), i); return null; } else { - return super.set(i.intValue(), super.rowdef.newEntry(row)).getCols(); + return super.set(i.intValue(), row); } } - public byte[][] remove(byte[] key) throws IOException { + public kelondroRow.Entry remove(byte[] key) throws IOException { Integer i = (Integer) this.index.get(key); if (i == null) return null; - byte[][] r = super.get(i.intValue()).getCols(); + kelondroRow.Entry r = super.get(i.intValue()); super.remove(i.intValue()); return r; } diff --git a/source/de/anomic/kelondro/kelondroFlexWidthArray.java b/source/de/anomic/kelondro/kelondroFlexWidthArray.java index 9bae107e8..2b3bbdcba 100644 --- a/source/de/anomic/kelondro/kelondroFlexWidthArray.java +++ b/source/de/anomic/kelondro/kelondroFlexWidthArray.java @@ -81,7 +81,7 @@ public class kelondroFlexWidthArray implements kelondroArray { columns[j - p] = rowdef.width(j); check = check.substring(0, j) + "X" + check.substring(j + 1); } - col[p] = new kelondroFixedWidthArray(new File(tabledir, colfilename(p, q)), columns, 16, true); + col[p] = new kelondroFixedWidthArray(new File(tabledir, colfilename(p, q)), new kelondroRow(columns), 16, true); } } @@ -94,12 +94,13 @@ public class kelondroFlexWidthArray implements kelondroArray { return "col." + f + ".list"; } - public int size() { - return col[0].size(); + + public kelondroRow row() { + return rowdef; } - public int columns() { - return rowdef.columns(); + public int size() { + return col[0].size(); } public kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException { @@ -112,12 +113,12 @@ public class kelondroFlexWidthArray implements kelondroArray { rowentry.bytes(), rowdef.colstart[r], rowdef.colstart[r] - - rowdef.colstart[r + col[r].columns() - 1] + - rowdef.colstart[r + col[r].row().columns() - 1] + rowdef.width(r)); e1 = col[r].set(index, e0); - for (int i = 0; i < col[r].columns(); i++) + for (int i = 0; i < col[r].row().columns(); i++) p.setCol(r + i, e1.getColBytes(i)); - r = r + col[r].columns(); + r = r + col[r].row().columns(); } } return p; @@ -130,9 +131,9 @@ public class kelondroFlexWidthArray implements kelondroArray { synchronized (col) { while (r < rowdef.columns()) { e = col[r].get(index); - for (int i = 0; i < col[r].columns(); i++) + for (int i = 0; i < col[r].row().columns(); i++) p.setCol(r + i, e.getColBytes(i)); - r = r + col[r].columns(); + r = r + col[r].row().columns(); } } return p; @@ -144,17 +145,17 @@ public class kelondroFlexWidthArray implements kelondroArray { synchronized (col) { e = col[0].row().newEntry(rowentry.bytes(), 0, rowdef.width(0)); index = col[0].add(e); - int r = col[0].columns(); + int r = col[0].row().columns(); while (r < rowdef.columns()) { e = col[r].row().newEntry( rowentry.bytes(), rowdef.colstart[r], - rowdef.colstart[r + col[r].columns() - 1] - + rowdef.width(r + col[r].columns() - 1) + rowdef.colstart[r + col[r].row().columns() - 1] + + rowdef.width(r + col[r].row().columns() - 1) - rowdef.colstart[r]); col[r].set(index, e); - r = r + col[r].columns(); + r = r + col[r].row().columns(); } } return index; @@ -165,7 +166,7 @@ public class kelondroFlexWidthArray implements kelondroArray { synchronized (col) { while (r < rowdef.columns()) { col[r].remove(index); - r = r + col[r].columns(); + r = r + col[r].row().columns(); } } } @@ -176,7 +177,7 @@ public class kelondroFlexWidthArray implements kelondroArray { for (int i = 0; i < size(); i++) { System.out.print("row " + i + ": "); row = get(i); - for (int j = 0; j < columns(); j++) System.out.print(((row.empty(j)) ? "NULL" : row.getColString(j, "UTF-8")) + ", "); + for (int j = 0; j < row().columns(); j++) System.out.print(((row.empty(j)) ? "NULL" : row.getColString(j, "UTF-8")) + ", "); System.out.println(); } System.out.println("EndOfTable"); diff --git a/source/de/anomic/kelondro/kelondroHashtable.java b/source/de/anomic/kelondro/kelondroHashtable.java index aedca9a0e..b46484ccf 100644 --- a/source/de/anomic/kelondro/kelondroHashtable.java +++ b/source/de/anomic/kelondro/kelondroHashtable.java @@ -142,7 +142,7 @@ public class kelondroHashtable { private static final byte[] dummyKey = kelondroBase64Order.enhancedCoder.encodeLong(0, 5).getBytes(); - public kelondroHashtable(File file, int[] columns, int offset, int maxsize, int maxrehash, boolean exitOnFail) { + public kelondroHashtable(File file, kelondroRow rowdef, int offset, int maxsize, int maxrehash, boolean exitOnFail) { // this creates a new hashtable // the key element is not part of the columns array // this is unlike the kelondroTree, where the key is part of a row @@ -152,14 +152,14 @@ public class kelondroHashtable { // this number is needed to omit grow of the table in case of re-hashing // the maxsize is re-computed to a virtual folding height and will result in a tablesize // less than the given maxsize. The actual maxsize can be retrieved by maxsize() - this.hashArray = new kelondroFixedWidthArray(file, extCol(columns), 6, exitOnFail); + this.hashArray = new kelondroFixedWidthArray(file, extCol(rowdef), 6, exitOnFail); this.offset = offset; this.maxk = kelondroMSetTools.log2a(maxsize); // equal to |log2(maxsize)| + 1 if (this.maxk >= kelondroMSetTools.log2a(maxsize + power2(offset + 1) + 1) - 1) this.maxk--; this.maxrehash = maxrehash; dummyRow = this.hashArray.row().newEntry(); dummyRow.setCol(0, dummyKey); - for (int i = 0; i < hashArray.columns(); i++) + for (int i = 0; i < hashArray.row().columns(); i++) try { hashArray.seti(0, this.offset); hashArray.seti(1, this.maxk); @@ -179,11 +179,11 @@ public class kelondroHashtable { this.maxrehash = hashArray.geti(2); } - private int[] extCol(int[] columns) { - int[] newCol = new int[columns.length + 1]; + private kelondroRow extCol(kelondroRow rowdef) { + int[] newCol = new int[rowdef.columns() + 1]; newCol[0] = 4; - System.arraycopy(columns, 0, newCol, 1, columns.length); - return newCol; + for (int i = 0; i < rowdef.columns(); i++) newCol[i + 1] = rowdef.width(i); + return new kelondroRow(newCol); } public static int power2(int x) { @@ -219,7 +219,7 @@ public class kelondroHashtable { // write row kelondroRow.Entry newhkrow = hashArray.row().newEntry(); - newhkrow.setCol(0, hash.key()); + newhkrow.setColLongB256(0, hash.key()); newhkrow.setCol(1, rowentry.bytes()); hashArray.set(rowNumber, newhkrow); return hashArray.row().newEntry(oldhkrow.getColBytes(1)); diff --git a/source/de/anomic/kelondro/kelondroIndex.java b/source/de/anomic/kelondro/kelondroIndex.java index 16f603435..fbfbac386 100644 --- a/source/de/anomic/kelondro/kelondroIndex.java +++ b/source/de/anomic/kelondro/kelondroIndex.java @@ -54,12 +54,11 @@ import java.io.IOException; public interface kelondroIndex { - public int columns(); - public int columnSize(int column); + public kelondroRow row(); public kelondroRow.Entry get(byte[] key) throws IOException; - public byte[][] put(byte[][] row) throws IOException; - public byte[][] remove(byte[] key) throws IOException; + public kelondroRow.Entry put(kelondroRow.Entry row) throws IOException; + public kelondroRow.Entry remove(byte[] key) throws IOException; //public Iterator rows(boolean up, boolean rotating, byte[] startKey) throws IOException; // Objects are of type byte[][] //public Iterator keys(boolean up, boolean rotating, byte[] startKey) throws IOException; // Objects are of type String //public TreeMap rowMap(boolean up, boolean rotating, byte[] firstKey, boolean including, int count) throws IOException; diff --git a/source/de/anomic/kelondro/kelondroMap.java b/source/de/anomic/kelondro/kelondroMap.java index 44417f564..a4a84f972 100644 --- a/source/de/anomic/kelondro/kelondroMap.java +++ b/source/de/anomic/kelondro/kelondroMap.java @@ -130,7 +130,7 @@ public class kelondroMap { } public int keySize() { - return dyn.columnSize(0); + return dyn.row().width(0); } public int[] cacheNodeChunkSize() { diff --git a/source/de/anomic/kelondro/kelondroMapTable.java b/source/de/anomic/kelondro/kelondroMapTable.java index be00a9cb4..54979a9c7 100644 --- a/source/de/anomic/kelondro/kelondroMapTable.java +++ b/source/de/anomic/kelondro/kelondroMapTable.java @@ -87,7 +87,7 @@ public class kelondroMapTable { mTables.put(tablename, map); } - public void declareTree(String tablename, int[] columns, long buffersize /*bytes*/, boolean exitOnFail) { + public void declareTree(String tablename, kelondroRow rowdef, long buffersize /*bytes*/, boolean exitOnFail) { if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared already in other context."); if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared twice."); File tablefile = new File(tablesPath, "table." + tablename + ".tdb"); @@ -96,10 +96,10 @@ public class kelondroMapTable { Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent); } catch (IOException e) { tablefile.getParentFile().mkdirs(); - Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent, columns, exitOnFail); + Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent, rowdef, exitOnFail); } else { tablefile.getParentFile().mkdirs(); - Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent, columns, exitOnFail); + Tree = new kelondroTree(tablefile, buffersize, kelondroTree.defaultObjectCachePercent, rowdef, exitOnFail); } tTables.put(tablename, Tree); } @@ -112,7 +112,7 @@ public class kelondroMapTable { mTables.put(tablename, table); } - public synchronized void update(String tablename, byte[][] row /* first element is the unique key = index */) throws IOException { + public synchronized void update(String tablename, kelondroRow.Entry row /* first element is the unique key = index */) throws IOException { kelondroTree tree = (kelondroTree) tTables.get(tablename); if (tree == null) throw new RuntimeException("kelondroTables.update: tree table '" + tablename + "' does not exist."); tree.put(row); diff --git a/source/de/anomic/kelondro/kelondroNaturalOrder.java b/source/de/anomic/kelondro/kelondroNaturalOrder.java index a6437ad72..7af4f7962 100644 --- a/source/de/anomic/kelondro/kelondroNaturalOrder.java +++ b/source/de/anomic/kelondro/kelondroNaturalOrder.java @@ -56,25 +56,25 @@ public class kelondroNaturalOrder extends kelondroAbstractOrder implements kelon this.zero = null; } - public Object clone() { + public final Object clone() { kelondroNaturalOrder o = new kelondroNaturalOrder(this.asc); o.rotate(this.zero); return o; } - public static kelondroOrder bySignature(String signature) { + public final static kelondroOrder bySignature(String signature) { if (signature.equals("nd")) return new kelondroNaturalOrder(false); if (signature.equals("nu")) return new kelondroNaturalOrder(true); return null; } - public String signature() { + public final String signature() { if (!asc) return "nd"; if ( asc) return "nu"; return null; } - private static long cardinalI(byte[] key) { + private final static long cardinalI(byte[] key) { // returns a cardinal number in the range of 0 .. Long.MAX_VALUE long c = 0; int p = 0; @@ -84,7 +84,7 @@ public class kelondroNaturalOrder extends kelondroAbstractOrder implements kelon return c; } - public long cardinal(byte[] key) { + public final long cardinal(byte[] key) { if (this.zero == null) return cardinalI(key); long zeroCardinal = cardinalI(this.zero); long keyCardinal = cardinalI(key); @@ -131,11 +131,11 @@ public class kelondroNaturalOrder extends kelondroAbstractOrder implements kelon // is less than, equal to, or greater than the second. // two arrays are also equal if one array is a subset of the other's array // with filled-up char(0)-values - public int compare(byte[] a, byte[] b) { + public final int compare(byte[] a, byte[] b) { return (asc) ? compare0(a, b) : compare0(b, a); } - public int compare0(byte[] a, byte[] b) { + public final int compare0(byte[] a, byte[] b) { if (zero == null) return compares(a, b); // we have an artificial start point. check all combinations int az = compares(a, zero); // -1 if a < z; 0 if a == z; 1 if a > z diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java index 2d5238e46..6fdd4c25c 100644 --- a/source/de/anomic/kelondro/kelondroRecords.java +++ b/source/de/anomic/kelondro/kelondroRecords.java @@ -193,7 +193,7 @@ public class kelondroRecords { public kelondroRecords(File file, long buffersize /* bytes */, short ohbytec, short ohhandlec, - int[] columns, int FHandles, int txtProps, int txtPropWidth, + kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth, boolean exitOnFail) { // creates a new file // file: the file that shall be created @@ -208,7 +208,7 @@ public class kelondroRecords { kelondroRA raf = new kelondroFileRA(this.filename); // kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100); // kelondroRA raf = new kelondroNIOFileRA(this.filename, false, 10000); - init(raf, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth, buffersize / 10); + init(raf, ohbytec, ohhandlec, rowdef, FHandles, txtProps, txtPropWidth, buffersize / 10); } catch (IOException e) { logFailure("cannot create / " + e.getMessage()); if (exitOnFail) @@ -219,11 +219,11 @@ public class kelondroRecords { public kelondroRecords(kelondroRA ra, long buffersize /* bytes */, short ohbytec, short ohhandlec, - int[] columns, int FHandles, int txtProps, int txtPropWidth, + kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth, boolean exitOnFail) { this.filename = null; try { - init(ra, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth, buffersize / 10); + init(ra, ohbytec, ohhandlec, rowdef, FHandles, txtProps, txtPropWidth, buffersize / 10); } catch (IOException e) { logFailure("cannot create / " + e.getMessage()); if (exitOnFail) System.exit(-1); @@ -232,7 +232,7 @@ public class kelondroRecords { } private void init(kelondroRA ra, short ohbytec, short ohhandlec, - int[] columns, int FHandles, int txtProps, int txtPropWidth, long writeBufferSize) throws IOException { + kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth, long writeBufferSize) throws IOException { // create new Chunked IO if (useWriteBuffer) { @@ -242,16 +242,16 @@ public class kelondroRecords { } // create row - ROW = new kelondroRow(columns); + ROW = rowdef; // store dynamic run-time data this.overhead = ohbytec + 4 * ohhandlec; - this.recordsize = this.overhead + ROW.size(); - this.headchunksize = overhead + columns[0]; + this.recordsize = this.overhead + ROW.objectsize(); + this.headchunksize = overhead + ROW.width(0); this.tailchunksize = this.recordsize - this.headchunksize; // store dynamic run-time seek pointers - POS_HANDLES = POS_COLWIDTHS + columns.length * 4; + POS_HANDLES = POS_COLWIDTHS + ROW.columns() * 4; POS_TXTPROPS = POS_HANDLES + FHandles * 4; POS_NODES = POS_TXTPROPS + txtProps * txtPropWidth; @@ -410,7 +410,7 @@ public class kelondroRecords { // assign remaining values that are only present at run-time this.overhead = OHBYTEC + 4 * OHHANDLEC; this.recordsize = this.overhead; - this.recordsize = this.overhead + ROW.size(); + this.recordsize = this.overhead + ROW.objectsize(); this.headchunksize = this.overhead + this.ROW.width(0); this.tailchunksize = this.recordsize - this.headchunksize; } @@ -751,13 +751,13 @@ public class kelondroRecords { */ public byte[] setValueRow(byte[] row) throws IOException { // if the index is defined, then write values directly to the file, else only to the object - assert row.length == ROW.size(); + assert row.length == ROW.objectsize(); byte[] result = getValueRow(); // previous value (this loads the values if not already happened) // set values if (this.handle.index != NUL) { setValue(row, 0, ROW.width(0), headChunk, overhead); - if (ROW.columns() > 0) setValue(row, ROW.width(0), ROW.size() - ROW.width(0), tailChunk, 0); + if (ROW.columns() > 1) setValue(row, ROW.width(0), ROW.objectsize() - ROW.width(0), tailChunk, 0); } this.headChanged = true; this.tailChanged = true; @@ -805,7 +805,7 @@ public class kelondroRecords { } // create return value - byte[] row = new byte[ROW.size()]; + byte[] row = new byte[ROW.objectsize()]; // read key System.arraycopy(headChunk, overhead, row, 0, ROW.width(0)); @@ -1051,15 +1051,6 @@ public class kelondroRecords { public final kelondroRow row() { return this.ROW; } - - public final int columns() { - return this.ROW.columns(); - } - - public final int columnSize(int column) { - if ((column < 0) || (column >= this.ROW.columns())) return -1; - return ROW.width(column); - } private final long seekpos(Handle handle) { assert (handle.index >= 0): "handle index too low: " + handle.index; @@ -1263,8 +1254,8 @@ public class kelondroRecords { System.out.println(" Data Offset: 0x" + Long.toHexString(POS_NODES)); System.out.println("--"); System.out.println("RECORDS"); - System.out.print(" Columns : " + columns() + " columns {" + ROW.width(0)); - for (int i = 1; i < columns(); i++) System.out.print(", " + ROW.width(i)); + System.out.print(" Columns : " + row().columns() + " columns {" + ROW.width(0)); + for (int i = 1; i < row().columns(); i++) System.out.print(", " + ROW.width(i)); System.out.println("}"); System.out.println(" Overhead : " + this.overhead + " bytes (" + OHBYTEC + " OH bytes, " + OHHANDLEC + " OH Handles)"); System.out.println(" Recordsize : " + this.recordsize + " bytes"); diff --git a/source/de/anomic/kelondro/kelondroRow.java b/source/de/anomic/kelondro/kelondroRow.java index b7b9ba187..fc27990b3 100644 --- a/source/de/anomic/kelondro/kelondroRow.java +++ b/source/de/anomic/kelondro/kelondroRow.java @@ -73,7 +73,7 @@ public class kelondroRow { return this.row.length; } - public int size() { + public int objectsize() { return this.objectsize; } @@ -164,6 +164,10 @@ public class kelondroRow { return rowinstance; } + public int columns() { + return row.length; + } + public boolean empty(int column) { return rowinstance[colstart[column]] == 0; } @@ -181,10 +185,15 @@ public class kelondroRow { } } - public void setCol(int column, long cell) { + public void setColLongB256(int column, long cell) { kelondroNaturalOrder.encodeLong(cell, rowinstance, colstart[column], row[column].cellwidth()); } + public void setColLongB64E(int column, long cell) { + kelondroBase64Order.enhancedCoder.encodeLong(cell, rowinstance, colstart[column], row[column].cellwidth()); + } + + /* public byte[][] getCols() { byte[][] values = new byte[row.length][]; @@ -203,7 +212,7 @@ public class kelondroRow { return values; } - + */ public String getColString(int column, String encoding) { int length = row[column].cellwidth(); int offset = colstart[column]; diff --git a/source/de/anomic/kelondro/kelondroSplittedTree.java b/source/de/anomic/kelondro/kelondroSplittedTree.java index 06f8f3c72..7f38ca09b 100644 --- a/source/de/anomic/kelondro/kelondroSplittedTree.java +++ b/source/de/anomic/kelondro/kelondroSplittedTree.java @@ -75,25 +75,25 @@ public class kelondroSplittedTree implements kelondroIndex { public kelondroSplittedTree(File pathToFiles, String filenameStub, kelondroOrder objectOrder, long buffersize, int forkfactor, - int[] columns, + kelondroRow rowdef, int txtProps, int txtPropsWidth, boolean exitOnFail) { ktfs = new kelondroTree[forkfactor]; File f; for (int i = 0; i < forkfactor; i++) { - f = dbFile(pathToFiles, filenameStub, forkfactor, columns.length, i); + f = dbFile(pathToFiles, filenameStub, forkfactor, rowdef.columns(), i); if (f.exists()) { try { ktfs[i] = new kelondroTree(f, buffersize/forkfactor, kelondroTree.defaultObjectCachePercent); this.order = ktfs[i].order(); } catch (IOException e) { ktfs[i] = new kelondroTree(f, buffersize/forkfactor, kelondroTree.defaultObjectCachePercent, - columns, objectOrder, txtProps, txtPropsWidth, exitOnFail); + rowdef, objectOrder, txtProps, txtPropsWidth, exitOnFail); this.order = objectOrder; } } else { ktfs[i] = new kelondroTree(f, buffersize/forkfactor, kelondroTree.defaultObjectCachePercent, - columns, objectOrder, txtProps, txtPropsWidth, exitOnFail); + rowdef, objectOrder, txtProps, txtPropsWidth, exitOnFail); this.order = objectOrder; } } @@ -113,27 +113,23 @@ public class kelondroSplittedTree implements kelondroIndex { public static kelondroSplittedTree open(File pathToFiles, String filenameStub, kelondroOrder objectOrder, long buffersize, int forkfactor, - int[] columns, int txtProps, int txtPropsWidth, + kelondroRow rowdef, int txtProps, int txtPropsWidth, boolean exitOnFail) throws IOException { // generated a new splittet tree if it not exists or // opens an existing one - if (existsAll(pathToFiles, filenameStub, forkfactor, columns.length)) { - return new kelondroSplittedTree(pathToFiles, filenameStub, objectOrder, buffersize, forkfactor, columns.length); + if (existsAll(pathToFiles, filenameStub, forkfactor, rowdef.columns())) { + return new kelondroSplittedTree(pathToFiles, filenameStub, objectOrder, buffersize, forkfactor, rowdef.columns()); } else { return new kelondroSplittedTree(pathToFiles, filenameStub, objectOrder, buffersize, forkfactor, - columns, txtProps, txtPropsWidth, + rowdef, txtProps, txtPropsWidth, exitOnFail); } } - public int columns() { - return ktfs[0].columns(); - } - - public int columnSize(int column) { - return ktfs[0].columnSize(column); + public kelondroRow row() { + return ktfs[0].row(); } private int partition(byte[] key) { @@ -145,11 +141,11 @@ public class kelondroSplittedTree implements kelondroIndex { return ktfs[partition(key)].get(key); } - public byte[][] put(byte[][] row) throws IOException { - return ktfs[partition(row[0])].put(row); + public kelondroRow.Entry put(kelondroRow.Entry row) throws IOException { + return ktfs[partition(row.getColBytes(0))].put(row); } - public byte[][] remove(byte[] key) throws IOException { + public kelondroRow.Entry remove(byte[] key) throws IOException { return ktfs[partition(key)].remove(key); } diff --git a/source/de/anomic/kelondro/kelondroStack.java b/source/de/anomic/kelondro/kelondroStack.java index 97bd25fea..174c88662 100644 --- a/source/de/anomic/kelondro/kelondroStack.java +++ b/source/de/anomic/kelondro/kelondroStack.java @@ -68,12 +68,12 @@ public final class kelondroStack extends kelondroRecords { private static int toor = 1; // pointer for FHandles-array: pointer to root node public kelondroStack(File file, long buffersize, int key, int value, boolean exitOnFail) { - this(file, buffersize, new int[] { key, value }, exitOnFail); + this(file, buffersize, new kelondroRow(new int[] { key, value }), exitOnFail); } - public kelondroStack(File file, long buffersize, int[] columns, boolean exitOnFail) { + public kelondroStack(File file, long buffersize, kelondroRow rowdef, boolean exitOnFail) { // this creates a new stack - super(file, buffersize, thisOHBytes, thisOHHandles, columns, thisFHandles, columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail); + super(file, buffersize, thisOHBytes, thisOHHandles, rowdef, thisFHandles, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail); try { setHandle(root, null); // define the root value setHandle(toor, null); // define the toor value @@ -107,7 +107,7 @@ public final class kelondroStack extends kelondroRecords { if (f.exists()) f.delete(); // re-open a database with same settings as before - return new kelondroStack(f, bz, row.widths(), true); + return new kelondroStack(f, bz, row, true); } public class Counter implements Iterator { @@ -307,10 +307,10 @@ public final class kelondroStack extends kelondroRecords { st = new StringTokenizer(s, separator); // buffer the entry c = 0; - while ((c < columns()) && (st.hasMoreTokens())) { + while ((c < row().columns()) && (st.hasMoreTokens())) { buffer.setCol(c++, st.nextToken().trim().getBytes()); } - if ((st.hasMoreTokens()) || (c != columns())) { + if ((st.hasMoreTokens()) || (c != row().columns())) { System.err.println("inapropriate number of entries in line " + line); } else { push(buffer); @@ -346,7 +346,7 @@ public final class kelondroStack extends kelondroRecords { + hp(n.getOHHandle(left)) + ", right " + hp(n.getOHHandle(right))); System.out.print(" KEY:'" + r.getColString(0, null) + "'"); - for (int j = 1; j < columns(); j++) + for (int j = 1; j < row().columns(); j++) System.out.print(", V[" + j + "]:'" + r.getColString(j, null) + "'"); System.out.println(); } @@ -414,9 +414,7 @@ public final class kelondroStack extends kelondroRecords { // create File f = new File(args[3]); if (f.exists()) f.delete(); - int[] lens = new int[2]; - lens[0] = Integer.parseInt(args[1]); - lens[1] = Integer.parseInt(args[2]); + kelondroRow lens = new kelondroRow(new int[]{Integer.parseInt(args[1]), Integer.parseInt(args[2])}); kelondroStack fm = new kelondroStack(f, 0x100000, lens, true); fm.close(); } else if (args[0].equals("-p")) { diff --git a/source/de/anomic/kelondro/kelondroTree.java b/source/de/anomic/kelondro/kelondroTree.java index 55f236292..5d113f996 100644 --- a/source/de/anomic/kelondro/kelondroTree.java +++ b/source/de/anomic/kelondro/kelondroTree.java @@ -93,19 +93,19 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { public kelondroTree(File file, long buffersize, int objectCachePercent, int key, int value, boolean exitOnFail) { - this(file, buffersize, objectCachePercent, new int[] { key, value }, new kelondroNaturalOrder(true), 1, 8, exitOnFail); + this(file, buffersize, objectCachePercent, new kelondroRow(new int[] { key, value }), new kelondroNaturalOrder(true), 1, 8, exitOnFail); } - public kelondroTree(File file, long buffersize, int objectCachePercent, int[] columns, boolean exitOnFail) { + public kelondroTree(File file, long buffersize, int objectCachePercent, kelondroRow rowdef, boolean exitOnFail) { // this creates a new tree file - this(file, buffersize, objectCachePercent, columns, new kelondroNaturalOrder(true), columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail); + this(file, buffersize, objectCachePercent, rowdef, new kelondroNaturalOrder(true), rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail); } - public kelondroTree(File file, long buffersize, int objectCachePercent, int[] columns, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) { + public kelondroTree(File file, long buffersize, int objectCachePercent, kelondroRow rowdef, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) { // this creates a new tree file super(file, (100 - objectCachePercent) * buffersize / 100, - thisOHBytes, thisOHHandles, columns, + thisOHBytes, thisOHHandles, rowdef, thisFHandles, txtProps, txtPropsWidth, exitOnFail); try { setHandle(root, null); // define the root value @@ -120,16 +120,16 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { initObjectCache(buffersize, objectCachePercent); } - public kelondroTree(kelondroRA ra, long buffersize, int objectCachePercent, int[] columns, boolean exitOnFail) { + public kelondroTree(kelondroRA ra, long buffersize, int objectCachePercent, kelondroRow rowdef, boolean exitOnFail) { // this creates a new tree within a kelondroRA - this(ra, buffersize, objectCachePercent, columns, new kelondroNaturalOrder(true), columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail); + this(ra, buffersize, objectCachePercent, rowdef, new kelondroNaturalOrder(true), rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail); } - public kelondroTree(kelondroRA ra, long buffersize, int objectCachePercent, int[] columns, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) { + public kelondroTree(kelondroRA ra, long buffersize, int objectCachePercent, kelondroRow rowdef, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) { // this creates a new tree within a kelondroRA super(ra, (100 - objectCachePercent) * buffersize / 100, - thisOHBytes, thisOHHandles, columns, + thisOHBytes, thisOHHandles, rowdef, thisFHandles, txtProps, txtPropsWidth, exitOnFail); try { setHandle(root, null); // define the root value @@ -171,7 +171,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { } public final int cacheObjectChunkSize() { - return row().size() + 8 * super.columns(); + return row().objectsize() + /* overhead */ 8 * super.row().columns(); } public String[] cacheObjectStatus() { @@ -394,19 +394,18 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { } // Associates the specified value with the specified key in this map - public byte[][] put(byte[][] newrow) throws IOException { + public kelondroRow.Entry put(kelondroRow.Entry newrow) throws IOException { kelondroRow.Entry result = null; //writeLock.stay(2000, 1000); - if (newrow.length != columns()) throw new IllegalArgumentException("put: wrong row length " + newrow.length + "; must be " + columns()); + if (newrow.columns() != row().columns()) throw new IllegalArgumentException("put: wrong row length " + newrow.columns() + "; must be " + row().columns()); // first try to find the key element in the database synchronized(writeSearchObj) { - kelondroRow.Entry newentry = row().newEntry(newrow); - if (objectCache != null) objectCache.put(newentry.getColBytes(0), newentry); - writeSearchObj.process(newrow[0]); + if (objectCache != null) objectCache.put(newrow.getColBytes(0), newrow); + writeSearchObj.process(newrow.getColBytes(0)); if (writeSearchObj.found()) { // a node with this key exist. simply overwrite the content and return old content Node e = writeSearchObj.getMatcher(); - result = row().newEntry(e.setValueRow(newentry.bytes())); + result = row().newEntry(e.setValueRow(newrow.bytes())); commitNode(e); } else if (writeSearchObj.isRoot()) { // a node with this key does not exist and there is no node at all @@ -415,7 +414,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { throw new kelondroException(filename, "tried to create root node twice"); // we dont have any Nodes in the file, so start here to create one Node e = newNode(); - e.setValueRow(newentry.bytes()); + e.setValueRow(newrow.bytes()); // write the propetries e.setOHByte(magic, (byte) 1); e.setOHByte(balance, (byte) 0); @@ -437,7 +436,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { // create new node and assign values Node parentNode = writeSearchObj.getParent(); Node theNode = newNode(); - theNode.setValueRow(newentry.bytes()); + theNode.setValueRow(newrow.bytes()); theNode.setOHByte(0, (byte) 1); // fresh magic theNode.setOHByte(1, (byte) 0); // fresh balance theNode.setOHHandle(parent, parentNode.handle()); @@ -537,7 +536,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { } } //writeLock.release(); - return (result == null) ? null : result.getCols(); + return result; } private void assignChild(Node parentNode, Node childNode, int childType) throws IOException { @@ -656,15 +655,13 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { // Associates the specified value with the specified key in this map public byte[] put(byte[] key, byte[] value) throws IOException { - byte[][] row = new byte[2][]; - row[0] = key; - row[1] = value; - byte[][] ret = put(row); - if (ret == null) return null; else return ret[1]; + kelondroRow.Entry row = row().newEntry(new byte[][]{key, value}); + kelondroRow.Entry ret = put(row); + if (ret == null) return null; else return ret.getColBytes(0); } // Removes the mapping for this key from this map if present (optional operation). - public byte[][] remove(byte[] key) throws IOException { + public kelondroRow.Entry remove(byte[] key) throws IOException { synchronized(writeSearchObj) { if (objectCache != null) objectCache.remove(key); writeSearchObj.process(key); @@ -672,7 +669,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { Node result = writeSearchObj.getMatcher(); kelondroRow.Entry values = row().newEntry(result.getValueRow()); remove(result, writeSearchObj.getParent()); - return values.getCols(); + return values; } else { return null; } @@ -1029,7 +1026,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { Iterator i = (firstKey == null) ? new nodeIterator(up, rotating) : new nodeIterator(up, rotating, firstKey, including); while ((rows.size() < count) && (i.hasNext())) { n = (Node) i.next(); - if (n != null) rows.put(new String(n.getKey()), row().newEntry(n.getValueRow()).getCols()); + if (n != null) rows.put(new String(n.getKey()), row().newEntry(n.getValueRow())); } } return rows; @@ -1196,7 +1193,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { String s; StringTokenizer st; int recs = 0; - byte[][] buffer = new byte[columns()][]; + kelondroRow.Entry buffer = row().newEntry(); int c; int line = 0; while ((s = f.readLine()) != null) { @@ -1206,10 +1203,10 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { st = new StringTokenizer(s, separator); // buffer the entry c = 0; - while ((c < columns()) && (st.hasMoreTokens())) { - buffer[c++] = st.nextToken().trim().getBytes(); + while ((c < row().columns()) && (st.hasMoreTokens())) { + buffer.setCol(c++, st.nextToken().trim().getBytes()); } - if ((st.hasMoreTokens()) || (c != columns())) { + if ((st.hasMoreTokens()) || (c != row().columns())) { System.err.println("inapropriate number of entries in line " + line); } else { put(buffer); @@ -1253,7 +1250,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { Vector nextline; Handle handle; Node node; - int linelength, width = (1 << (height - 1)) * (columnSize(0) + 1); + int linelength, width = (1 << (height - 1)) * (row().width(0) + 1); String key; for (int h = 1; h < height; h++) { linelength = width / (thisline.size() * 2); @@ -1399,9 +1396,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { // create File f = new File(args[3]); if (f.exists()) f.delete(); - int[] lens = new int[2]; - lens[0] = Integer.parseInt(args[1]); - lens[1] = Integer.parseInt(args[2]); + kelondroRow lens = new kelondroRow(new int[]{Integer.parseInt(args[1]), Integer.parseInt(args[2])}); kelondroTree fm = new kelondroTree(f, 0x100000, 10, lens, true); fm.close(); } else if (args[0].equals("-u")) { diff --git a/source/de/anomic/plasma/plasmaCrawlBalancer.java b/source/de/anomic/plasma/plasmaCrawlBalancer.java index 8b539b4a6..b2a96f1f1 100644 --- a/source/de/anomic/plasma/plasmaCrawlBalancer.java +++ b/source/de/anomic/plasma/plasmaCrawlBalancer.java @@ -50,6 +50,7 @@ import java.util.ArrayList; import de.anomic.index.indexURL; import de.anomic.kelondro.kelondroRecords; +import de.anomic.kelondro.kelondroRow; import de.anomic.kelondro.kelondroStack; public class plasmaCrawlBalancer { @@ -62,10 +63,10 @@ public class plasmaCrawlBalancer { try { stack = new kelondroStack(stackFile, buffersize); } catch (IOException e) { - stack = new kelondroStack(stackFile, buffersize, new int[] {indexURL.urlHashLength}, true); + stack = new kelondroStack(stackFile, buffersize, new kelondroRow(new int[] {indexURL.urlHashLength}), true); } } else { - stack = new kelondroStack(stackFile, buffersize, new int[] {indexURL.urlHashLength}, true); + stack = new kelondroStack(stackFile, buffersize, new kelondroRow(new int[] {indexURL.urlHashLength}), true); } domainStacks = new HashMap(); } diff --git a/source/de/anomic/plasma/plasmaCrawlEURL.java b/source/de/anomic/plasma/plasmaCrawlEURL.java index 87ff9f729..ab9333383 100644 --- a/source/de/anomic/plasma/plasmaCrawlEURL.java +++ b/source/de/anomic/plasma/plasmaCrawlEURL.java @@ -73,8 +73,8 @@ public class plasmaCrawlEURL extends indexURL { urlNameLength, // the name of the url, from anchor tag name urlDateLength, // the time when the url was first time appeared urlDateLength, // the time when the url was last time tried to load - urlRetryLength, // number of load retries - urlErrorLength, // string describing load failure + urlRetryLength, // number of load retries + urlErrorLength, // string describing load failure urlFlagLength // extra space }; if (cachePath.exists()) try { @@ -82,11 +82,11 @@ public class plasmaCrawlEURL extends indexURL { urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent); } catch (IOException e) { cachePath.delete(); - urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true); + urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true); } else { // create new cache cachePath.getParentFile().mkdirs(); - urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true); + urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true); } } @@ -216,7 +216,7 @@ public class plasmaCrawlEURL extends indexURL { this.failreason.getBytes(), this.flags.getBytes() }; - urlHashCache.put(entry); + urlHashCache.put(urlHashCache.row().newEntry(entry)); } catch (IOException e) { System.out.println("INTERNAL ERROR AT plasmaEURL:url2hash:" + e.toString()); } diff --git a/source/de/anomic/plasma/plasmaCrawlLURL.java b/source/de/anomic/plasma/plasmaCrawlLURL.java index 92c09ddef..db28cfcc2 100644 --- a/source/de/anomic/plasma/plasmaCrawlLURL.java +++ b/source/de/anomic/plasma/plasmaCrawlLURL.java @@ -119,12 +119,12 @@ public final class plasmaCrawlLURL extends indexURL { urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent); } catch (IOException e) { cachePath.getParentFile().mkdirs(); - urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true); + urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true); } } else { // create new cache cachePath.getParentFile().mkdirs(); - urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true); + urlHashCache = new kelondroTree(cachePath, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true); } // init result stacks @@ -575,7 +575,7 @@ public final class plasmaCrawlLURL extends indexURL { kelondroBase64Order.enhancedCoder.encodeLong(size, urlSizeLength).getBytes(), kelondroBase64Order.enhancedCoder.encodeLong(wordCount, urlWordCountLength).getBytes(), }; - urlHashCache.put(entry); + urlHashCache.put(urlHashCache.row().newEntry(entry)); serverLog.logFine("PLASMA","STORED new LURL " + url.toString()); this.stored = true; } catch (Exception e) { @@ -846,7 +846,7 @@ public final class plasmaCrawlLURL extends indexURL { if (res.statusCode == 200) { entry.setCol(1, newUrl.toString().getBytes()); - urlHashCache.put(entry.getCols()); + urlHashCache.put(entry); log.logInfo("UrlDB-Entry with urlHash '" + urlHash + "' corrected\n\tURL: " + oldUrlStr + " -> " + newUrlStr); } else { remove(urlHash); diff --git a/source/de/anomic/plasma/plasmaCrawlNURL.java b/source/de/anomic/plasma/plasmaCrawlNURL.java index 1b389bb1c..581b28d77 100644 --- a/source/de/anomic/plasma/plasmaCrawlNURL.java +++ b/source/de/anomic/plasma/plasmaCrawlNURL.java @@ -124,26 +124,27 @@ public class plasmaCrawlNURL extends indexURL { limitStack = new plasmaCrawlBalancer(limitStackFile, 0); overhangStack = new plasmaCrawlBalancer(overhangStackFile, 0); remoteStack = new plasmaCrawlBalancer(remoteStackFile, 0); + kelondroRow rowdef = new kelondroRow(new int[] {indexURL.urlHashLength}); if (imageStackFile.exists()) try { imageStack = new kelondroStack(imageStackFile, 0); } catch (IOException e) { - imageStack = new kelondroStack(imageStackFile, 0, new int[] {indexURL.urlHashLength}, true); + imageStack = new kelondroStack(imageStackFile, 0, rowdef, true); } else { - imageStack = new kelondroStack(imageStackFile, 0, new int[] {indexURL.urlHashLength}, true); + imageStack = new kelondroStack(imageStackFile, 0, rowdef, true); } if (movieStackFile.exists()) try { movieStack = new kelondroStack(movieStackFile, 0); } catch (IOException e) { - movieStack = new kelondroStack(movieStackFile, 0, new int[] {indexURL.urlHashLength}, true); + movieStack = new kelondroStack(movieStackFile, 0, rowdef, true); } else { - movieStack = new kelondroStack(movieStackFile, 0, new int[] {indexURL.urlHashLength}, true); + movieStack = new kelondroStack(movieStackFile, 0, rowdef, true); } if (musicStackFile.exists()) try { musicStack = new kelondroStack(musicStackFile, 0); } catch (IOException e) { - musicStack = new kelondroStack(musicStackFile, 0, new int[] {indexURL.urlHashLength}, true); + musicStack = new kelondroStack(musicStackFile, 0, rowdef, true); } else { - musicStack = new kelondroStack(musicStackFile, 0, new int[] {indexURL.urlHashLength}, true); + musicStack = new kelondroStack(musicStackFile, 0, rowdef, true); } // init stack Index @@ -168,11 +169,11 @@ public class plasmaCrawlNURL extends indexURL { urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent); } catch (IOException e) { cacheFile.delete(); - urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true); + urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true); } else { // create new cache cacheFile.getParentFile().mkdirs(); - urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, ce, true); + urlHashCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(ce), true); } } @@ -553,7 +554,7 @@ public class plasmaCrawlNURL extends indexURL { this.flags.getBytes(), normalizeHandle(this.handle).getBytes() }; - urlHashCache.put(entry); + urlHashCache.put(urlHashCache.row().newEntry(entry)); } catch (IOException e) { serverLog.logSevere("PLASMA", "INTERNAL ERROR AT plasmaNURL:store:" + e.toString() + ", resetting NURL-DB"); e.printStackTrace(); diff --git a/source/de/anomic/plasma/plasmaCrawlStacker.java b/source/de/anomic/plasma/plasmaCrawlStacker.java index e13518fd6..cba2b2d20 100644 --- a/source/de/anomic/plasma/plasmaCrawlStacker.java +++ b/source/de/anomic/plasma/plasmaCrawlStacker.java @@ -62,6 +62,7 @@ import de.anomic.http.httpc; import de.anomic.index.indexURL; import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroException; +import de.anomic.kelondro.kelondroRow; import de.anomic.kelondro.kelondroTree; import de.anomic.server.serverSemaphore; import de.anomic.server.logging.serverLog; @@ -452,23 +453,23 @@ public final class plasmaCrawlStacker { } } - public stackCrawlMessage(String urlHash, byte[][] entryBytes) { + public stackCrawlMessage(String urlHash, kelondroRow.Entry entry) { if (urlHash == null) throw new NullPointerException(); - if (entryBytes == null) throw new NullPointerException(); + if (entry == null) throw new NullPointerException(); try { this.urlHash = urlHash; - this.initiator = new String(entryBytes[1], "UTF-8"); - this.url = new String(entryBytes[2], "UTF-8").trim(); - this.referrerHash = (entryBytes[3]==null) ? indexURL.dummyHash : new String(entryBytes[3], "UTF-8"); - this.name = (entryBytes[4] == null) ? "" : new String(entryBytes[4], "UTF-8").trim(); - this.loaddate = new Date(86400000 * kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[5], "UTF-8"))); - this.profileHandle = (entryBytes[6] == null) ? null : new String(entryBytes[6], "UTF-8").trim(); - this.depth = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[7], "UTF-8")); - this.anchors = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[8], "UTF-8")); - this.forkfactor = (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(entryBytes[9], "UTF-8")); - this.flags = new bitfield(entryBytes[10]); - this.handle = Integer.parseInt(new String(entryBytes[11], "UTF-8")); + this.initiator = entry.getColString(1, "UTF-8"); + this.url = entry.getColString(2, "UTF-8").trim(); + this.referrerHash = (entry.empty(3)) ? indexURL.dummyHash : entry.getColString(3, "UTF-8"); + this.name = (entry.empty(4)) ? "" : entry.getColString(4, "UTF-8").trim(); + this.loaddate = new Date(86400000 * entry.getColLongB64E(5)); + this.profileHandle = (entry.empty(6)) ? null : entry.getColString(6, "UTF-8").trim(); + this.depth = (int) entry.getColLongB64E(7); + this.anchors = (int) entry.getColLongB64E(8); + this.forkfactor = (int) entry.getColLongB64E(9); + this.flags = new bitfield(entry.getColBytes(10)); + this.handle = Integer.parseInt(new String(entry.getColBytes(11), "UTF-8")); } catch (Exception e) { e.printStackTrace(); throw new IllegalStateException(e.toString()); @@ -578,7 +579,7 @@ public final class plasmaCrawlStacker { this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent); } catch (IOException e) { cacheFile.delete(); - this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.ce, true); + this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(plasmaCrawlNURL.ce), true); } try { // loop through the list and fill the messageList with url hashs @@ -600,7 +601,7 @@ public final class plasmaCrawlStacker { // deleting old db and creating a new db try {this.urlEntryCache.close();}catch(Exception ex){} cacheFile.delete(); - this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.ce, true); + this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(plasmaCrawlNURL.ce), true); } catch (IOException e) { /* if we have an error, we start with a fresh database */ plasmaCrawlStacker.this.log.logSevere("Unable to initialize crawl stacker queue, IOException:" + e.getMessage() + ". Reseting DB.\n",e); @@ -608,12 +609,12 @@ public final class plasmaCrawlStacker { // deleting old db and creating a new db try {this.urlEntryCache.close();}catch(Exception ex){} cacheFile.delete(); - this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.ce, true); + this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(plasmaCrawlNURL.ce), true); } } else { // create new cache cacheFile.getParentFile().mkdirs(); - this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, plasmaCrawlNURL.ce, true); + this.urlEntryCache = new kelondroTree(cacheFile, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new kelondroRow(plasmaCrawlNURL.ce), true); } } @@ -634,7 +635,7 @@ public final class plasmaCrawlStacker { boolean insertionDoneSuccessfully = false; synchronized(this.urlEntryHashCache) { - byte[][] oldValue = this.urlEntryCache.put(newMessage.getBytes()); + kelondroRow.Entry oldValue = this.urlEntryCache.put(this.urlEntryCache.row().newEntry(newMessage.getBytes())); if (oldValue == null) { insertionDoneSuccessfully = this.urlEntryHashCache.add(newMessage.urlHash); } @@ -659,7 +660,7 @@ public final class plasmaCrawlStacker { this.writeSync.P(); String urlHash = null; - byte[][] entryBytes = null; + kelondroRow.Entry entryBytes = null; stackCrawlMessage newMessage = null; try { synchronized(this.urlEntryHashCache) { @@ -670,7 +671,7 @@ public final class plasmaCrawlStacker { this.writeSync.V(); } - newMessage = new stackCrawlMessage(urlHash,entryBytes); + newMessage = new stackCrawlMessage(urlHash, entryBytes); return newMessage; } } diff --git a/source/de/anomic/plasma/plasmaSwitchboardQueue.java b/source/de/anomic/plasma/plasmaSwitchboardQueue.java index 036fa1d8d..0497332e2 100644 --- a/source/de/anomic/plasma/plasmaSwitchboardQueue.java +++ b/source/de/anomic/plasma/plasmaSwitchboardQueue.java @@ -80,22 +80,7 @@ public class plasmaSwitchboardQueue { } private void initQueueStack() { - if (sbQueueStackPath.exists()) try { - sbQueueStack = new kelondroStack(sbQueueStackPath, 0); - } catch (IOException e) { - sbQueueStackPath.delete(); - sbQueueStack = new kelondroStack(sbQueueStackPath, 0, new int[] { - indexURL.urlStringLength, - indexURL.urlHashLength, - 11, - 1, - yacySeedDB.commonHashLength, - indexURL.urlCrawlDepthLength, - indexURL.urlCrawlProfileHandleLength, - indexURL.urlDescrLength - }, true); - } else { - sbQueueStack = new kelondroStack(sbQueueStackPath, 0, new int[] { + kelondroRow rowdef = new kelondroRow(new int[] { indexURL.urlStringLength, indexURL.urlHashLength, 11, @@ -104,7 +89,14 @@ public class plasmaSwitchboardQueue { indexURL.urlCrawlDepthLength, indexURL.urlCrawlProfileHandleLength, indexURL.urlDescrLength - }, true); + }); + if (sbQueueStackPath.exists()) try { + sbQueueStack = new kelondroStack(sbQueueStackPath, 0); + } catch (IOException e) { + sbQueueStackPath.delete(); + sbQueueStack = new kelondroStack(sbQueueStackPath, 0, rowdef, true); + } else { + sbQueueStack = new kelondroStack(sbQueueStackPath, 0, rowdef, true); } } diff --git a/source/de/anomic/plasma/plasmaWordConnotation.java b/source/de/anomic/plasma/plasmaWordConnotation.java index ad31f89ff..381cfc9c9 100644 --- a/source/de/anomic/plasma/plasmaWordConnotation.java +++ b/source/de/anomic/plasma/plasmaWordConnotation.java @@ -48,8 +48,8 @@ package de.anomic.plasma; import java.io.File; import java.io.IOException; -import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroDynTree; +import de.anomic.kelondro.kelondroRow; public class plasmaWordConnotation { @@ -62,19 +62,19 @@ public class plasmaWordConnotation { if (refDBfile.exists()) try { refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, fillChar); } catch (IOException e) { - refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, wordlength, nodesize, new int[] {wordlength, countlength}, fillChar, true); + refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, wordlength, nodesize, new kelondroRow(new int[] {wordlength, countlength}), fillChar, true); } else { - refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, wordlength, nodesize, new int[] {wordlength, countlength}, fillChar, true); + refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, wordlength, nodesize, new kelondroRow(new int[] {wordlength, countlength}), fillChar, true); } } private void addSingleRef(String word, String reference) throws IOException { //word = word.toLowerCase(); //reference = reference.toLowerCase(); - byte[][] record = refDB.get(word, reference.getBytes()); + kelondroRow.Entry record = refDB.get(word, reference.getBytes()); long c; - if (record == null) c = 0; else c = kelondroBase64Order.enhancedCoder.decodeLong(new String(record[1], "UTF-8")); - record[1] = kelondroBase64Order.enhancedCoder.encodeLong(c++, countlength).getBytes(); + if (record == null) c = 0; else c = record.getColLongB64E(1); + record.setColLongB64E(1, c++); refDB.put(word, record); } diff --git a/source/de/anomic/plasma/plasmaWordIndexAssortment.java b/source/de/anomic/plasma/plasmaWordIndexAssortment.java index 527156e9f..eddd95af7 100644 --- a/source/de/anomic/plasma/plasmaWordIndexAssortment.java +++ b/source/de/anomic/plasma/plasmaWordIndexAssortment.java @@ -61,7 +61,6 @@ import de.anomic.index.indexEntryAttribute; import de.anomic.index.indexTreeMapContainer; import de.anomic.index.indexURLEntry; import de.anomic.kelondro.kelondroException; -import de.anomic.kelondro.kelondroNaturalOrder; import de.anomic.kelondro.kelondroTree; import de.anomic.kelondro.kelondroRow; import de.anomic.server.logging.serverLog; @@ -84,7 +83,6 @@ public final class plasmaWordIndexAssortment { private serverLog log; private kelondroTree assortments; private long bufferSize; - private int bufferStructureLength; private static String intx(int x) { String s = Integer.toString(x); @@ -93,22 +91,22 @@ public final class plasmaWordIndexAssortment { } private static int[] bufferStructure(int assortmentCapacity) { - int[] structure = new int[3 + 2 * assortmentCapacity]; - structure[0] = bufferStructureBasis[0]; - structure[1] = bufferStructureBasis[1]; - structure[2] = bufferStructureBasis[2]; - for (int i = 0; i < assortmentCapacity; i++) { - structure[3 + 2 * i] = bufferStructureBasis[3]; - structure[4 + 2 * i] = bufferStructureBasis[4]; - } - return structure; + int[] structure = new int[3 + 2 * assortmentCapacity]; + structure[0] = bufferStructureBasis[0]; + structure[1] = bufferStructureBasis[1]; + structure[2] = bufferStructureBasis[2]; + for (int i = 0; i < assortmentCapacity; i++) { + structure[3 + 2 * i] = bufferStructureBasis[3]; + structure[4 + 2 * i] = bufferStructureBasis[4]; + } + return structure; } public plasmaWordIndexAssortment(File storagePath, int assortmentLength, int bufferkb, serverLog log) { - if (!(storagePath.exists())) storagePath.mkdirs(); - this.assortmentFile = new File(storagePath, assortmentFileName + intx(assortmentLength) + ".db"); - this.assortmentLength = assortmentLength; - this.bufferStructureLength = 3 + 2 * assortmentLength; + if (!(storagePath.exists())) storagePath.mkdirs(); + this.assortmentFile = new File(storagePath, assortmentFileName + intx(assortmentLength) + ".db"); + this.assortmentLength = assortmentLength; + //this.bufferStructureLength = 3 + 2 * assortmentLength; this.bufferSize = bufferkb * 1024; this.log = log; if (assortmentFile.exists()) { @@ -125,7 +123,7 @@ public final class plasmaWordIndexAssortment { assortmentFile.delete(); // make space for new one } // create new assortment tree file - assortments = new kelondroTree(assortmentFile, bufferSize, kelondroTree.defaultObjectCachePercent, bufferStructure(assortmentLength), true); + assortments = new kelondroTree(assortmentFile, bufferSize, kelondroTree.defaultObjectCachePercent, new kelondroRow(bufferStructure(assortmentLength)), true); if (log != null) log.logConfig("Created new Assortment Database, width " + assortmentLength + ", " + bufferkb + "kb buffer"); } @@ -134,18 +132,18 @@ public final class plasmaWordIndexAssortment { // this throws an exception if the word hash already existed //log.logDebug("storeAssortment: wordHash=" + wordHash + ", urlHash=" + entry.getUrlHash() + ", time=" + creationTime); if (newContainer.size() != assortmentLength) throw new RuntimeException("plasmaWordIndexAssortment.store: wrong container size"); - byte[][] row = new byte[this.bufferStructureLength][]; - row[0] = newContainer.wordHash().getBytes(); - row[1] = kelondroNaturalOrder.encodeLong(1, 4); - row[2] = kelondroNaturalOrder.encodeLong(newContainer.updated(), 8); + kelondroRow.Entry row = assortments.row().newEntry(); + row.setCol(0, newContainer.wordHash().getBytes()); + row.setColLongB256(1, 1); + row.setColLongB256(2, newContainer.updated()); Iterator entries = newContainer.entries(); indexURLEntry entry; for (int i = 0; i < assortmentLength; i++) { entry = (indexURLEntry) entries.next(); - row[3 + 2 * i] = entry.getUrlHash().getBytes(); - row[4 + 2 * i] = entry.toEncodedStringForm().getBytes(); + row.setCol(3 + 2 * i, entry.getUrlHash().getBytes()); + row.setCol(4 + 2 * i, entry.toEncodedStringForm().getBytes()); } - byte[][] oldrow = null; + kelondroRow.Entry oldrow = null; try { oldrow = assortments.put(row); } catch (IOException e) { @@ -163,7 +161,7 @@ public final class plasmaWordIndexAssortment { // and returns the content record kelondroRow.Entry row = null; try { - row = assortments.row().newEntry(assortments.remove(wordHash.getBytes())); + row = assortments.remove(wordHash.getBytes()); } catch (IOException e) { log.logSevere("removeAssortment/IO-error: " + e.getMessage() + " - reset assortment-DB " + assortments.file(), e); @@ -245,7 +243,7 @@ public final class plasmaWordIndexAssortment { if (!(assortmentFile.delete())) throw new RuntimeException("cannot delete assortment database"); } if (assortmentFile.exists()) assortmentFile.delete(); - assortments = new kelondroTree(assortmentFile, bufferSize, kelondroTree.defaultObjectCachePercent, bufferStructure(assortmentLength), true); + assortments = new kelondroTree(assortmentFile, bufferSize, kelondroTree.defaultObjectCachePercent, new kelondroRow(bufferStructure(assortmentLength)), true); } public Iterator hashes(String startWordHash, boolean up, boolean rot) throws IOException { diff --git a/source/de/anomic/yacy/yacyNewsDB.java b/source/de/anomic/yacy/yacyNewsDB.java index 78c52ec70..04cc0a8af 100644 --- a/source/de/anomic/yacy/yacyNewsDB.java +++ b/source/de/anomic/yacy/yacyNewsDB.java @@ -50,6 +50,7 @@ import java.util.Iterator; import de.anomic.yacy.yacyCore; import de.anomic.kelondro.kelondroBase64Order; +import de.anomic.kelondro.kelondroColumn; import de.anomic.kelondro.kelondroTree; import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroRow; @@ -80,15 +81,17 @@ public class yacyNewsDB { news = createDB(path, bufferkb); } } + + public static final kelondroRow rowdef = new kelondroRow(new kelondroColumn[]{ + new kelondroColumn(kelondroColumn.celltype_string, yacyNewsRecord.idLength(), "newsid", "id = created + originator"), + new kelondroColumn(kelondroColumn.celltype_string, yacyNewsRecord.categoryStringLength, "category", ""), + new kelondroColumn(kelondroColumn.celltype_string, yacyCore.universalDateShortPattern.length(), "received", ""), + new kelondroColumn(kelondroColumn.celltype_string, 2, "", ""), + new kelondroColumn(kelondroColumn.celltype_string, attributesMaxLength, "", ""), + }); private static kelondroTree createDB(File path, int bufferkb) { - return new kelondroTree(path, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, new int[] { - yacyNewsRecord.idLength(), // id = created + originator - yacyNewsRecord.categoryStringLength, // category - yacyCore.universalDateShortPattern.length(), // received - 2, - attributesMaxLength - }, true); + return new kelondroTree(path, bufferkb * 0x400, kelondroTree.defaultObjectCachePercent, rowdef, true); } private void resetDB() { @@ -153,7 +156,7 @@ public class yacyNewsDB { } public Object next() { - return b2r((byte[][]) rowIterator.next()); + return b2r((kelondroRow.Entry) rowIterator.next()); } public void remove() { @@ -181,6 +184,7 @@ public class yacyNewsDB { ); } + /* private static yacyNewsRecord b2r(byte[][] b) { if (b == null) return null; return new yacyNewsRecord( @@ -191,18 +195,18 @@ public class yacyNewsDB { serverCodings.string2map(new String(b[4])) ); } - - private static byte[][] r2b(yacyNewsRecord r) { +*/ + private kelondroRow.Entry r2b(yacyNewsRecord r) { if (r == null) return null; String attributes = r.attributes().toString(); if (attributes.length() > attributesMaxLength) throw new IllegalArgumentException("attribute length=" + attributes.length() + " exceeds maximum size=" + attributesMaxLength); - byte[][] b = new byte[5][]; - b[0] = r.id().getBytes(); - b[1] = r.category().getBytes(); - b[2] = (r.received() == null) ? null : yacyCore.universalDateShortString(r.received()).getBytes(); - b[3] = kelondroBase64Order.enhancedCoder.encodeLong(r.distributed(), 2).getBytes(); - b[4] = attributes.getBytes(); - return b; + kelondroRow.Entry entry = news.row().newEntry(); + entry.setCol(0, r.id().getBytes()); + entry.setCol(1, r.category().getBytes()); + entry.setCol(2, (r.received() == null) ? null : yacyCore.universalDateShortString(r.received()).getBytes()); + entry.setCol(3, kelondroBase64Order.enhancedCoder.encodeLong(r.distributed(), 2).getBytes()); + entry.setCol(4, attributes.getBytes()); + return entry; } } \ No newline at end of file diff --git a/source/de/anomic/yacy/yacyNewsQueue.java b/source/de/anomic/yacy/yacyNewsQueue.java index 86ebaac2e..212dd2f29 100644 --- a/source/de/anomic/yacy/yacyNewsQueue.java +++ b/source/de/anomic/yacy/yacyNewsQueue.java @@ -48,6 +48,7 @@ import java.io.File; import java.io.IOException; import java.util.Date; +import de.anomic.kelondro.kelondroColumn; import de.anomic.kelondro.kelondroRow; import de.anomic.kelondro.kelondroStack; import de.anomic.kelondro.kelondroException; @@ -74,12 +75,14 @@ public class yacyNewsQueue { queueStack = createStack(path); } } + + public static final kelondroRow rowdef = new kelondroRow(new kelondroColumn[]{ + new kelondroColumn(kelondroColumn.celltype_string, yacyNewsRecord.idLength(), "newsid", "id = created + originator"), + new kelondroColumn(kelondroColumn.celltype_string, yacyCore.universalDateShortPattern.length(), "last touched", "") + }); private static kelondroStack createStack(File path) { - return new kelondroStack(path, 0, new int[] { - yacyNewsRecord.idLength(), // id = created + originator - yacyCore.universalDateShortPattern.length() // last touched - }, true); + return new kelondroStack(path, 0, rowdef, true); } private void resetDB() {