diff --git a/source/de/anomic/index/indexRAMCacheRI.java b/source/de/anomic/index/indexRAMCacheRI.java index 0458cb32e..c6383306a 100644 --- a/source/de/anomic/index/indexRAMCacheRI.java +++ b/source/de/anomic/index/indexRAMCacheRI.java @@ -175,7 +175,7 @@ public final class indexRAMCacheRI extends indexAbstractRI implements indexRI { private long restore() throws IOException { File indexDumpFile = new File(databaseRoot, indexArrayFileName); if (!(indexDumpFile.exists())) return 0; - kelondroFixedWidthArray dumpArray = new kelondroFixedWidthArray(indexDumpFile); + kelondroFixedWidthArray dumpArray = new kelondroFixedWidthArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis); log.logConfig("restore array dump of index cache, " + dumpArray.size() + " word/URL relations"); long startTime = System.currentTimeMillis(); long messageTime = System.currentTimeMillis() + 5000; diff --git a/source/de/anomic/kelondro/kelondroAbstractIOChunks.java b/source/de/anomic/kelondro/kelondroAbstractIOChunks.java index 9466358e5..6fdeed9e5 100644 --- a/source/de/anomic/kelondro/kelondroAbstractIOChunks.java +++ b/source/de/anomic/kelondro/kelondroAbstractIOChunks.java @@ -58,6 +58,7 @@ public abstract class kelondroAbstractIOChunks { } // pseudo-native methods: + abstract public long length() throws IOException; abstract public int read(long pos, byte[] b, int off, int len) throws IOException; abstract public void write(long pos, byte[] b, int off, int len) throws IOException; abstract public void close() throws IOException; diff --git a/source/de/anomic/kelondro/kelondroAbstractRA.java b/source/de/anomic/kelondro/kelondroAbstractRA.java index 1d27ec5ce..d569e5dd6 100644 --- a/source/de/anomic/kelondro/kelondroAbstractRA.java +++ b/source/de/anomic/kelondro/kelondroAbstractRA.java @@ -64,6 +64,7 @@ abstract class kelondroAbstractRA implements kelondroRA { } // pseudo-native methods: + abstract public long length() throws IOException; abstract public long available() throws IOException; abstract public int read() throws IOException; diff --git a/source/de/anomic/kelondro/kelondroBufferedIOChunks.java b/source/de/anomic/kelondro/kelondroBufferedIOChunks.java index 116054d3a..f1119e273 100644 --- a/source/de/anomic/kelondro/kelondroBufferedIOChunks.java +++ b/source/de/anomic/kelondro/kelondroBufferedIOChunks.java @@ -70,6 +70,10 @@ public final class kelondroBufferedIOChunks extends kelondroAbstractIOChunks imp this.lastCommit = System.currentTimeMillis(); } + public long length() throws IOException { + return ra.length(); + } + public int read(long pos, byte[] b, int off, int len) throws IOException { assert (b.length >= off + len): "read pos=" + pos + ", b.length=" + b.length + ", off=" + off + ", len=" + len; diff --git a/source/de/anomic/kelondro/kelondroBufferedRA.java b/source/de/anomic/kelondro/kelondroBufferedRA.java index 2ed5817cb..261a180d1 100644 --- a/source/de/anomic/kelondro/kelondroBufferedRA.java +++ b/source/de/anomic/kelondro/kelondroBufferedRA.java @@ -77,6 +77,10 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA this.bufferWritten = true; } + public long length() throws IOException { + return ra.length(); + } + public long available() throws IOException { synchronized (ra) { ra.seek(seekpos); diff --git a/source/de/anomic/kelondro/kelondroCachedRA.java b/source/de/anomic/kelondro/kelondroCachedRA.java index 87dee1bf1..7a005beaa 100644 --- a/source/de/anomic/kelondro/kelondroCachedRA.java +++ b/source/de/anomic/kelondro/kelondroCachedRA.java @@ -65,6 +65,10 @@ public class kelondroCachedRA extends kelondroAbstractRA implements kelondroRA { this.seekpos = 0; } + public long length() throws IOException { + return ra.available(); + } + public long available() throws IOException { synchronized (ra) { ra.seek(seekpos); diff --git a/source/de/anomic/kelondro/kelondroCollectionIndex.java b/source/de/anomic/kelondro/kelondroCollectionIndex.java index 4fddce3af..1f6e4ab07 100644 --- a/source/de/anomic/kelondro/kelondroCollectionIndex.java +++ b/source/de/anomic/kelondro/kelondroCollectionIndex.java @@ -114,16 +114,15 @@ public class kelondroCollectionIndex { private kelondroFixedWidthArray openArrayFile(int partitionNumber, int serialNumber, boolean create) throws IOException { File f = arrayFile(path, filenameStub, loadfactor, rowdef.objectsize(), partitionNumber, serialNumber); - + int load = arrayCapacity(partitionNumber); + kelondroRow rowdef = new kelondroRow( + "byte[] key-" + index.row().width(0) + "," + + "byte[] collection-" + (kelondroRowCollection.exportOverheadSize + load * this.rowdef.objectsize()) + ); if (f.exists()) { - return new kelondroFixedWidthArray(f); + return new kelondroFixedWidthArray(f, rowdef); } else if (create) { - int load = arrayCapacity(partitionNumber); - kelondroRow row = new kelondroRow( - "byte[] key-" + index.row().width(0) + "," + - "byte[] collection-" + (kelondroRowCollection.exportOverheadSize + load * this.rowdef.objectsize()) - ); - return new kelondroFixedWidthArray(f, row, 0, true); + return new kelondroFixedWidthArray(f, rowdef, 0, true); } else { return null; } diff --git a/source/de/anomic/kelondro/kelondroDyn.java b/source/de/anomic/kelondro/kelondroDyn.java index 9de99a341..38c45927f 100644 --- a/source/de/anomic/kelondro/kelondroDyn.java +++ b/source/de/anomic/kelondro/kelondroDyn.java @@ -334,6 +334,10 @@ public class kelondroDyn extends kelondroTree { this.filekey = filekey; } + public long length() throws IOException { + return Long.MAX_VALUE; + } + public long available() throws IOException { return Long.MAX_VALUE; } diff --git a/source/de/anomic/kelondro/kelondroFileRA.java b/source/de/anomic/kelondro/kelondroFileRA.java index 169dbdbf0..929fa56d4 100644 --- a/source/de/anomic/kelondro/kelondroFileRA.java +++ b/source/de/anomic/kelondro/kelondroFileRA.java @@ -60,6 +60,10 @@ public final class kelondroFileRA extends kelondroAbstractRA implements kelondro RAFile = new RandomAccessFile(file, "rw"); } + public long length() throws IOException { + return RAFile.length(); + } + public long available() throws IOException { return RAFile.length() - RAFile.getFilePointer(); } diff --git a/source/de/anomic/kelondro/kelondroFixedWidthArray.java b/source/de/anomic/kelondro/kelondroFixedWidthArray.java index 0d186f0dc..737db4fb1 100644 --- a/source/de/anomic/kelondro/kelondroFixedWidthArray.java +++ b/source/de/anomic/kelondro/kelondroFixedWidthArray.java @@ -67,9 +67,10 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro } } - public kelondroFixedWidthArray(File file) throws IOException{ + public kelondroFixedWidthArray(File file, kelondroRow rowdef) throws IOException{ // this opens a file with an existing array super(file, 0, 0); + super.assignRowdef(rowdef); } public synchronized kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException { @@ -92,7 +93,6 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro } public synchronized kelondroRow.Entry get(int index) throws IOException { - //if (index >= size()) throw new kelondroException(filename, "out of bounds, index=" + index + ", size=" + size()); return row().newEntry(getNode(new Handle(index)).getValueRow()); } @@ -134,7 +134,8 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro public static void main(String[] args) { File f = new File("d:\\\\mc\\privat\\fixtest.db"); f.delete(); - kelondroFixedWidthArray k = new kelondroFixedWidthArray(f, new kelondroRow("byte[] a-12, byte[] b-4"), 6, true); + kelondroRow rowdef = new kelondroRow("byte[] a-12, byte[] b-4"); + kelondroFixedWidthArray k = new kelondroFixedWidthArray(f, rowdef, 6, true); try { k.set(3, k.row().newEntry(new byte[][]{ "test123".getBytes(), "abcd".getBytes()})); @@ -142,7 +143,7 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro "test456".getBytes(), "efgh".getBytes()})); k.close(); - k = new kelondroFixedWidthArray(f); + k = new kelondroFixedWidthArray(f, rowdef); System.out.println(k.get(2).toString()); System.out.println(k.get(3).toString()); System.out.println(k.get(4).toString()); diff --git a/source/de/anomic/kelondro/kelondroFlexWidthArray.java b/source/de/anomic/kelondro/kelondroFlexWidthArray.java index 28816dd75..0ccf3fe15 100644 --- a/source/de/anomic/kelondro/kelondroFlexWidthArray.java +++ b/source/de/anomic/kelondro/kelondroFlexWidthArray.java @@ -78,12 +78,10 @@ public class kelondroFlexWidthArray implements kelondroArray { if ((files[i].startsWith("col.") && (files[i].endsWith(".list")))) { int colstart = Integer.parseInt(files[i].substring(4, 7)); int colend = (files[i].charAt(7) == '-') ? Integer.parseInt(files[i].substring(8, 11)) : colstart; - /* - int columns[] = new int[colend - colstart + 1]; - for (int j = colstart; j <= colend; j++) columns[j-colstart] = rowdef.width(j); - col[colstart] = new kelondroFixedWidthArray(new File(path, files[i]), columns, 0, true); - */ - col[colstart] = new kelondroFixedWidthArray(new File(tabledir, files[i])); + + kelondroColumn columns[] = new kelondroColumn[colend - colstart + 1]; + for (int j = colstart; j <= colend; j++) columns[j-colstart] = rowdef.column(j); + col[colstart] = new kelondroFixedWidthArray(new File(tabledir, files[i]), new kelondroRow(columns)); for (int j = colstart; j <= colend; j++) check = check.substring(0, j) + "X" + check.substring(j + 1); } } diff --git a/source/de/anomic/kelondro/kelondroHashtable.java b/source/de/anomic/kelondro/kelondroHashtable.java index 37403d88e..b5fe8e096 100644 --- a/source/de/anomic/kelondro/kelondroHashtable.java +++ b/source/de/anomic/kelondro/kelondroHashtable.java @@ -171,9 +171,9 @@ public class kelondroHashtable { } } - public kelondroHashtable(File file) throws IOException{ - // this opens a file with an existing hashtable - this.hashArray = new kelondroFixedWidthArray(file); + public kelondroHashtable(File file, kelondroRow rowdef) throws IOException{ + // this opens a file with an existing hashtable + this.hashArray = new kelondroFixedWidthArray(file, rowdef); this.offset = hashArray.geti(0); this.maxk = hashArray.geti(1); this.maxrehash = hashArray.geti(2); diff --git a/source/de/anomic/kelondro/kelondroIOChunks.java b/source/de/anomic/kelondro/kelondroIOChunks.java index ba0029b7c..5aa1d0a9d 100644 --- a/source/de/anomic/kelondro/kelondroIOChunks.java +++ b/source/de/anomic/kelondro/kelondroIOChunks.java @@ -49,6 +49,7 @@ public interface kelondroIOChunks { public String name(); // pseudo-native methods: + public long length() throws IOException; public int read(long pos, byte[] b, int off, int len) throws IOException; public void write(long pos, byte[] b, int off, int len) throws IOException; public void commit() throws IOException; diff --git a/source/de/anomic/kelondro/kelondroNIOFileRA.java b/source/de/anomic/kelondro/kelondroNIOFileRA.java index bfbddbcfb..ee51d6b6e 100644 --- a/source/de/anomic/kelondro/kelondroNIOFileRA.java +++ b/source/de/anomic/kelondro/kelondroNIOFileRA.java @@ -113,6 +113,10 @@ public class kelondroNIOFileRA extends kelondroAbstractRA implements kelondroRA return true; } + public long length() throws IOException { + return RAFile.length(); + } + public long available() throws IOException { return RAFile.length() - RAFile.getFilePointer(); } diff --git a/source/de/anomic/kelondro/kelondroRA.java b/source/de/anomic/kelondro/kelondroRA.java index bc831d3f1..14036246f 100644 --- a/source/de/anomic/kelondro/kelondroRA.java +++ b/source/de/anomic/kelondro/kelondroRA.java @@ -59,6 +59,7 @@ public interface kelondroRA { public String name(); // pseudo-native methods: + public long length() throws IOException; public long available() throws IOException; public int read() throws IOException; diff --git a/source/de/anomic/kelondro/kelondroRAIOChunks.java b/source/de/anomic/kelondro/kelondroRAIOChunks.java index 9d36482d9..777723f53 100644 --- a/source/de/anomic/kelondro/kelondroRAIOChunks.java +++ b/source/de/anomic/kelondro/kelondroRAIOChunks.java @@ -52,6 +52,10 @@ public final class kelondroRAIOChunks extends kelondroAbstractIOChunks implement this.ra = ra; } + public long length() throws IOException { + return ra.length(); + } + public int read(long pos, byte[] b, int off, int len) throws IOException { if (len == 0) return 0; synchronized (this.ra) { diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java index 7fcbaba36..6a7f57f4f 100644 --- a/source/de/anomic/kelondro/kelondroRecords.java +++ b/source/de/anomic/kelondro/kelondroRecords.java @@ -693,12 +693,10 @@ public class kelondroRecords { private void setValue(byte[] value, int valueoffset, int valuewidth, byte[] targetarray, int targetoffset) { if (value == null) { - while (valuewidth-- > 0) targetarray[targetoffset + valuewidth] = 0; + while (valuewidth-- > 0) targetarray[targetoffset++] = 0; } else { System.arraycopy(value, valueoffset, targetarray, targetoffset, Math.min(value.length, valuewidth)); // error? - if (value.length < valuewidth) { - while (valuewidth-- > value.length) targetarray[targetoffset + valuewidth] = 0; - } + while (valuewidth-- > value.length) targetarray[targetoffset + valuewidth] = 0; } } @@ -1088,12 +1086,15 @@ public class kelondroRecords { // if the initTime is exceeded, the method throws an kelondroException markedDeleted = new HashSet(); long timeLimit = (maxInitTime < 0) ? Long.MAX_VALUE : System.currentTimeMillis() + maxInitTime; + long seekp; synchronized (USAGE) { if (USAGE.FREEC != 0) { Handle h = USAGE.FREEH; while (h.index != NUL) { markedDeleted.add(h); - h = new Handle(entryFile.readInt(seekpos(h))); + seekp = seekpos(h); + if (seekp > entryFile.length()) throw new kelondroException("contentNodeIterator: seek position " + seekp + "/" + h.index + " out of file size " + entryFile.length() + "/" + ((entryFile.length() - POS_NODES) / recordsize)); + h = new Handle(entryFile.readInt(seekp)); if (System.currentTimeMillis() > timeLimit) throw new kelondroException(filename, "time limit of " + maxInitTime + " exceeded; > " + markedDeleted.size() + " deleted entries"); } } @@ -1278,8 +1279,13 @@ public class kelondroRecords { index = USAGE.USEDC - 1; } else { index = USAGE.FREEH.index; - // read link to next element to FREEH chain - USAGE.FREEH.index = entryFile.readInt(seekpos(USAGE.FREEH)); + + // check for valid seek position + long seekp = seekpos(USAGE.FREEH); + if (seekp > entryFile.length()) throw new kelondroException("new Handle: seek position " + seekp + "/" + USAGE.FREEH.index + " out of file size " + entryFile.length() + "/" + ((entryFile.length() - POS_NODES) / recordsize)); + + // read link to next element of FREEH chain + USAGE.FREEH.index = entryFile.readInt(seekp); } USAGE.write(); } diff --git a/source/de/anomic/kelondro/kelondroRow.java b/source/de/anomic/kelondro/kelondroRow.java index 3990e7ca0..dd30ea701 100644 --- a/source/de/anomic/kelondro/kelondroRow.java +++ b/source/de/anomic/kelondro/kelondroRow.java @@ -55,19 +55,11 @@ public class kelondroRow { // example: //# Structure=,'=',,,,,,,,,, - // parse a structure string - kelondroColumn pivot_col = null; - // parse pivot definition: - int p = structure.indexOf(",'='"); - if (p >= 0) { - String pivot = structure.substring(0, p); - structure = structure.substring(p + 5); - pivot_col = new kelondroColumn(pivot); - } + structure.replace('=', ','); // parse property part definition: - p = structure.indexOf(",'|'"); + int p = structure.indexOf('|'); if (p < 0) p = structure.length(); ArrayList l = new ArrayList(); String attr = structure.substring(0, p); @@ -77,21 +69,14 @@ public class kelondroRow { } // define columns - int piv_offset = (pivot_col == null) ? 0 : 1; - this.row = new kelondroColumn[l.size() + piv_offset]; + this.row = new kelondroColumn[l.size()]; this.colstart = new int[row.length]; this.objectsize = 0; - if (pivot_col != null) { - this.colstart[0] = 0; - this.row[0] = pivot_col; - this.objectsize += this.row[0].cellwidth(); - } for (int i = 0; i < l.size(); i++) { - this.colstart[i + piv_offset] = this.objectsize; - this.row[i + piv_offset] = (kelondroColumn) l.get(i); - this.objectsize += this.row[i + piv_offset].cellwidth(); + this.colstart[i] = this.objectsize; + this.row[i] = (kelondroColumn) l.get(i); + this.objectsize += this.row[i].cellwidth(); } - } private void genNickRef() { diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 3595bb8ca..4a8c93abc 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -134,7 +134,6 @@ import de.anomic.htmlFilter.htmlFilterContentScraper; import de.anomic.http.httpHeader; import de.anomic.http.httpRemoteProxyConfig; import de.anomic.http.httpc; -import de.anomic.http.httpdHandler; import de.anomic.index.indexContainer; import de.anomic.index.indexEntry; import de.anomic.index.indexEntryAttribute; diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index d6e4cd088..53bdf4c76 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -202,9 +202,9 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI { public void flushCacheSome() { synchronized (ramCache) { ramCache.shiftK2W(); } - int flushCount = ramCache.wSize() / 400; - if (flushCount > 200) flushCount = 200; - if (flushCount < 20) flushCount = Math.min(20, ramCache.wSize()); + int flushCount = ramCache.wSize() / 420; + if (flushCount > 100) flushCount = 100; + if (flushCount < 10) flushCount = Math.min(10, ramCache.wSize()); flushCache(flushCount); }