added several consistency checks and small changes

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2400 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/index/indexRAMCacheRI.java

@@ -175,7 +175,7 @@ public final class indexRAMCacheRI extends indexAbstractRI implements indexRI {
     private long restore() throws IOException {
         File indexDumpFile = new File(databaseRoot, indexArrayFileName);
         if (!(indexDumpFile.exists())) return 0;
-        kelondroFixedWidthArray dumpArray = new kelondroFixedWidthArray(indexDumpFile);
+        kelondroFixedWidthArray dumpArray = new kelondroFixedWidthArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis);
         log.logConfig("restore array dump of index cache, " + dumpArray.size() + " word/URL relations");
         long startTime = System.currentTimeMillis();
         long messageTime = System.currentTimeMillis() + 5000;

source/de/anomic/kelondro/kelondroAbstractIOChunks.java

@@ -58,6 +58,7 @@ public abstract class kelondroAbstractIOChunks {
     }
     
     // pseudo-native methods:
+    abstract public long length() throws IOException;
     abstract public int read(long pos, byte[] b, int off, int len) throws IOException;
     abstract public void write(long pos, byte[] b, int off, int len) throws IOException;
     abstract public void close() throws IOException;

source/de/anomic/kelondro/kelondroAbstractRA.java

@@ -64,6 +64,7 @@ abstract class kelondroAbstractRA implements kelondroRA {
     }
 
     // pseudo-native methods:
+    abstract public long length() throws IOException;
     abstract public long available() throws IOException;
     
     abstract public int read() throws IOException;

source/de/anomic/kelondro/kelondroBufferedIOChunks.java

@@ -70,6 +70,10 @@ public final class kelondroBufferedIOChunks extends kelondroAbstractIOChunks imp
         this.lastCommit = System.currentTimeMillis();
     }
 
+    public long length() throws IOException {
+        return ra.length();
+    }
+    
     public int read(long pos, byte[] b, int off, int len) throws IOException {
         assert (b.length >= off + len): "read pos=" + pos  + ", b.length=" + b.length + ", off=" + off + ", len=" + len;
         

source/de/anomic/kelondro/kelondroBufferedRA.java

@@ -77,6 +77,10 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA
         this.bufferWritten = true;
     }
 
+    public long length() throws IOException {
+        return ra.length();
+    }
+    
     public long available() throws IOException {
         synchronized (ra) {
             ra.seek(seekpos);

source/de/anomic/kelondro/kelondroCachedRA.java

@@ -65,6 +65,10 @@ public class kelondroCachedRA extends kelondroAbstractRA implements kelondroRA {
         this.seekpos = 0;
     }
     
+    public long length() throws IOException {
+        return ra.available();
+    }
+    
     public long available() throws IOException {
         synchronized (ra) {
             ra.seek(seekpos);

source/de/anomic/kelondro/kelondroCollectionIndex.java

@@ -114,16 +114,15 @@ public class kelondroCollectionIndex {
     
     private kelondroFixedWidthArray openArrayFile(int partitionNumber, int serialNumber, boolean create) throws IOException {
         File f = arrayFile(path, filenameStub, loadfactor, rowdef.objectsize(), partitionNumber, serialNumber);
-        
+        int load = arrayCapacity(partitionNumber);
+        kelondroRow rowdef = new kelondroRow(
+                "byte[] key-" + index.row().width(0) + "," +
+                "byte[] collection-" + (kelondroRowCollection.exportOverheadSize + load * this.rowdef.objectsize())
+                );
         if (f.exists()) {
-            return new kelondroFixedWidthArray(f);
+            return new kelondroFixedWidthArray(f, rowdef);
         } else if (create) {
-            int load = arrayCapacity(partitionNumber);
-            kelondroRow row = new kelondroRow(
-                    "byte[] key-" + index.row().width(0) + "," +
-                    "byte[] collection-" + (kelondroRowCollection.exportOverheadSize + load * this.rowdef.objectsize())
-                    );
-            return new kelondroFixedWidthArray(f, row, 0, true);
+            return new kelondroFixedWidthArray(f, rowdef, 0, true);
         } else {
             return null;
         }

source/de/anomic/kelondro/kelondroDyn.java

@@ -334,6 +334,10 @@ public class kelondroDyn extends kelondroTree {
             this.filekey = filekey;
         }
 
+        public long length() throws IOException {
+            return Long.MAX_VALUE;
+        }
+        
         public long available() throws IOException {
             return Long.MAX_VALUE;
         }

source/de/anomic/kelondro/kelondroFileRA.java

@@ -60,6 +60,10 @@ public final class kelondroFileRA extends kelondroAbstractRA implements kelondro
         RAFile = new RandomAccessFile(file, "rw");
     }
     
+    public long length() throws IOException {
+        return RAFile.length();
+    }
+    
     public long available() throws IOException {
         return RAFile.length() - RAFile.getFilePointer();
     }

source/de/anomic/kelondro/kelondroFixedWidthArray.java

@@ -67,9 +67,10 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
         }
     }
 
-    public kelondroFixedWidthArray(File file) throws IOException{
+    public kelondroFixedWidthArray(File file, kelondroRow rowdef) throws IOException{
         // this opens a file with an existing array
         super(file, 0, 0);
+        super.assignRowdef(rowdef);
     }
     
     public synchronized kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException {
@@ -92,7 +93,6 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
     }
     
     public synchronized kelondroRow.Entry get(int index) throws IOException {
-        //if (index >= size()) throw new kelondroException(filename, "out of bounds, index=" + index + ", size=" + size());
         return row().newEntry(getNode(new Handle(index)).getValueRow());
     }
 
@@ -134,7 +134,8 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
     public static void main(String[] args) {
         File f = new File("d:\\\\mc\\privat\\fixtest.db");
         f.delete();
-        kelondroFixedWidthArray k = new kelondroFixedWidthArray(f, new kelondroRow("byte[] a-12, byte[] b-4"), 6, true);
+        kelondroRow rowdef = new kelondroRow("byte[] a-12, byte[] b-4");
+        kelondroFixedWidthArray k = new kelondroFixedWidthArray(f, rowdef, 6, true);
         try {
             k.set(3, k.row().newEntry(new byte[][]{
                 "test123".getBytes(), "abcd".getBytes()}));
@@ -142,7 +143,7 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
                 "test456".getBytes(), "efgh".getBytes()}));
             k.close();
             
-            k = new kelondroFixedWidthArray(f);
+            k = new kelondroFixedWidthArray(f, rowdef);
             System.out.println(k.get(2).toString());
             System.out.println(k.get(3).toString());
             System.out.println(k.get(4).toString());

source/de/anomic/kelondro/kelondroFlexWidthArray.java

@@ -78,12 +78,10 @@ public class kelondroFlexWidthArray implements kelondroArray {
             if ((files[i].startsWith("col.") && (files[i].endsWith(".list")))) {
                 int colstart = Integer.parseInt(files[i].substring(4, 7));
                 int colend   = (files[i].charAt(7) == '-') ? Integer.parseInt(files[i].substring(8, 11)) : colstart;
-                /*
-                int columns[] = new int[colend - colstart + 1];
-                for (int j = colstart; j <= colend; j++) columns[j-colstart] = rowdef.width(j);
-                col[colstart] = new kelondroFixedWidthArray(new File(path, files[i]), columns, 0, true);
-                */
-                col[colstart] = new kelondroFixedWidthArray(new File(tabledir, files[i]));
+                
+                kelondroColumn columns[] = new kelondroColumn[colend - colstart + 1];
+                for (int j = colstart; j <= colend; j++) columns[j-colstart] = rowdef.column(j);
+                col[colstart] = new kelondroFixedWidthArray(new File(tabledir, files[i]), new kelondroRow(columns));
                 for (int j = colstart; j <= colend; j++) check = check.substring(0, j) + "X" + check.substring(j + 1);
             }
         }

source/de/anomic/kelondro/kelondroHashtable.java

@@ -171,9 +171,9 @@ public class kelondroHashtable {
         }
     }
 
-    public kelondroHashtable(File file) throws IOException{
-	// this opens a file with an existing hashtable
-	this.hashArray = new kelondroFixedWidthArray(file);
+    public kelondroHashtable(File file, kelondroRow rowdef) throws IOException{
+        // this opens a file with an existing hashtable
+        this.hashArray = new kelondroFixedWidthArray(file, rowdef);
         this.offset    = hashArray.geti(0);
         this.maxk      = hashArray.geti(1);
         this.maxrehash = hashArray.geti(2);

source/de/anomic/kelondro/kelondroIOChunks.java

@@ -49,6 +49,7 @@ public interface kelondroIOChunks {
     public String name();
     
     // pseudo-native methods:
+    public long length() throws IOException;
     public int read(long pos, byte[] b, int off, int len) throws IOException;
     public void write(long pos, byte[] b, int off, int len) throws IOException;
     public void commit() throws IOException;

source/de/anomic/kelondro/kelondroNIOFileRA.java

@@ -113,6 +113,10 @@ public class kelondroNIOFileRA extends kelondroAbstractRA implements kelondroRA
         return true;
     }
     
+    public long length() throws IOException {
+        return RAFile.length();
+    }
+    
     public long available() throws IOException {
         return RAFile.length() - RAFile.getFilePointer();
     }

source/de/anomic/kelondro/kelondroRA.java

@@ -59,6 +59,7 @@ public interface kelondroRA {
     public String name();
     
     // pseudo-native methods:
+    public long length() throws IOException;
     public long available() throws IOException;
     
     public int read() throws IOException;

source/de/anomic/kelondro/kelondroRAIOChunks.java

@@ -52,6 +52,10 @@ public final class kelondroRAIOChunks extends kelondroAbstractIOChunks implement
         this.ra = ra;
     }
 
+    public long length() throws IOException {
+        return ra.length();
+    }
+    
     public int read(long pos, byte[] b, int off, int len) throws IOException {
         if (len == 0) return 0;
         synchronized (this.ra) {

source/de/anomic/kelondro/kelondroRecords.java

@@ -693,12 +693,10 @@ public class kelondroRecords {
         
         private void setValue(byte[] value, int valueoffset, int valuewidth, byte[] targetarray, int targetoffset) {
             if (value == null) {
-                while (valuewidth-- > 0) targetarray[targetoffset + valuewidth] = 0;
+                while (valuewidth-- > 0) targetarray[targetoffset++] = 0;
             } else {
                 System.arraycopy(value, valueoffset, targetarray, targetoffset, Math.min(value.length, valuewidth)); // error?
-                if (value.length < valuewidth) {
-                    while (valuewidth-- > value.length) targetarray[targetoffset + valuewidth] = 0;
-                }
+                while (valuewidth-- > value.length) targetarray[targetoffset + valuewidth] = 0;
             }
         }
         
@@ -1088,12 +1086,15 @@ public class kelondroRecords {
             // if the initTime is exceeded, the method throws an kelondroException
             markedDeleted = new HashSet();
             long timeLimit = (maxInitTime < 0) ? Long.MAX_VALUE : System.currentTimeMillis() + maxInitTime;
+            long seekp;
             synchronized (USAGE) {
                 if (USAGE.FREEC != 0) {
                     Handle h = USAGE.FREEH;
                     while (h.index != NUL) {
                         markedDeleted.add(h);
-                        h = new Handle(entryFile.readInt(seekpos(h)));
+                        seekp = seekpos(h);
+                        if (seekp > entryFile.length()) throw new kelondroException("contentNodeIterator: seek position " + seekp + "/" + h.index + " out of file size " + entryFile.length() + "/" + ((entryFile.length() - POS_NODES) / recordsize));
+                        h = new Handle(entryFile.readInt(seekp));
                         if (System.currentTimeMillis() > timeLimit) throw new kelondroException(filename, "time limit of " + maxInitTime + " exceeded; > " + markedDeleted.size() + " deleted entries");
                     }
                 }
@@ -1278,8 +1279,13 @@ public class kelondroRecords {
                         index = USAGE.USEDC - 1;
                     } else {
                         index = USAGE.FREEH.index;
-                        // read link to next element to FREEH chain
-                        USAGE.FREEH.index = entryFile.readInt(seekpos(USAGE.FREEH));
+                        
+                        // check for valid seek position
+                        long seekp = seekpos(USAGE.FREEH);
+                        if (seekp > entryFile.length()) throw new kelondroException("new Handle: seek position " + seekp + "/" + USAGE.FREEH.index + " out of file size " + entryFile.length() + "/" + ((entryFile.length() - POS_NODES) / recordsize));
+                        
+                        // read link to next element of FREEH chain
+                        USAGE.FREEH.index = entryFile.readInt(seekp);
                     }
                     USAGE.write();
                 }

source/de/anomic/kelondro/kelondroRow.java

@@ -55,19 +55,11 @@ public class kelondroRow {
         // example:
         //# Structure=<pivot-12>,'=',<UDate-3>,<VDate-3>,<LCount-2>,<GCount-2>,<ICount-2>,<DCount-2>,<TLength-3>,<WACount-3>,<WUCount-3>,<Flags-1>
 
-        // parse a structure string
-        kelondroColumn pivot_col = null;
-
         // parse pivot definition:
-        int p = structure.indexOf(",'='");
-        if (p >= 0) {
-            String pivot = structure.substring(0, p);
-            structure = structure.substring(p + 5);
-            pivot_col = new kelondroColumn(pivot);
-        }
+        structure.replace('=', ',');
         
         // parse property part definition:
-        p = structure.indexOf(",'|'");
+        int p = structure.indexOf('|');
         if (p < 0) p = structure.length();
         ArrayList l = new ArrayList();
         String attr = structure.substring(0, p);
@@ -77,21 +69,14 @@ public class kelondroRow {
         }
         
         // define columns
-        int piv_offset = (pivot_col == null) ? 0 : 1;
-        this.row = new kelondroColumn[l.size() + piv_offset];
+        this.row = new kelondroColumn[l.size()];
         this.colstart = new int[row.length];
         this.objectsize = 0;
-        if (pivot_col != null) {
-            this.colstart[0] = 0;
-            this.row[0] = pivot_col;
-            this.objectsize += this.row[0].cellwidth();
-        }
         for (int i = 0; i < l.size(); i++) {
-            this.colstart[i + piv_offset] = this.objectsize;
-            this.row[i + piv_offset] = (kelondroColumn) l.get(i);
-            this.objectsize += this.row[i + piv_offset].cellwidth();
+            this.colstart[i] = this.objectsize;
+            this.row[i] = (kelondroColumn) l.get(i);
+            this.objectsize += this.row[i].cellwidth();
         }
-
     }
     
     private void genNickRef() {

source/de/anomic/plasma/plasmaSwitchboard.java

@@ -134,7 +134,6 @@ import de.anomic.htmlFilter.htmlFilterContentScraper;
 import de.anomic.http.httpHeader;
 import de.anomic.http.httpRemoteProxyConfig;
 import de.anomic.http.httpc;
-import de.anomic.http.httpdHandler;
 import de.anomic.index.indexContainer;
 import de.anomic.index.indexEntry;
 import de.anomic.index.indexEntryAttribute;

source/de/anomic/plasma/plasmaWordIndex.java

@@ -202,9 +202,9 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI {
 
     public void flushCacheSome() {
         synchronized (ramCache) { ramCache.shiftK2W(); }
-        int flushCount = ramCache.wSize() / 400;
-        if (flushCount > 200) flushCount = 200;
-        if (flushCount < 20) flushCount = Math.min(20, ramCache.wSize());
+        int flushCount = ramCache.wSize() / 420;
+        if (flushCount > 100) flushCount = 100;
+        if (flushCount < 10) flushCount = Math.min(10, ramCache.wSize());
         flushCache(flushCount);
     }