- more asserts

- simplified indexURLEntry

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2891 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/index/indexEntry.java

@@ -31,7 +31,6 @@ import de.anomic.kelondro.kelondroRow;
 public interface indexEntry {
 
     public Object clone();
-    public byte[] toEncodedByteArrayForm(boolean includeUrlHash); // shall be replaced by toKelondroEntry()
     public String toPropertyForm(boolean displayFormat);
     public kelondroRow.Entry toKelondroEntry();
     

source/de/anomic/index/indexRAMCacheRI.java

@@ -38,7 +38,6 @@ import de.anomic.kelondro.kelondroFixedWidthArray;
 import de.anomic.kelondro.kelondroMScoreCluster;
 import de.anomic.kelondro.kelondroNaturalOrder;
 import de.anomic.kelondro.kelondroRow;
-import de.anomic.plasma.plasmaWordIndexAssortment;
 import de.anomic.server.logging.serverLog;
 import de.anomic.yacy.yacySeedDB;
 
@@ -46,6 +45,11 @@ public final class indexRAMCacheRI implements indexRI {
 
     // environment constants
     public  static final long wCacheMaxAge = 1000 * 60 * 30; // milliseconds; 30 minutes
+    public  static final kelondroRow bufferStructureBasis = new kelondroRow(
+            "byte[] wordhash-" + indexEntryAttribute.wordHashLength + ", " +
+            "Cardinal occ-4 {b256}, " +
+            "Cardinal time-8 {b256}, " +
+            "byte[] urlprops-" + indexURLEntry.urlEntryRow.objectsize());
         
     // class variables
     private final File databaseRoot;
@@ -98,7 +102,7 @@ public final class indexRAMCacheRI implements indexRI {
         File indexDumpFile = new File(databaseRoot, indexArrayFileName);
         if (indexDumpFile.exists()) indexDumpFile.delete();
         kelondroFixedWidthArray dumpArray = null;
-            dumpArray = new kelondroFixedWidthArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0);
+            dumpArray = new kelondroFixedWidthArray(indexDumpFile, bufferStructureBasis, 0);
             long startTime = System.currentTimeMillis();
             long messageTime = System.currentTimeMillis() + 5000;
             long wordsPerSecond = 0, wordcount = 0, urlcount = 0;
@@ -127,8 +131,7 @@ public final class indexRAMCacheRI implements indexRI {
                             row.setCol(0, wordHash.getBytes());
                             row.setCol(1, kelondroNaturalOrder.encodeLong(container.size(), 4));
                             row.setCol(2, kelondroNaturalOrder.encodeLong(updateTime, 8));
-                            row.setCol(3, iEntry.urlHash().getBytes());
+                            row.setCol(3, iEntry.toKelondroEntry().bytes());
-                            row.setCol(4, iEntry.toEncodedByteArrayForm(false));
                             dumpArray.set((int) urlcount++, row);
                         }
                     }
@@ -152,7 +155,7 @@ public final class indexRAMCacheRI implements indexRI {
     private long restore() throws IOException {
         File indexDumpFile = new File(databaseRoot, indexArrayFileName);
         if (!(indexDumpFile.exists())) return 0;
-        kelondroFixedWidthArray dumpArray = new kelondroFixedWidthArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0);
+        kelondroFixedWidthArray dumpArray = new kelondroFixedWidthArray(indexDumpFile, bufferStructureBasis, 0);
         log.logConfig("restore array dump of index cache '" + indexArrayFileName + "', " + dumpArray.size() + " word/URL relations");
         long startTime = System.currentTimeMillis();
         long messageTime = System.currentTimeMillis() + 5000;
@@ -168,10 +171,10 @@ public final class indexRAMCacheRI implements indexRI {
                 while (i.hasNext()) {
                     // get out one entry
                     row = (kelondroRow.Entry) i.next();
-                    if ((row == null) || (row.empty(0)) || (row.empty(3)) || (row.empty(4))) continue;
+                    if ((row == null) || (row.empty(0)) || (row.empty(3))) continue;
                     wordHash = row.getColString(0, "UTF-8");
                     //creationTime = kelondroRecords.bytes2long(row[2]);
-                    wordEntry = new indexURLEntry(row.getColString(3, null), row.getColString(4, null));
+                    wordEntry = new indexURLEntry(row.getColBytes(3));
                     // store to cache
                     addEntry(wordHash, wordEntry, startTime, false);
                     urlCount++;

source/de/anomic/index/indexURLEntry.java

@@ -135,18 +135,6 @@ public class indexURLEntry implements Cloneable, indexEntry {
         return new indexURLEntry(b);
     }
     
-    public static int encodedByteArrayFormLength(boolean includingHeader) {
-        // the size of the index entry attributes when encoded to string
-        return (includingHeader) ? urlEntryRow.objectsize() : urlEntryRow.objectsize() - indexURL.urlHashLength;
-    }
-    
-    public byte[] toEncodedByteArrayForm(boolean includeHash) {
-        if (includeHash) return entry.bytes();
-        byte[] b = new byte[urlEntryRow.objectsize() - indexURL.urlHashLength];
-        System.arraycopy(entry.bytes(), indexURL.urlHashLength, b, 0, b.length);
-        return b;
-    }
-
     public String toPropertyForm(boolean displayFormat) {
         return entry.toPropertyForm(true, displayFormat, displayFormat);
     }

source/de/anomic/kelondro/kelondroFlexSplitTable.java

@@ -151,6 +151,7 @@ public class kelondroFlexSplitTable implements kelondroIndex {
     }
     
     public synchronized kelondroRow.Entry put(kelondroRow.Entry row, Date entryDate) throws IOException {
+        assert row.bytes().length <= this.rowdef.objectsize;
         Object[] keeper = keeperOf(row.getColBytes(0));
         if (keeper != null) return ((kelondroIndex) keeper[0]).put(row);
         String suffix = dateSuffix(entryDate);
@@ -182,6 +183,7 @@ public class kelondroFlexSplitTable implements kelondroIndex {
     }
     
     public synchronized void addUnique(kelondroRow.Entry row, Date entryDate) throws IOException {
+        assert row.bytes().length <= this.rowdef.objectsize;
         String suffix = dateSuffix(entryDate);
         if (suffix == null) return;
         kelondroIndex table = (kelondroIndex) tables.get(suffix);

source/de/anomic/kelondro/kelondroFlexTable.java

@@ -171,6 +171,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
     public synchronized kelondroRow.Entry put(kelondroRow.Entry row) throws IOException {
         assert (row != null);
         assert (!(serverLog.allZero(row.getColBytes(0))));
+        assert row.bytes().length <= this.rowdef.objectsize;
         int i = index.geti(row.getColBytes(0));
         if (i < 0) {
             index.puti(row.getColBytes(0), super.add(row));
@@ -184,6 +185,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
     }
     
     public synchronized void addUnique(kelondroRow.Entry row) throws IOException {
+        assert row.bytes().length <= this.rowdef.objectsize;
         index.addi(row.getColBytes(0), super.add(row));
     }
     

source/de/anomic/kelondro/kelondroFlexWidthArray.java

@@ -160,6 +160,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
     }
     
     public kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException {
+        assert rowentry.bytes().length <= this.rowdef.objectsize;
         int c = 0;
         kelondroRow.Entry e0, e1, p;
         p = rowdef.newEntry();
@@ -183,6 +184,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
     }
     
     public int add(kelondroRow.Entry rowentry) throws IOException {
+        assert rowentry.bytes().length <= this.rowdef.objectsize;
         kelondroRow.Entry e;
         int index = -1;
         int lastcol;

source/de/anomic/kelondro/kelondroRecords.java

@@ -925,16 +925,22 @@ public class kelondroRecords {
     private final void assignRowdef(kelondroRow rowdef) {
         // overwrites a given rowdef
         // the new rowdef must be compatible
-        if (rowdef.columns() < ROW.columns())
+        /*
+        if ((rowdef.columns() != ROW.columns()) &&
+            ((rowdef.columns() + 1 != ROW.columns()) ||
+             (rowdef.column(rowdef.columns() - 1).cellwidth() != (ROW.column(ROW.columns() - 1).cellwidth() + ROW.column(ROW.columns() - 2).cellwidth()))))
             throw new kelondroException(this.filename,
                     "new rowdef '" + rowdef.toString() + "' is not compatible with old rowdef '" + ROW.toString() + "', they have a different number of columns");
-
+         */
         // adopt encoder and cell type
+        /*
         kelondroColumn col;
         for (int i = 0; i < ROW.columns(); i++) {
             col = rowdef.column(i);
             ROW.column(i).setAttributes(col.nickname(), col.celltype(), col.encoder());
         }
+        */
+        this.ROW = rowdef;
     }
 
     protected final long seekpos(Handle handle) {

source/de/anomic/kelondro/kelondroTree.java

@@ -379,6 +379,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
         assert (newrow != null);
         assert (newrow.columns() == row().columns());
         assert (!(serverLog.allZero(newrow.getColBytes(primarykey()))));
+        assert newrow.bytes().length <= super.ROW.objectsize;
         // Associates the specified value with the specified key in this map
         kelondroRow.Entry result = null;
         //writeLock.stay(2000, 1000);

source/de/anomic/plasma/plasmaWordIndexAssortment.java

@@ -58,7 +58,7 @@ import java.util.Iterator;
 
 import de.anomic.index.indexContainer;
 import de.anomic.index.indexEntry;
-import de.anomic.index.indexEntryAttribute;
+import de.anomic.index.indexRAMCacheRI;
 import de.anomic.index.indexURLEntry;
 import de.anomic.kelondro.kelondroCache;
 import de.anomic.kelondro.kelondroColumn;
@@ -71,12 +71,6 @@ public final class plasmaWordIndexAssortment {
     
     // environment constants
     private static final String assortmentFileName = "indexAssortment";
-    public  static final kelondroRow bufferStructureBasis = new kelondroRow(
-        "byte[] wordhash-" + indexEntryAttribute.wordHashLength + ", " +
-        "Cardinal occ-4 {b256}, " +
-        "Cardinal time-8 {b256}, " +
-        "byte[] urlhash-" + indexEntryAttribute.urlHashLength + ", " +
-        "byte[] urlattr-" + indexURLEntry.encodedByteArrayFormLength(false));
     
     // class variables
     private File assortmentFile;
@@ -93,19 +87,18 @@ public final class plasmaWordIndexAssortment {
     }
 
     private static kelondroRow bufferStructure(int assortmentCapacity) {
-        kelondroColumn[] structure = new kelondroColumn[3 + 2 * assortmentCapacity];
+        kelondroColumn[] structure = new kelondroColumn[3 + assortmentCapacity];
-        structure[0] = bufferStructureBasis.column(0);
+        structure[0] = indexRAMCacheRI.bufferStructureBasis.column(0);
-        structure[1] = bufferStructureBasis.column(1);
+        structure[1] = indexRAMCacheRI.bufferStructureBasis.column(1);
-        structure[2] = bufferStructureBasis.column(2);
+        structure[2] = indexRAMCacheRI.bufferStructureBasis.column(2);
         for (int i = 0; i < assortmentCapacity; i++) {
-            structure[3 + 2 * i] = bufferStructureBasis.column(3);
+            structure[3 + i] = indexRAMCacheRI.bufferStructureBasis.column(3);
-            structure[4 + 2 * i] = bufferStructureBasis.column(4);
         }
         return new kelondroRow(structure);
     }
     
     private static int assortmentCapacity(int rowsize) {
-        return (rowsize - bufferStructureBasis.width(0) - bufferStructureBasis.width(1) - bufferStructureBasis.width(2)) / (bufferStructureBasis.width(3) + bufferStructureBasis.width(4));
+        return (rowsize - indexRAMCacheRI.bufferStructureBasis.width(0) - indexRAMCacheRI.bufferStructureBasis.width(1) - indexRAMCacheRI.bufferStructureBasis.width(2)) / indexRAMCacheRI.bufferStructureBasis.width(3);
     }
 
     public plasmaWordIndexAssortment(File storagePath, int assortmentLength, int bufferkb, long preloadTime, serverLog log) throws IOException {
@@ -141,8 +134,7 @@ public final class plasmaWordIndexAssortment {
         indexEntry entry;
         for (int i = 0; i < assortmentLength; i++) {
             entry = (indexEntry) entries.next();
-            row.setCol(3 + 2 * i, entry.urlHash().getBytes());
+            row.setCol(3 + i, entry.toKelondroEntry().bytes());
-            row.setCol(4 + 2 * i, entry.toEncodedByteArrayForm(false));
         }
         kelondroRow.Entry oldrow = null;
         try {
@@ -227,9 +219,7 @@ public final class plasmaWordIndexAssortment {
         indexContainer container = new indexContainer(wordHash);
         int al = assortmentCapacity(row.objectsize());
         for (int i = 0; i < al; i++) {
-            container.add(
+            container.add(new indexEntry[] { new indexURLEntry(row.getColBytes(3 + i)) }, updateTime);
-                    new indexEntry[] { new indexURLEntry(
-                            new String(row.getColBytes(3 + 2 * i)), new String(row.getColBytes(4 + 2 * i))) }, updateTime);
         }
         return container;
     }

source/de/anomic/plasma/plasmaWordIndexFile.java

@@ -91,7 +91,7 @@ public final class plasmaWordIndexFile {
         long cacheSize = theLocation.length();
         if (cacheSize > 1048576) cacheSize = 1048576;
         return kelondroTree.open(theLocation, cacheSize, 0, 
-                    new kelondroRow("byte[] urlhash-" + indexURL.urlHashLength + ", byte[] ba-" + indexURLEntry.encodedByteArrayFormLength(false)));
+                    new kelondroRow("byte[] urlhash-" + indexURL.urlHashLength + ", byte[] ba-" + (indexURLEntry.urlEntryRow.objectsize() - indexURL.urlHashLength)));
     }
 
     public static File wordHash2path(File databaseRoot, String hash) {
@@ -148,7 +148,7 @@ public final class plasmaWordIndexFile {
         if ((oldEntry != null) && (entry.isOlder(oldEntry))) { // A more recent Entry is already in this entity
             return false;
         }
-        return (theIndex.put(entry.urlHash().getBytes(), entry.toEncodedByteArrayForm(false)) == null);
+        return (theIndex.put(entry.toKelondroEntry()) == null);
     }
     
     public int addEntries(indexContainer container) throws IOException {