From 46a712e195379d4f2aa75a5ccba1024be1f35122 Mon Sep 17 00:00:00 2001 From: orbiter Date: Wed, 1 Nov 2006 14:00:15 +0000 Subject: [PATCH] - more asserts - simplified indexURLEntry git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2891 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/de/anomic/index/indexEntry.java | 1 - source/de/anomic/index/indexRAMCacheRI.java | 23 ++++++++------- source/de/anomic/index/indexURLEntry.java | 14 +--------- .../kelondro/kelondroFlexSplitTable.java | 2 ++ .../de/anomic/kelondro/kelondroFlexTable.java | 2 ++ .../kelondro/kelondroFlexWidthArray.java | 2 ++ .../de/anomic/kelondro/kelondroRecords.java | 10 +++++-- source/de/anomic/kelondro/kelondroTree.java | 1 + .../plasma/plasmaWordIndexAssortment.java | 28 ++++++------------- .../de/anomic/plasma/plasmaWordIndexFile.java | 4 +-- 10 files changed, 40 insertions(+), 47 deletions(-) diff --git a/source/de/anomic/index/indexEntry.java b/source/de/anomic/index/indexEntry.java index a4881ab8f..8fc17aa22 100644 --- a/source/de/anomic/index/indexEntry.java +++ b/source/de/anomic/index/indexEntry.java @@ -31,7 +31,6 @@ import de.anomic.kelondro.kelondroRow; public interface indexEntry { public Object clone(); - public byte[] toEncodedByteArrayForm(boolean includeUrlHash); // shall be replaced by toKelondroEntry() public String toPropertyForm(boolean displayFormat); public kelondroRow.Entry toKelondroEntry(); diff --git a/source/de/anomic/index/indexRAMCacheRI.java b/source/de/anomic/index/indexRAMCacheRI.java index 8a2dfe02b..0b4c1f77f 100644 --- a/source/de/anomic/index/indexRAMCacheRI.java +++ b/source/de/anomic/index/indexRAMCacheRI.java @@ -38,15 +38,19 @@ import de.anomic.kelondro.kelondroFixedWidthArray; import de.anomic.kelondro.kelondroMScoreCluster; import de.anomic.kelondro.kelondroNaturalOrder; import de.anomic.kelondro.kelondroRow; -import de.anomic.plasma.plasmaWordIndexAssortment; import de.anomic.server.logging.serverLog; import de.anomic.yacy.yacySeedDB; public final class indexRAMCacheRI implements indexRI { // environment constants - public static final long wCacheMaxAge = 1000 * 60 * 30; // milliseconds; 30 minutes - + public static final long wCacheMaxAge = 1000 * 60 * 30; // milliseconds; 30 minutes + public static final kelondroRow bufferStructureBasis = new kelondroRow( + "byte[] wordhash-" + indexEntryAttribute.wordHashLength + ", " + + "Cardinal occ-4 {b256}, " + + "Cardinal time-8 {b256}, " + + "byte[] urlprops-" + indexURLEntry.urlEntryRow.objectsize()); + // class variables private final File databaseRoot; protected final TreeMap cache; // wordhash-container @@ -64,7 +68,7 @@ public final class indexRAMCacheRI implements indexRI { maxKey = ""; for (int i = 0; i < yacySeedDB.commonHashLength; i++) maxKey += 'z'; //minKey = ""; for (int i = 0; i < yacySeedDB.commonHashLength; i++) maxKey += '-'; } - + public indexRAMCacheRI(File databaseRoot, int wCacheReferenceLimitInit, String dumpname, serverLog log) { // creates a new index cache @@ -98,7 +102,7 @@ public final class indexRAMCacheRI implements indexRI { File indexDumpFile = new File(databaseRoot, indexArrayFileName); if (indexDumpFile.exists()) indexDumpFile.delete(); kelondroFixedWidthArray dumpArray = null; - dumpArray = new kelondroFixedWidthArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0); + dumpArray = new kelondroFixedWidthArray(indexDumpFile, bufferStructureBasis, 0); long startTime = System.currentTimeMillis(); long messageTime = System.currentTimeMillis() + 5000; long wordsPerSecond = 0, wordcount = 0, urlcount = 0; @@ -127,8 +131,7 @@ public final class indexRAMCacheRI implements indexRI { row.setCol(0, wordHash.getBytes()); row.setCol(1, kelondroNaturalOrder.encodeLong(container.size(), 4)); row.setCol(2, kelondroNaturalOrder.encodeLong(updateTime, 8)); - row.setCol(3, iEntry.urlHash().getBytes()); - row.setCol(4, iEntry.toEncodedByteArrayForm(false)); + row.setCol(3, iEntry.toKelondroEntry().bytes()); dumpArray.set((int) urlcount++, row); } } @@ -152,7 +155,7 @@ public final class indexRAMCacheRI implements indexRI { private long restore() throws IOException { File indexDumpFile = new File(databaseRoot, indexArrayFileName); if (!(indexDumpFile.exists())) return 0; - kelondroFixedWidthArray dumpArray = new kelondroFixedWidthArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0); + kelondroFixedWidthArray dumpArray = new kelondroFixedWidthArray(indexDumpFile, bufferStructureBasis, 0); log.logConfig("restore array dump of index cache '" + indexArrayFileName + "', " + dumpArray.size() + " word/URL relations"); long startTime = System.currentTimeMillis(); long messageTime = System.currentTimeMillis() + 5000; @@ -168,10 +171,10 @@ public final class indexRAMCacheRI implements indexRI { while (i.hasNext()) { // get out one entry row = (kelondroRow.Entry) i.next(); - if ((row == null) || (row.empty(0)) || (row.empty(3)) || (row.empty(4))) continue; + if ((row == null) || (row.empty(0)) || (row.empty(3))) continue; wordHash = row.getColString(0, "UTF-8"); //creationTime = kelondroRecords.bytes2long(row[2]); - wordEntry = new indexURLEntry(row.getColString(3, null), row.getColString(4, null)); + wordEntry = new indexURLEntry(row.getColBytes(3)); // store to cache addEntry(wordHash, wordEntry, startTime, false); urlCount++; diff --git a/source/de/anomic/index/indexURLEntry.java b/source/de/anomic/index/indexURLEntry.java index 4a6b1411e..54f307440 100644 --- a/source/de/anomic/index/indexURLEntry.java +++ b/source/de/anomic/index/indexURLEntry.java @@ -135,19 +135,7 @@ public class indexURLEntry implements Cloneable, indexEntry { return new indexURLEntry(b); } - public static int encodedByteArrayFormLength(boolean includingHeader) { - // the size of the index entry attributes when encoded to string - return (includingHeader) ? urlEntryRow.objectsize() : urlEntryRow.objectsize() - indexURL.urlHashLength; - } - - public byte[] toEncodedByteArrayForm(boolean includeHash) { - if (includeHash) return entry.bytes(); - byte[] b = new byte[urlEntryRow.objectsize() - indexURL.urlHashLength]; - System.arraycopy(entry.bytes(), indexURL.urlHashLength, b, 0, b.length); - return b; - } - - public String toPropertyForm(boolean displayFormat) { + public String toPropertyForm(boolean displayFormat) { return entry.toPropertyForm(true, displayFormat, displayFormat); } diff --git a/source/de/anomic/kelondro/kelondroFlexSplitTable.java b/source/de/anomic/kelondro/kelondroFlexSplitTable.java index 64f6999f6..5c84f2893 100644 --- a/source/de/anomic/kelondro/kelondroFlexSplitTable.java +++ b/source/de/anomic/kelondro/kelondroFlexSplitTable.java @@ -151,6 +151,7 @@ public class kelondroFlexSplitTable implements kelondroIndex { } public synchronized kelondroRow.Entry put(kelondroRow.Entry row, Date entryDate) throws IOException { + assert row.bytes().length <= this.rowdef.objectsize; Object[] keeper = keeperOf(row.getColBytes(0)); if (keeper != null) return ((kelondroIndex) keeper[0]).put(row); String suffix = dateSuffix(entryDate); @@ -182,6 +183,7 @@ public class kelondroFlexSplitTable implements kelondroIndex { } public synchronized void addUnique(kelondroRow.Entry row, Date entryDate) throws IOException { + assert row.bytes().length <= this.rowdef.objectsize; String suffix = dateSuffix(entryDate); if (suffix == null) return; kelondroIndex table = (kelondroIndex) tables.get(suffix); diff --git a/source/de/anomic/kelondro/kelondroFlexTable.java b/source/de/anomic/kelondro/kelondroFlexTable.java index 04e42b890..547454fef 100644 --- a/source/de/anomic/kelondro/kelondroFlexTable.java +++ b/source/de/anomic/kelondro/kelondroFlexTable.java @@ -171,6 +171,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr public synchronized kelondroRow.Entry put(kelondroRow.Entry row) throws IOException { assert (row != null); assert (!(serverLog.allZero(row.getColBytes(0)))); + assert row.bytes().length <= this.rowdef.objectsize; int i = index.geti(row.getColBytes(0)); if (i < 0) { index.puti(row.getColBytes(0), super.add(row)); @@ -184,6 +185,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr } public synchronized void addUnique(kelondroRow.Entry row) throws IOException { + assert row.bytes().length <= this.rowdef.objectsize; index.addi(row.getColBytes(0), super.add(row)); } diff --git a/source/de/anomic/kelondro/kelondroFlexWidthArray.java b/source/de/anomic/kelondro/kelondroFlexWidthArray.java index 5c80c51bf..d16a55cda 100644 --- a/source/de/anomic/kelondro/kelondroFlexWidthArray.java +++ b/source/de/anomic/kelondro/kelondroFlexWidthArray.java @@ -160,6 +160,7 @@ public class kelondroFlexWidthArray implements kelondroArray { } public kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException { + assert rowentry.bytes().length <= this.rowdef.objectsize; int c = 0; kelondroRow.Entry e0, e1, p; p = rowdef.newEntry(); @@ -183,6 +184,7 @@ public class kelondroFlexWidthArray implements kelondroArray { } public int add(kelondroRow.Entry rowentry) throws IOException { + assert rowentry.bytes().length <= this.rowdef.objectsize; kelondroRow.Entry e; int index = -1; int lastcol; diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java index afd3916f7..9643db938 100644 --- a/source/de/anomic/kelondro/kelondroRecords.java +++ b/source/de/anomic/kelondro/kelondroRecords.java @@ -925,16 +925,22 @@ public class kelondroRecords { private final void assignRowdef(kelondroRow rowdef) { // overwrites a given rowdef // the new rowdef must be compatible - if (rowdef.columns() < ROW.columns()) + /* + if ((rowdef.columns() != ROW.columns()) && + ((rowdef.columns() + 1 != ROW.columns()) || + (rowdef.column(rowdef.columns() - 1).cellwidth() != (ROW.column(ROW.columns() - 1).cellwidth() + ROW.column(ROW.columns() - 2).cellwidth())))) throw new kelondroException(this.filename, "new rowdef '" + rowdef.toString() + "' is not compatible with old rowdef '" + ROW.toString() + "', they have a different number of columns"); - + */ // adopt encoder and cell type + /* kelondroColumn col; for (int i = 0; i < ROW.columns(); i++) { col = rowdef.column(i); ROW.column(i).setAttributes(col.nickname(), col.celltype(), col.encoder()); } + */ + this.ROW = rowdef; } protected final long seekpos(Handle handle) { diff --git a/source/de/anomic/kelondro/kelondroTree.java b/source/de/anomic/kelondro/kelondroTree.java index 3842979a4..997472810 100644 --- a/source/de/anomic/kelondro/kelondroTree.java +++ b/source/de/anomic/kelondro/kelondroTree.java @@ -379,6 +379,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { assert (newrow != null); assert (newrow.columns() == row().columns()); assert (!(serverLog.allZero(newrow.getColBytes(primarykey())))); + assert newrow.bytes().length <= super.ROW.objectsize; // Associates the specified value with the specified key in this map kelondroRow.Entry result = null; //writeLock.stay(2000, 1000); diff --git a/source/de/anomic/plasma/plasmaWordIndexAssortment.java b/source/de/anomic/plasma/plasmaWordIndexAssortment.java index e44aff388..756eb14d6 100644 --- a/source/de/anomic/plasma/plasmaWordIndexAssortment.java +++ b/source/de/anomic/plasma/plasmaWordIndexAssortment.java @@ -58,7 +58,7 @@ import java.util.Iterator; import de.anomic.index.indexContainer; import de.anomic.index.indexEntry; -import de.anomic.index.indexEntryAttribute; +import de.anomic.index.indexRAMCacheRI; import de.anomic.index.indexURLEntry; import de.anomic.kelondro.kelondroCache; import de.anomic.kelondro.kelondroColumn; @@ -71,12 +71,6 @@ public final class plasmaWordIndexAssortment { // environment constants private static final String assortmentFileName = "indexAssortment"; - public static final kelondroRow bufferStructureBasis = new kelondroRow( - "byte[] wordhash-" + indexEntryAttribute.wordHashLength + ", " + - "Cardinal occ-4 {b256}, " + - "Cardinal time-8 {b256}, " + - "byte[] urlhash-" + indexEntryAttribute.urlHashLength + ", " + - "byte[] urlattr-" + indexURLEntry.encodedByteArrayFormLength(false)); // class variables private File assortmentFile; @@ -93,19 +87,18 @@ public final class plasmaWordIndexAssortment { } private static kelondroRow bufferStructure(int assortmentCapacity) { - kelondroColumn[] structure = new kelondroColumn[3 + 2 * assortmentCapacity]; - structure[0] = bufferStructureBasis.column(0); - structure[1] = bufferStructureBasis.column(1); - structure[2] = bufferStructureBasis.column(2); + kelondroColumn[] structure = new kelondroColumn[3 + assortmentCapacity]; + structure[0] = indexRAMCacheRI.bufferStructureBasis.column(0); + structure[1] = indexRAMCacheRI.bufferStructureBasis.column(1); + structure[2] = indexRAMCacheRI.bufferStructureBasis.column(2); for (int i = 0; i < assortmentCapacity; i++) { - structure[3 + 2 * i] = bufferStructureBasis.column(3); - structure[4 + 2 * i] = bufferStructureBasis.column(4); + structure[3 + i] = indexRAMCacheRI.bufferStructureBasis.column(3); } return new kelondroRow(structure); } private static int assortmentCapacity(int rowsize) { - return (rowsize - bufferStructureBasis.width(0) - bufferStructureBasis.width(1) - bufferStructureBasis.width(2)) / (bufferStructureBasis.width(3) + bufferStructureBasis.width(4)); + return (rowsize - indexRAMCacheRI.bufferStructureBasis.width(0) - indexRAMCacheRI.bufferStructureBasis.width(1) - indexRAMCacheRI.bufferStructureBasis.width(2)) / indexRAMCacheRI.bufferStructureBasis.width(3); } public plasmaWordIndexAssortment(File storagePath, int assortmentLength, int bufferkb, long preloadTime, serverLog log) throws IOException { @@ -141,8 +134,7 @@ public final class plasmaWordIndexAssortment { indexEntry entry; for (int i = 0; i < assortmentLength; i++) { entry = (indexEntry) entries.next(); - row.setCol(3 + 2 * i, entry.urlHash().getBytes()); - row.setCol(4 + 2 * i, entry.toEncodedByteArrayForm(false)); + row.setCol(3 + i, entry.toKelondroEntry().bytes()); } kelondroRow.Entry oldrow = null; try { @@ -227,9 +219,7 @@ public final class plasmaWordIndexAssortment { indexContainer container = new indexContainer(wordHash); int al = assortmentCapacity(row.objectsize()); for (int i = 0; i < al; i++) { - container.add( - new indexEntry[] { new indexURLEntry( - new String(row.getColBytes(3 + 2 * i)), new String(row.getColBytes(4 + 2 * i))) }, updateTime); + container.add(new indexEntry[] { new indexURLEntry(row.getColBytes(3 + i)) }, updateTime); } return container; } diff --git a/source/de/anomic/plasma/plasmaWordIndexFile.java b/source/de/anomic/plasma/plasmaWordIndexFile.java index 286382fd6..4eb5785c0 100644 --- a/source/de/anomic/plasma/plasmaWordIndexFile.java +++ b/source/de/anomic/plasma/plasmaWordIndexFile.java @@ -91,7 +91,7 @@ public final class plasmaWordIndexFile { long cacheSize = theLocation.length(); if (cacheSize > 1048576) cacheSize = 1048576; return kelondroTree.open(theLocation, cacheSize, 0, - new kelondroRow("byte[] urlhash-" + indexURL.urlHashLength + ", byte[] ba-" + indexURLEntry.encodedByteArrayFormLength(false))); + new kelondroRow("byte[] urlhash-" + indexURL.urlHashLength + ", byte[] ba-" + (indexURLEntry.urlEntryRow.objectsize() - indexURL.urlHashLength))); } public static File wordHash2path(File databaseRoot, String hash) { @@ -148,7 +148,7 @@ public final class plasmaWordIndexFile { if ((oldEntry != null) && (entry.isOlder(oldEntry))) { // A more recent Entry is already in this entity return false; } - return (theIndex.put(entry.urlHash().getBytes(), entry.toEncodedByteArrayForm(false)) == null); + return (theIndex.put(entry.toKelondroEntry()) == null); } public int addEntries(indexContainer container) throws IOException {