boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- more asserts

Browse Source 
- simplified indexURLEntry

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2891 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent 8c9bc7e341
commit 46a712e195
10 changed files with 40 additions and 47 deletions
Show Stats Download Patch File Download Diff File

1
source/de/anomic/index/indexEntry.java
Unescape View File

@ -31,7 +31,6 @@ import de.anomic.kelondro.kelondroRow;
public interface indexEntry {
public Object clone();
public byte[] toEncodedByteArrayForm(boolean includeUrlHash); // shall be replaced by toKelondroEntry()
public String toPropertyForm(boolean displayFormat);
public kelondroRow.Entry toKelondroEntry();

23
source/de/anomic/index/indexRAMCacheRI.java
Unescape View File

@ -38,15 +38,19 @@ import de.anomic.kelondro.kelondroFixedWidthArray;
import de.anomic.kelondro.kelondroMScoreCluster;
import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.kelondro.kelondroRow;
import de.anomic.plasma.plasmaWordIndexAssortment;
import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacySeedDB;
public final class indexRAMCacheRI implements indexRI {
// environment constants
public static final long wCacheMaxAge = 1000 * 60 * 30; // milliseconds; 30 minutes
public static final long wCacheMaxAge = 1000 * 60 * 30; // milliseconds; 30 minutes
public static final kelondroRow bufferStructureBasis = new kelondroRow(
"byte[] wordhash-" + indexEntryAttribute.wordHashLength + ", " +
"Cardinal occ-4 {b256}, " +
"Cardinal time-8 {b256}, " +
"byte[] urlprops-" + indexURLEntry.urlEntryRow.objectsize());
// class variables
private final File databaseRoot;
protected final TreeMap cache; // wordhash-container
@ -64,7 +68,7 @@ public final class indexRAMCacheRI implements indexRI {
maxKey = ""; for (int i = 0; i < yacySeedDB.commonHashLength; i++) maxKey += 'z';
//minKey = ""; for (int i = 0; i < yacySeedDB.commonHashLength; i++) maxKey += '-';
}
public indexRAMCacheRI(File databaseRoot, int wCacheReferenceLimitInit, String dumpname, serverLog log) {
// creates a new index cache
@ -98,7 +102,7 @@ public final class indexRAMCacheRI implements indexRI {
File indexDumpFile = new File(databaseRoot, indexArrayFileName);
if (indexDumpFile.exists()) indexDumpFile.delete();
kelondroFixedWidthArray dumpArray = null;
dumpArray = new kelondroFixedWidthArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0);
dumpArray = new kelondroFixedWidthArray(indexDumpFile, bufferStructureBasis, 0);
long startTime = System.currentTimeMillis();
long messageTime = System.currentTimeMillis() + 5000;
long wordsPerSecond = 0, wordcount = 0, urlcount = 0;
@ -127,8 +131,7 @@ public final class indexRAMCacheRI implements indexRI {
row.setCol(0, wordHash.getBytes());
row.setCol(1, kelondroNaturalOrder.encodeLong(container.size(), 4));
row.setCol(2, kelondroNaturalOrder.encodeLong(updateTime, 8));
row.setCol(3, iEntry.urlHash().getBytes());
row.setCol(4, iEntry.toEncodedByteArrayForm(false));
row.setCol(3, iEntry.toKelondroEntry().bytes());
dumpArray.set((int) urlcount++, row);
}
}
@ -152,7 +155,7 @@ public final class indexRAMCacheRI implements indexRI {
private long restore() throws IOException {
File indexDumpFile = new File(databaseRoot, indexArrayFileName);
if (!(indexDumpFile.exists())) return 0;
kelondroFixedWidthArray dumpArray = new kelondroFixedWidthArray(indexDumpFile, plasmaWordIndexAssortment.bufferStructureBasis, 0);
kelondroFixedWidthArray dumpArray = new kelondroFixedWidthArray(indexDumpFile, bufferStructureBasis, 0);
log.logConfig("restore array dump of index cache '" + indexArrayFileName + "', " + dumpArray.size() + " word/URL relations");
long startTime = System.currentTimeMillis();
long messageTime = System.currentTimeMillis() + 5000;
@ -168,10 +171,10 @@ public final class indexRAMCacheRI implements indexRI {
while (i.hasNext()) {
// get out one entry
row = (kelondroRow.Entry) i.next();
if ((row == null) || (row.empty(0)) || (row.empty(3)) || (row.empty(4))) continue;
if ((row == null) || (row.empty(0)) || (row.empty(3))) continue;
wordHash = row.getColString(0, "UTF-8");
//creationTime = kelondroRecords.bytes2long(row[2]);
wordEntry = new indexURLEntry(row.getColString(3, null), row.getColString(4, null));
wordEntry = new indexURLEntry(row.getColBytes(3));
// store to cache
addEntry(wordHash, wordEntry, startTime, false);
urlCount++;

14
source/de/anomic/index/indexURLEntry.java
Unescape View File

@ -135,19 +135,7 @@ public class indexURLEntry implements Cloneable, indexEntry {
return new indexURLEntry(b);
}
public static int encodedByteArrayFormLength(boolean includingHeader) {
// the size of the index entry attributes when encoded to string
return (includingHeader) ? urlEntryRow.objectsize() : urlEntryRow.objectsize() - indexURL.urlHashLength;
}
public byte[] toEncodedByteArrayForm(boolean includeHash) {
if (includeHash) return entry.bytes();
byte[] b = new byte[urlEntryRow.objectsize() - indexURL.urlHashLength];
System.arraycopy(entry.bytes(), indexURL.urlHashLength, b, 0, b.length);
return b;
}
public String toPropertyForm(boolean displayFormat) {
public String toPropertyForm(boolean displayFormat) {
return entry.toPropertyForm(true, displayFormat, displayFormat);
}

2
source/de/anomic/kelondro/kelondroFlexSplitTable.java
Unescape View File

@ -151,6 +151,7 @@ public class kelondroFlexSplitTable implements kelondroIndex {
}
public synchronized kelondroRow.Entry put(kelondroRow.Entry row, Date entryDate) throws IOException {
assert row.bytes().length <= this.rowdef.objectsize;
Object[] keeper = keeperOf(row.getColBytes(0));
if (keeper != null) return ((kelondroIndex) keeper[0]).put(row);
String suffix = dateSuffix(entryDate);
@ -182,6 +183,7 @@ public class kelondroFlexSplitTable implements kelondroIndex {
}
public synchronized void addUnique(kelondroRow.Entry row, Date entryDate) throws IOException {
assert row.bytes().length <= this.rowdef.objectsize;
String suffix = dateSuffix(entryDate);
if (suffix == null) return;
kelondroIndex table = (kelondroIndex) tables.get(suffix);

2
source/de/anomic/kelondro/kelondroFlexTable.java
Unescape View File

@ -171,6 +171,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
public synchronized kelondroRow.Entry put(kelondroRow.Entry row) throws IOException {
assert (row != null);
assert (!(serverLog.allZero(row.getColBytes(0))));
assert row.bytes().length <= this.rowdef.objectsize;
int i = index.geti(row.getColBytes(0));
if (i < 0) {
index.puti(row.getColBytes(0), super.add(row));
@ -184,6 +185,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
}
public synchronized void addUnique(kelondroRow.Entry row) throws IOException {
assert row.bytes().length <= this.rowdef.objectsize;
index.addi(row.getColBytes(0), super.add(row));
}

2
source/de/anomic/kelondro/kelondroFlexWidthArray.java
Unescape View File

@ -160,6 +160,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
}
public kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException {
assert rowentry.bytes().length <= this.rowdef.objectsize;
int c = 0;
kelondroRow.Entry e0, e1, p;
p = rowdef.newEntry();
@ -183,6 +184,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
}
public int add(kelondroRow.Entry rowentry) throws IOException {
assert rowentry.bytes().length <= this.rowdef.objectsize;
kelondroRow.Entry e;
int index = -1;
int lastcol;

10
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -925,16 +925,22 @@ public class kelondroRecords {
private final void assignRowdef(kelondroRow rowdef) {
// overwrites a given rowdef
// the new rowdef must be compatible
if (rowdef.columns() < ROW.columns())
/*
if ((rowdef.columns() != ROW.columns()) &&
((rowdef.columns() + 1 != ROW.columns()) ||
(rowdef.column(rowdef.columns() - 1).cellwidth() != (ROW.column(ROW.columns() - 1).cellwidth() + ROW.column(ROW.columns() - 2).cellwidth()))))
throw new kelondroException(this.filename,
"new rowdef '" + rowdef.toString() + "' is not compatible with old rowdef '" + ROW.toString() + "', they have a different number of columns");
*/
// adopt encoder and cell type
/*
kelondroColumn col;
for (int i = 0; i < ROW.columns(); i++) {
col = rowdef.column(i);
ROW.column(i).setAttributes(col.nickname(), col.celltype(), col.encoder());
}
*/
this.ROW = rowdef;
}
protected final long seekpos(Handle handle) {

1
source/de/anomic/kelondro/kelondroTree.java
Unescape View File

@ -379,6 +379,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
assert (newrow != null);
assert (newrow.columns() == row().columns());
assert (!(serverLog.allZero(newrow.getColBytes(primarykey()))));
assert newrow.bytes().length <= super.ROW.objectsize;
// Associates the specified value with the specified key in this map
kelondroRow.Entry result = null;
//writeLock.stay(2000, 1000);

28
source/de/anomic/plasma/plasmaWordIndexAssortment.java
Unescape View File

@ -58,7 +58,7 @@ import java.util.Iterator;
import de.anomic.index.indexContainer;
import de.anomic.index.indexEntry;
import de.anomic.index.indexEntryAttribute;
import de.anomic.index.indexRAMCacheRI;
import de.anomic.index.indexURLEntry;
import de.anomic.kelondro.kelondroCache;
import de.anomic.kelondro.kelondroColumn;
@ -71,12 +71,6 @@ public final class plasmaWordIndexAssortment {
// environment constants
private static final String assortmentFileName = "indexAssortment";
public static final kelondroRow bufferStructureBasis = new kelondroRow(
"byte[] wordhash-" + indexEntryAttribute.wordHashLength + ", " +
"Cardinal occ-4 {b256}, " +
"Cardinal time-8 {b256}, " +
"byte[] urlhash-" + indexEntryAttribute.urlHashLength + ", " +
"byte[] urlattr-" + indexURLEntry.encodedByteArrayFormLength(false));
// class variables
private File assortmentFile;
@ -93,19 +87,18 @@ public final class plasmaWordIndexAssortment {
}
private static kelondroRow bufferStructure(int assortmentCapacity) {
kelondroColumn[] structure = new kelondroColumn[3 + 2 * assortmentCapacity];
structure[0] = bufferStructureBasis.column(0);
structure[1] = bufferStructureBasis.column(1);
structure[2] = bufferStructureBasis.column(2);
kelondroColumn[] structure = new kelondroColumn[3 + assortmentCapacity];
structure[0] = indexRAMCacheRI.bufferStructureBasis.column(0);
structure[1] = indexRAMCacheRI.bufferStructureBasis.column(1);
structure[2] = indexRAMCacheRI.bufferStructureBasis.column(2);
for (int i = 0; i < assortmentCapacity; i++) {
structure[3 + 2 * i] = bufferStructureBasis.column(3);
structure[4 + 2 * i] = bufferStructureBasis.column(4);
structure[3 + i] = indexRAMCacheRI.bufferStructureBasis.column(3);
}
return new kelondroRow(structure);
}
private static int assortmentCapacity(int rowsize) {
return (rowsize - bufferStructureBasis.width(0) - bufferStructureBasis.width(1) - bufferStructureBasis.width(2)) / (bufferStructureBasis.width(3) + bufferStructureBasis.width(4));
return (rowsize - indexRAMCacheRI.bufferStructureBasis.width(0) - indexRAMCacheRI.bufferStructureBasis.width(1) - indexRAMCacheRI.bufferStructureBasis.width(2)) / indexRAMCacheRI.bufferStructureBasis.width(3);
}
public plasmaWordIndexAssortment(File storagePath, int assortmentLength, int bufferkb, long preloadTime, serverLog log) throws IOException {
@ -141,8 +134,7 @@ public final class plasmaWordIndexAssortment {
indexEntry entry;
for (int i = 0; i < assortmentLength; i++) {
entry = (indexEntry) entries.next();
row.setCol(3 + 2 * i, entry.urlHash().getBytes());
row.setCol(4 + 2 * i, entry.toEncodedByteArrayForm(false));
row.setCol(3 + i, entry.toKelondroEntry().bytes());
}
kelondroRow.Entry oldrow = null;
try {
@ -227,9 +219,7 @@ public final class plasmaWordIndexAssortment {
indexContainer container = new indexContainer(wordHash);
int al = assortmentCapacity(row.objectsize());
for (int i = 0; i < al; i++) {
container.add(
new indexEntry[] { new indexURLEntry(
new String(row.getColBytes(3 + 2 * i)), new String(row.getColBytes(4 + 2 * i))) }, updateTime);
container.add(new indexEntry[] { new indexURLEntry(row.getColBytes(3 + i)) }, updateTime);
}
return container;
}

4
source/de/anomic/plasma/plasmaWordIndexFile.java
Unescape View File

@ -91,7 +91,7 @@ public final class plasmaWordIndexFile {
long cacheSize = theLocation.length();
if (cacheSize > 1048576) cacheSize = 1048576;
return kelondroTree.open(theLocation, cacheSize, 0,
new kelondroRow("byte[] urlhash-" + indexURL.urlHashLength + ", byte[] ba-" + indexURLEntry.encodedByteArrayFormLength(false)));
new kelondroRow("byte[] urlhash-" + indexURL.urlHashLength + ", byte[] ba-" + (indexURLEntry.urlEntryRow.objectsize() - indexURL.urlHashLength)));
}
public static File wordHash2path(File databaseRoot, String hash) {
@ -148,7 +148,7 @@ public final class plasmaWordIndexFile {
if ((oldEntry != null) && (entry.isOlder(oldEntry))) { // A more recent Entry is already in this entity
return false;
}
return (theIndex.put(entry.urlHash().getBytes(), entry.toEncodedByteArrayForm(false)) == null);
return (theIndex.put(entry.toKelondroEntry()) == null);
}
public int addEntries(indexContainer container) throws IOException {

Write Preview
Loading…
Cancel
Save