boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

bugfixing in collection methods

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2882 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent 62ad1476ac
commit 985fd807cc
7 changed files with 157 additions and 139 deletions
Show Stats Download Patch File Download Diff File

153
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -48,10 +48,10 @@ public class kelondroCollectionIndex {
private static final int idx_col_key = 0; // the index
private static final int idx_col_chunksize = 1; // chunksize (number of bytes in a single chunk, needed for migration option)
private static final int idx_col_chunkcount = 2; // chunkcount (number of chunks in this collection) needed to identify array file that has the chunks
private static final int idx_col_chunkcount = 2; // chunkcount (number of chunks in this collection)
private static final int idx_col_clusteridx = 3; // selector for right cluster file, must be >= arrayIndex(chunkcount)
private static final int idx_col_flags = 4; // flags (for future use)
private static final int idx_col_indexpos = 5; // indexpos (position in index file)
private static final int idx_col_indexpos = 5; // indexpos (position in array file)
private static final int idx_col_lastread = 6; // a time stamp, update time in days since 1.1.2000
private static final int idx_col_lastwrote = 7; // a time stamp, update time in days since 1.1.2000
@ -139,13 +139,17 @@ public class kelondroCollectionIndex {
if ((index != null) && (indexGeneration)) {
// loop over all elements in array and create index entry for each row
kelondroRow.Entry aentry, ientry;
kelondroRow.EntryIndex aentry;
kelondroRow.Entry ientry;
Iterator ei = array.contentRows(-1);
byte[] key;
long start = System.currentTimeMillis();
long lastlog = start;
for (int j = 0; j < array.USAGE.allCount(); j++) {
aentry = array.get(j);
int count = 0;
while (ei.hasNext()) {
aentry = (kelondroRow.EntryIndex) ei.next();
key = aentry.getColBytes(0);
assert (key != null);
if (key == null) continue; // skip deleted entries
kelondroRowSet indexrows = new kelondroRowSet(this.playloadrow, aentry.getColBytes(1));
ientry = irow.newEntry();
@ -154,14 +158,15 @@ public class kelondroCollectionIndex {
ientry.setCol(idx_col_chunkcount, indexrows.size());
ientry.setCol(idx_col_clusteridx, (byte) partitionNumber);
ientry.setCol(idx_col_flags, (byte) 0);
ientry.setCol(idx_col_indexpos, j);
ientry.setCol(idx_col_indexpos, aentry.index());
ientry.setCol(idx_col_lastread, t);
ientry.setCol(idx_col_lastwrote, t);
index.addUnique(ientry);
count++;
// write a log
if (System.currentTimeMillis() - lastlog > 30000) {
serverLog.logFine("STARTUP", "created " + j + " RWI index entries. " + (((System.currentTimeMillis() - start) * (array.USAGE.allCount() - j) / ((j == 0) ? 1 : j)) / 60000) + " minutes remaining for this array");
serverLog.logFine("STARTUP", "created " + count + " RWI index entries. " + (((System.currentTimeMillis() - start) * (array.USAGE.allCount() - count) / count) / 60000) + " minutes remaining for this array");
lastlog = System.currentTimeMillis();
}
}
@ -235,42 +240,56 @@ public class kelondroCollectionIndex {
}
}
public int size() throws IOException {
public synchronized int size() throws IOException {
return index.size();
}
public void put(byte[] key, kelondroRowCollection collection) throws IOException, kelondroOutOfLimitsException {
public synchronized void put(byte[] key, kelondroRowCollection collection) throws IOException, kelondroOutOfLimitsException {
// this replaces an old collection by a new one
// this method is not approriate to extend an existing collection with another collection
putmergeremove(key, collection, false, null, false);
putmergeremove(key, collection, false, null);
}
public void merge(byte[] key, kelondroRowCollection collection) throws IOException, kelondroOutOfLimitsException {
putmergeremove(key, collection, true, null, false);
public synchronized void merge(byte[] key, kelondroRowCollection collection) throws IOException, kelondroOutOfLimitsException {
putmergeremove(key, collection, true, null);
}
public int remove(byte[] key, Set removekeys, boolean deletecomplete) throws IOException, kelondroOutOfLimitsException {
return putmergeremove(key, null, false, removekeys, deletecomplete);
public synchronized int remove(byte[] key, Set removekeys, boolean deletecomplete) throws IOException, kelondroOutOfLimitsException {
return putmergeremove(key, null, false, removekeys);
}
private int putmergeremove(byte[] key, kelondroRowCollection collection, boolean merge, Set removekeys, boolean deletecomplete) throws IOException, kelondroOutOfLimitsException {
private int putmergeremove(byte[] key, kelondroRowCollection collection, boolean merge, Set removekeys) throws IOException, kelondroOutOfLimitsException {
//if (collection.size() > maxChunks) throw new kelondroOutOfLimitsException(maxChunks, collection.size());
if ((!merge) && (removekeys != null) && (collection != null) && (collection.size() == 0)) {
// this is not a replacement, it is a deletion
delete(key);
return 0;
}
synchronized (index) {
// first find an old entry, if one exists
kelondroRow.Entry indexrow = index.get(key);
if (indexrow == null) {
if ((collection != null) && (collection.size() > 0)) {
// the collection is new
overwrite(key, collection, arrayIndex(collection.size()), index.row().newEntry());
int newPartitionNumber = arrayIndex(collection.size());
indexrow = index.row().newEntry();
kelondroFixedWidthArray array = getArray(newPartitionNumber, 0, this.playloadrow.objectsize());
// define row
kelondroRow.Entry arrayEntry = array.row().newEntry();
arrayEntry.setCol(0, key);
arrayEntry.setCol(1, collection.exportCollection());
// write a new entry in this array
int newRowNumber = array.add(arrayEntry);
// store the new row number in the index
indexrow.setCol(idx_col_key, key);
indexrow.setCol(idx_col_chunksize, this.playloadrow.objectsize());
indexrow.setCol(idx_col_chunkcount, collection.size());
indexrow.setCol(idx_col_clusteridx, (byte) newPartitionNumber);
indexrow.setCol(idx_col_flags, (byte) 0);
indexrow.setCol(idx_col_indexpos, (long) newRowNumber);
indexrow.setCol(idx_col_lastread, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
indexrow.setCol(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
index.addUnique(indexrow);
}
return 0;
}
@ -286,7 +305,7 @@ public class kelondroCollectionIndex {
if (merge) {
// load the old collection and join it
kelondroRowSet oldcollection = getwithparams(indexrow, oldchunksize, oldchunkcount, oldPartitionNumber, oldrownumber, oldSerialNumber, false, false);
kelondroRowSet oldcollection = getwithparams(indexrow, oldchunksize, oldchunkcount, oldPartitionNumber, oldrownumber, oldSerialNumber, false);
// join with new collection
oldcollection.addAll(collection);
@ -298,7 +317,7 @@ public class kelondroCollectionIndex {
int removed = 0;
if (removekeys != null) {
// load the old collection and remove keys
kelondroRowSet oldcollection = getwithparams(indexrow, oldchunksize, oldchunkcount, oldPartitionNumber, oldrownumber, oldSerialNumber, false, false);
kelondroRowSet oldcollection = getwithparams(indexrow, oldchunksize, oldchunkcount, oldPartitionNumber, oldrownumber, oldSerialNumber, false);
// remove the keys from the set
Iterator i = removekeys.iterator();
@ -309,20 +328,15 @@ public class kelondroCollectionIndex {
if ((k instanceof String) && (oldcollection.remove(((String) k).getBytes()) != null)) removed++;
}
oldcollection.shape();
oldcollection.trim();
collection = oldcollection;
}
if (collection.size() == 0) {
if (deletecomplete) {
// delete the index entry and the array
kelondroFixedWidthArray array = getArray(oldPartitionNumber, oldSerialNumber, oldchunksize);
array.remove(oldrownumber);
index.remove(key);
} else {
// update the index entry
indexrow.setCol(idx_col_chunkcount, 0);
indexrow.setCol(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
index.put(indexrow);
}
return removed;
}
@ -347,7 +361,6 @@ public class kelondroCollectionIndex {
// update the index entry
indexrow.setCol(idx_col_chunkcount, collection.size());
indexrow.setCol(idx_col_clusteridx, (byte) oldPartitionNumber);
indexrow.setCol(idx_col_flags, (byte) 0);
indexrow.setCol(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
index.put(indexrow);
} else {
@ -359,18 +372,7 @@ public class kelondroCollectionIndex {
array.remove(oldrownumber);
// write a new entry in the other array
overwrite(key, collection, newPartitionNumber, indexrow);
}
return removed;
}
}
private void overwrite(byte[] key, kelondroRowCollection collection, int targetpartition, kelondroRow.Entry indexEntry) throws IOException {
// helper method, should not be called directly and only within a synchronized(index) environment
// simply store a collection without check if the collection existed before
// find array file
kelondroFixedWidthArray array = getArray(targetpartition, 0, this.playloadrow.objectsize());
array = getArray(newPartitionNumber, 0, this.playloadrow.objectsize());
// define row
kelondroRow.Entry arrayEntry = array.row().newEntry();
@ -381,46 +383,41 @@ public class kelondroCollectionIndex {
int newRowNumber = array.add(arrayEntry);
// store the new row number in the index
indexEntry.setCol(idx_col_key, key);
indexEntry.setCol(idx_col_chunksize, this.playloadrow.objectsize());
indexEntry.setCol(idx_col_chunkcount, collection.size());
indexEntry.setCol(idx_col_clusteridx, (byte) targetpartition);
indexEntry.setCol(idx_col_flags, (byte) 0);
indexEntry.setCol(idx_col_indexpos, (long) newRowNumber);
indexEntry.setCol(idx_col_lastread, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
indexEntry.setCol(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
index.put(indexEntry);
indexrow.setCol(idx_col_key, key);
indexrow.setCol(idx_col_chunkcount, collection.size());
indexrow.setCol(idx_col_clusteridx, (byte) newPartitionNumber);
indexrow.setCol(idx_col_indexpos, (long) newRowNumber);
indexrow.setCol(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
index.put(indexrow);
}
return removed;
}
public int indexSize(byte[] key) throws IOException {
synchronized (index) {
public synchronized int indexSize(byte[] key) throws IOException {
kelondroRow.Entry indexrow = index.get(key);
if (indexrow == null) return 0;
return (int) indexrow.getColLong(idx_col_chunkcount);
}
}
public kelondroRowSet get(byte[] key, boolean deleteIfEmpty) throws IOException {
public synchronized kelondroRowSet get(byte[] key, boolean deleteIfEmpty) throws IOException {
// find an entry, if one exists
synchronized (index) {
kelondroRow.Entry indexrow = index.get(key);
if (indexrow == null) return null;
return getdelete(indexrow, false, deleteIfEmpty);
}
kelondroRowSet col = getdelete(indexrow, false);
assert (col != null);
return col;
}
public kelondroRowSet delete(byte[] key) throws IOException {
public synchronized kelondroRowSet delete(byte[] key) throws IOException {
// find an entry, if one exists
synchronized (index) {
kelondroRow.Entry indexrow = index.get(key);
kelondroRow.Entry indexrow = index.remove(key);
if (indexrow == null) return null;
kelondroRowSet removedCollection = getdelete(indexrow, true, false);
index.remove(key);
kelondroRowSet removedCollection = getdelete(indexrow, true);
assert (removedCollection != null);
return removedCollection;
}
}
protected kelondroRowSet getdelete(kelondroRow.Entry indexrow, boolean remove, boolean deleteIfEmpty) throws IOException {
protected kelondroRowSet getdelete(kelondroRow.Entry indexrow, boolean remove) throws IOException {
// call this only within a synchronized(index) environment
// read values
@ -431,10 +428,10 @@ public class kelondroCollectionIndex {
assert(partitionnumber >= arrayIndex(chunkcount));
int serialnumber = 0;
return getwithparams(indexrow, chunksize, chunkcount, partitionnumber, rownumber, serialnumber, remove, deleteIfEmpty);
return getwithparams(indexrow, chunksize, chunkcount, partitionnumber, rownumber, serialnumber, remove);
}
private kelondroRowSet getwithparams(kelondroRow.Entry indexrow, int chunksize, int chunkcount, int clusteridx, int rownumber, int serialnumber, boolean remove, boolean deleteIfEmpty) throws IOException {
private kelondroRowSet getwithparams(kelondroRow.Entry indexrow, int chunksize, int chunkcount, int clusteridx, int rownumber, int serialnumber, boolean remove) throws IOException {
// open array entry
kelondroFixedWidthArray array = getArray(clusteridx, serialnumber, chunksize);
kelondroRow.Entry arrayrow = array.get(rownumber);
@ -442,9 +439,10 @@ public class kelondroCollectionIndex {
// read the row and define a collection
kelondroRowSet collection = new kelondroRowSet(this.playloadrow, arrayrow.getColBytes(1)); // FIXME: this does not yet work with different rowdef in case of several rowdef.objectsize()
if (index.order().compare(arrayrow.getColBytes(0), indexrow.getColBytes(idx_col_key)) != 0) {
byte[] key = indexrow.getColBytes(idx_col_key);
if (index.order().compare(arrayrow.getColBytes(0), key) != 0) {
// check if we got the right row; this row is wrong. Fix it:
index.remove(indexrow.getColBytes(idx_col_key)); // the wrong row cannot be fixed
index.remove(key); // the wrong row cannot be fixed
// store the row number in the index; this may be a double-entry, but better than nothing
kelondroRow.Entry indexEntry = index.row().newEntry();
indexEntry.setCol(idx_col_key, arrayrow.getColBytes(0));
@ -465,11 +463,11 @@ public class kelondroCollectionIndex {
index.put(indexrow);
array.logFailure("INCONSISTENCY in " + arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString() + ": array has different chunkcount than index: index = " + chunkcount + ", array = " + chunkcountInArray + "; the index has been auto-fixed");
}
if ((remove) || ((collection.size() == 0) && (deleteIfEmpty))) array.remove(rownumber);
if (remove) array.remove(rownumber); // index is removed in calling method
return collection;
}
public Iterator keycollections(byte[] startKey, boolean rot) {
public synchronized Iterator keycollections(byte[] startKey, boolean rot) {
// returns an iteration of {byte[], kelondroRowSet} Objects
try {
return new keycollectionIterator(startKey, rot);
@ -494,9 +492,10 @@ public class kelondroCollectionIndex {
public Object next() {
kelondroRow.Entry indexrow = (kelondroRow.Entry) indexRowIterator.next();
assert (indexrow != null);
if (indexrow == null) return null;
try {
return new Object[]{indexrow.getColBytes(0), getdelete(indexrow, false, false)};
return new Object[]{indexrow.getColBytes(0), getdelete(indexrow, false)};
} catch (IOException e) {
e.printStackTrace();
return null;
@ -509,15 +508,13 @@ public class kelondroCollectionIndex {
}
public void close() throws IOException {
synchronized (index) {
public synchronized void close() throws IOException {
this.index.close();
Iterator i = arrays.values().iterator();
while (i.hasNext()) {
((kelondroFixedWidthArray) i.next()).close();
}
}
}
public static void main(String[] args) {

1
source/de/anomic/kelondro/kelondroMap.java
Unescape View File

@ -380,6 +380,7 @@ public class kelondroMap {
}
try {
final Map map = get(nextKey);
//assert (map != null) : "nextKey = " + nextKey;
if (map == null) throw new kelondroException("no more elements available");
map.put("key", nextKey);
return map;

7
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -595,6 +595,7 @@ public class kelondroRecords {
protected Node(Handle handle, byte[] bulkchunk, int offset) {
// this initializer is used to create nodes from bulk-read byte arrays
this.handle = handle;
assert (bulkchunk.length >= offset + headchunksize) : "bulkchunk.length = " + bulkchunk.length + ", offset = " + offset + ", headchunksize = " + headchunksize;
// create empty chunks
this.headChunk = new byte[headchunksize];
@ -1029,7 +1030,8 @@ public class kelondroRecords {
public Object next() {
try {
return row().newEntry(((Node) nodeIterator.next()).getValueRow());
Node n = (Node) nodeIterator.next();
return row().newEntry(n.getValueRow(), n.handle.index);
} catch (IOException e) {
throw new kelondroException(filename, e.getMessage());
}
@ -1121,7 +1123,7 @@ public class kelondroRecords {
while ((markedDeleted.contains(pos)) && (pos.index < USAGE.allCount())) pos.index++;
// initialize bulk
bulksize = Math.min(65536 / recordsize, USAGE.allCount());
bulksize = Math.max(1, Math.min(65536 / recordsize, USAGE.allCount()));
bulkstart = -bulksize;
bulk = new byte[bulksize * recordsize];
next = (hasNext0()) ? next0() : null;
@ -1148,6 +1150,7 @@ public class kelondroRecords {
byte[] key = nn.getKey();
if ((key == null) ||
((key.length == 1) && (key[0] == (byte) 0x80)) || // the NUL pointer ('lost' chain terminator)
((key.length > 3) && (key[0] == (byte) 0x80) && (key[1] == 0) && (key[2] == 0) && (key[3] == 0)) ||
((key.length > 0) && (key[0] == 0)) // a 'lost' pointer within a deleted-chain
) {
// this is a deleted node; probably not commited with dispose

37
source/de/anomic/kelondro/kelondroRow.java
Unescape View File

@ -128,6 +128,11 @@ public class kelondroRow {
return new Entry(rowinstance);
}
public EntryIndex newEntry(byte[] rowinstance, int index) {
if (rowinstance == null) return null;
return new EntryIndex(rowinstance, index);
}
public Entry newEntry(byte[] rowinstance, int start, int length) {
if (rowinstance == null) return null;
return new Entry(rowinstance, start, length);
@ -142,12 +147,7 @@ public class kelondroRow {
if (external == null) return null;
return new Entry(external);
}
/*
public Entry newEntry(Properties prop) {
if (prop == null) return null;
return new Entry(prop);
}
*/
public class Entry implements Comparable {
private byte[] rowinstance;
@ -207,19 +207,7 @@ public class kelondroRow {
}
}
}
/*
public Entry(Properties prop) {
// parse external form
if (nickref == null) genNickRef();
rowinstance = new byte[objectsize];
Iterator i = prop.entrySet().iterator();
Map.Entry entry;
while (i.hasNext()) {
entry = (Map.Entry) i.next();
setCol(((String) entry.getKey()).trim(), ((String) entry.getValue()).trim().getBytes());
}
}
*/
public int compareTo(Object o) {
if (o instanceof Entry) {
return kelondroNaturalOrder.naturalOrder.compare(this.rowinstance, ((Entry) o).rowinstance);
@ -447,6 +435,17 @@ public class kelondroRow {
}
public final class EntryIndex extends Entry {
private int index;
public EntryIndex(byte[] row, int i) {
super(row);
this.index = i;
}
public int index() {
return index;
}
}
public final static void long2bytes(long x, byte[] b, int offset, int length) {
for (int i = length - 1; i >= 0; i--) {
b[offset + i] = (byte) (x & 0XFF);

6
source/de/anomic/kelondro/kelondroRowCollection.java
Unescape View File

@ -202,7 +202,7 @@ public class kelondroRowCollection {
}
public void addUnique(kelondroRow.Entry row) {
add(row.bytes(), 0, row.bytes().length);
addUnique(row.bytes(), 0, row.bytes().length);
}
public void addUnique(kelondroRow.Entry row, Date entryDate) {
@ -210,10 +210,10 @@ public class kelondroRowCollection {
}
public void add(byte[] a) {
add(a, 0, a.length);
addUnique(a, 0, a.length);
}
private final void add(byte[] a, int astart, int alength) {
private final void addUnique(byte[] a, int astart, int alength) {
assert (a != null);
assert (astart >= 0) && (astart < a.length) : " astart = " + a;
assert (!(serverLog.allZero(a, astart, alength))) : "a = " + serverLog.arrayList(a, astart, alength);

21
source/de/anomic/kelondro/kelondroRowSet.java
Unescape View File

@ -24,7 +24,6 @@
package de.anomic.kelondro;
import java.io.IOException;
import java.util.Date;
import java.util.Iterator;
import java.util.Random;
@ -83,7 +82,23 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
return entry;
}
public kelondroRow.Entry put(kelondroRow.Entry row, Date entryDate) throws IOException {
public void addUnique(kelondroRow.Entry row) {
if (removeMarker.size() == 0) {
super.addUnique(row);
} else {
this.put(row);
}
}
public void addUnique(kelondroRow.Entry row, Date entryDate) {
if (removeMarker.size() == 0) {
super.addUnique(row, entryDate);
} else {
this.put(row, entryDate);
}
}
public kelondroRow.Entry put(kelondroRow.Entry row, Date entryDate) {
return put(row);
}
@ -100,7 +115,7 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
set(index, entry);
removeMarker.remove(new Integer(index));
} else if (index < 0) {
addUnique(entry);
super.addUnique(entry);
} else {
oldentry = get(index);
set(index, entry);

9
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -854,12 +854,14 @@ public final class yacySeedDB {
it = (firstKey == null) ? database.maps(up, rot) : database.maps(up, rot, firstKey);
nextSeed = internalNext();
} catch (IOException e) {
yacyCore.log.logFine("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
e.printStackTrace();
yacyCore.log.logSevere("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
it = null;
} catch (kelondroException e) {
yacyCore.log.logFine("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
e.printStackTrace();
yacyCore.log.logSevere("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
it = null;
@ -872,7 +874,8 @@ public final class yacySeedDB {
it = database.maps(up, field);
nextSeed = internalNext();
} catch (kelondroException e) {
yacyCore.log.logFine("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
e.printStackTrace();
yacyCore.log.logSevere("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
if (database == seedPotentialDB) seedPotentialDB = resetSeedTable(seedPotentialDB, seedPotentialDBFile);

Write Preview
Loading…
Cancel
Save