boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

bugfixing in collection methods

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2882 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent 62ad1476ac
commit 985fd807cc
7 changed files with 157 additions and 139 deletions
Show Stats Download Patch File Download Diff File

153
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -48,10 +48,10 @@ public class kelondroCollectionIndex {
private static final int idx_col_key = 0; // the index private static final int idx_col_key = 0; // the index
private static final int idx_col_chunksize = 1; // chunksize (number of bytes in a single chunk, needed for migration option) private static final int idx_col_chunksize = 1; // chunksize (number of bytes in a single chunk, needed for migration option)
private static final int idx_col_chunkcount = 2; // chunkcount (number of chunks in this collection) needed to identify array file that has the chunks private static final int idx_col_chunkcount = 2; // chunkcount (number of chunks in this collection)
private static final int idx_col_clusteridx = 3; // selector for right cluster file, must be >= arrayIndex(chunkcount) private static final int idx_col_clusteridx = 3; // selector for right cluster file, must be >= arrayIndex(chunkcount)
private static final int idx_col_flags = 4; // flags (for future use) private static final int idx_col_flags = 4; // flags (for future use)
private static final int idx_col_indexpos = 5; // indexpos (position in index file) private static final int idx_col_indexpos = 5; // indexpos (position in array file)
private static final int idx_col_lastread = 6; // a time stamp, update time in days since 1.1.2000 private static final int idx_col_lastread = 6; // a time stamp, update time in days since 1.1.2000
private static final int idx_col_lastwrote = 7; // a time stamp, update time in days since 1.1.2000 private static final int idx_col_lastwrote = 7; // a time stamp, update time in days since 1.1.2000
@ -139,13 +139,17 @@ public class kelondroCollectionIndex {
if ((index != null) && (indexGeneration)) { if ((index != null) && (indexGeneration)) {
// loop over all elements in array and create index entry for each row // loop over all elements in array and create index entry for each row
kelondroRow.Entry aentry, ientry; kelondroRow.EntryIndex aentry;
kelondroRow.Entry ientry;
Iterator ei = array.contentRows(-1);
byte[] key; byte[] key;
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
long lastlog = start; long lastlog = start;
for (int j = 0; j < array.USAGE.allCount(); j++) { int count = 0;
aentry = array.get(j); while (ei.hasNext()) {
aentry = (kelondroRow.EntryIndex) ei.next();
key = aentry.getColBytes(0); key = aentry.getColBytes(0);
assert (key != null);
if (key == null) continue; // skip deleted entries if (key == null) continue; // skip deleted entries
kelondroRowSet indexrows = new kelondroRowSet(this.playloadrow, aentry.getColBytes(1)); kelondroRowSet indexrows = new kelondroRowSet(this.playloadrow, aentry.getColBytes(1));
ientry = irow.newEntry(); ientry = irow.newEntry();
@ -154,14 +158,15 @@ public class kelondroCollectionIndex {
ientry.setCol(idx_col_chunkcount, indexrows.size()); ientry.setCol(idx_col_chunkcount, indexrows.size());
ientry.setCol(idx_col_clusteridx, (byte) partitionNumber); ientry.setCol(idx_col_clusteridx, (byte) partitionNumber);
ientry.setCol(idx_col_flags, (byte) 0); ientry.setCol(idx_col_flags, (byte) 0);
ientry.setCol(idx_col_indexpos, j); ientry.setCol(idx_col_indexpos, aentry.index());
ientry.setCol(idx_col_lastread, t); ientry.setCol(idx_col_lastread, t);
ientry.setCol(idx_col_lastwrote, t); ientry.setCol(idx_col_lastwrote, t);
index.addUnique(ientry); index.addUnique(ientry);
count++;
// write a log // write a log
if (System.currentTimeMillis() - lastlog > 30000) { if (System.currentTimeMillis() - lastlog > 30000) {
serverLog.logFine("STARTUP", "created " + j + " RWI index entries. " + (((System.currentTimeMillis() - start) * (array.USAGE.allCount() - j) / ((j == 0) ? 1 : j)) / 60000) + " minutes remaining for this array"); serverLog.logFine("STARTUP", "created " + count + " RWI index entries. " + (((System.currentTimeMillis() - start) * (array.USAGE.allCount() - count) / count) / 60000) + " minutes remaining for this array");
lastlog = System.currentTimeMillis(); lastlog = System.currentTimeMillis();
} }
} }
@ -235,42 +240,56 @@ public class kelondroCollectionIndex {
} }
} }
public int size() throws IOException { public synchronized int size() throws IOException {
return index.size(); return index.size();
} }
public void put(byte[] key, kelondroRowCollection collection) throws IOException, kelondroOutOfLimitsException { public synchronized void put(byte[] key, kelondroRowCollection collection) throws IOException, kelondroOutOfLimitsException {
// this replaces an old collection by a new one // this replaces an old collection by a new one
// this method is not approriate to extend an existing collection with another collection // this method is not approriate to extend an existing collection with another collection
putmergeremove(key, collection, false, null, false); putmergeremove(key, collection, false, null);
} }
public void merge(byte[] key, kelondroRowCollection collection) throws IOException, kelondroOutOfLimitsException { public synchronized void merge(byte[] key, kelondroRowCollection collection) throws IOException, kelondroOutOfLimitsException {
putmergeremove(key, collection, true, null, false); putmergeremove(key, collection, true, null);
} }
public int remove(byte[] key, Set removekeys, boolean deletecomplete) throws IOException, kelondroOutOfLimitsException { public synchronized int remove(byte[] key, Set removekeys, boolean deletecomplete) throws IOException, kelondroOutOfLimitsException {
return putmergeremove(key, null, false, removekeys, deletecomplete); return putmergeremove(key, null, false, removekeys);
} }
private int putmergeremove(byte[] key, kelondroRowCollection collection, boolean merge, Set removekeys, boolean deletecomplete) throws IOException, kelondroOutOfLimitsException { private int putmergeremove(byte[] key, kelondroRowCollection collection, boolean merge, Set removekeys) throws IOException, kelondroOutOfLimitsException {
//if (collection.size() > maxChunks) throw new kelondroOutOfLimitsException(maxChunks, collection.size()); //if (collection.size() > maxChunks) throw new kelondroOutOfLimitsException(maxChunks, collection.size());
if ((!merge) && (removekeys != null) && (collection != null) && (collection.size() == 0)) {
// this is not a replacement, it is a deletion
delete(key);
return 0;
}
synchronized (index) {
// first find an old entry, if one exists // first find an old entry, if one exists
kelondroRow.Entry indexrow = index.get(key); kelondroRow.Entry indexrow = index.get(key);
if (indexrow == null) { if (indexrow == null) {
if ((collection != null) && (collection.size() > 0)) { if ((collection != null) && (collection.size() > 0)) {
// the collection is new // the collection is new
overwrite(key, collection, arrayIndex(collection.size()), index.row().newEntry()); int newPartitionNumber = arrayIndex(collection.size());
indexrow = index.row().newEntry();
kelondroFixedWidthArray array = getArray(newPartitionNumber, 0, this.playloadrow.objectsize());
// define row
kelondroRow.Entry arrayEntry = array.row().newEntry();
arrayEntry.setCol(0, key);
arrayEntry.setCol(1, collection.exportCollection());
// write a new entry in this array
int newRowNumber = array.add(arrayEntry);
// store the new row number in the index
indexrow.setCol(idx_col_key, key);
indexrow.setCol(idx_col_chunksize, this.playloadrow.objectsize());
indexrow.setCol(idx_col_chunkcount, collection.size());
indexrow.setCol(idx_col_clusteridx, (byte) newPartitionNumber);
indexrow.setCol(idx_col_flags, (byte) 0);
indexrow.setCol(idx_col_indexpos, (long) newRowNumber);
indexrow.setCol(idx_col_lastread, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
indexrow.setCol(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
index.addUnique(indexrow);
} }
return 0; return 0;
} }
@ -286,7 +305,7 @@ public class kelondroCollectionIndex {
if (merge) { if (merge) {
// load the old collection and join it // load the old collection and join it
kelondroRowSet oldcollection = getwithparams(indexrow, oldchunksize, oldchunkcount, oldPartitionNumber, oldrownumber, oldSerialNumber, false, false); kelondroRowSet oldcollection = getwithparams(indexrow, oldchunksize, oldchunkcount, oldPartitionNumber, oldrownumber, oldSerialNumber, false);
// join with new collection // join with new collection
oldcollection.addAll(collection); oldcollection.addAll(collection);
@ -298,7 +317,7 @@ public class kelondroCollectionIndex {
int removed = 0; int removed = 0;
if (removekeys != null) { if (removekeys != null) {
// load the old collection and remove keys // load the old collection and remove keys
kelondroRowSet oldcollection = getwithparams(indexrow, oldchunksize, oldchunkcount, oldPartitionNumber, oldrownumber, oldSerialNumber, false, false); kelondroRowSet oldcollection = getwithparams(indexrow, oldchunksize, oldchunkcount, oldPartitionNumber, oldrownumber, oldSerialNumber, false);
// remove the keys from the set // remove the keys from the set
Iterator i = removekeys.iterator(); Iterator i = removekeys.iterator();
@ -309,20 +328,15 @@ public class kelondroCollectionIndex {
if ((k instanceof String) && (oldcollection.remove(((String) k).getBytes()) != null)) removed++; if ((k instanceof String) && (oldcollection.remove(((String) k).getBytes()) != null)) removed++;
} }
oldcollection.shape(); oldcollection.shape();
oldcollection.trim();
collection = oldcollection; collection = oldcollection;
} }
if (collection.size() == 0) { if (collection.size() == 0) {
if (deletecomplete) { // delete the index entry and the array
kelondroFixedWidthArray array = getArray(oldPartitionNumber, oldSerialNumber, oldchunksize); kelondroFixedWidthArray array = getArray(oldPartitionNumber, oldSerialNumber, oldchunksize);
array.remove(oldrownumber); array.remove(oldrownumber);
index.remove(key); index.remove(key);
} else {
// update the index entry
indexrow.setCol(idx_col_chunkcount, 0);
indexrow.setCol(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
index.put(indexrow);
}
return removed; return removed;
} }
@ -347,7 +361,6 @@ public class kelondroCollectionIndex {
// update the index entry // update the index entry
indexrow.setCol(idx_col_chunkcount, collection.size()); indexrow.setCol(idx_col_chunkcount, collection.size());
indexrow.setCol(idx_col_clusteridx, (byte) oldPartitionNumber); indexrow.setCol(idx_col_clusteridx, (byte) oldPartitionNumber);
indexrow.setCol(idx_col_flags, (byte) 0);
indexrow.setCol(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis())); indexrow.setCol(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
index.put(indexrow); index.put(indexrow);
} else { } else {
@ -359,18 +372,7 @@ public class kelondroCollectionIndex {
array.remove(oldrownumber); array.remove(oldrownumber);
// write a new entry in the other array // write a new entry in the other array
overwrite(key, collection, newPartitionNumber, indexrow); array = getArray(newPartitionNumber, 0, this.playloadrow.objectsize());
}
return removed;
}
}
private void overwrite(byte[] key, kelondroRowCollection collection, int targetpartition, kelondroRow.Entry indexEntry) throws IOException {
// helper method, should not be called directly and only within a synchronized(index) environment
// simply store a collection without check if the collection existed before
// find array file
kelondroFixedWidthArray array = getArray(targetpartition, 0, this.playloadrow.objectsize());
// define row // define row
kelondroRow.Entry arrayEntry = array.row().newEntry(); kelondroRow.Entry arrayEntry = array.row().newEntry();
@ -381,46 +383,41 @@ public class kelondroCollectionIndex {
int newRowNumber = array.add(arrayEntry); int newRowNumber = array.add(arrayEntry);
// store the new row number in the index // store the new row number in the index
indexEntry.setCol(idx_col_key, key); indexrow.setCol(idx_col_key, key);
indexEntry.setCol(idx_col_chunksize, this.playloadrow.objectsize()); indexrow.setCol(idx_col_chunkcount, collection.size());
indexEntry.setCol(idx_col_chunkcount, collection.size()); indexrow.setCol(idx_col_clusteridx, (byte) newPartitionNumber);
indexEntry.setCol(idx_col_clusteridx, (byte) targetpartition); indexrow.setCol(idx_col_indexpos, (long) newRowNumber);
indexEntry.setCol(idx_col_flags, (byte) 0); indexrow.setCol(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
indexEntry.setCol(idx_col_indexpos, (long) newRowNumber); index.put(indexrow);
indexEntry.setCol(idx_col_lastread, kelondroRowCollection.daysSince2000(System.currentTimeMillis())); }
indexEntry.setCol(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis())); return removed;
index.put(indexEntry);
} }
public int indexSize(byte[] key) throws IOException { public synchronized int indexSize(byte[] key) throws IOException {
synchronized (index) {
kelondroRow.Entry indexrow = index.get(key); kelondroRow.Entry indexrow = index.get(key);
if (indexrow == null) return 0; if (indexrow == null) return 0;
return (int) indexrow.getColLong(idx_col_chunkcount); return (int) indexrow.getColLong(idx_col_chunkcount);
} }
}
public kelondroRowSet get(byte[] key, boolean deleteIfEmpty) throws IOException { public synchronized kelondroRowSet get(byte[] key, boolean deleteIfEmpty) throws IOException {
// find an entry, if one exists // find an entry, if one exists
synchronized (index) {
kelondroRow.Entry indexrow = index.get(key); kelondroRow.Entry indexrow = index.get(key);
if (indexrow == null) return null; if (indexrow == null) return null;
return getdelete(indexrow, false, deleteIfEmpty); kelondroRowSet col = getdelete(indexrow, false);
} assert (col != null);
return col;
} }
public kelondroRowSet delete(byte[] key) throws IOException { public synchronized kelondroRowSet delete(byte[] key) throws IOException {
// find an entry, if one exists // find an entry, if one exists
synchronized (index) { kelondroRow.Entry indexrow = index.remove(key);
kelondroRow.Entry indexrow = index.get(key);
if (indexrow == null) return null; if (indexrow == null) return null;
kelondroRowSet removedCollection = getdelete(indexrow, true, false); kelondroRowSet removedCollection = getdelete(indexrow, true);
index.remove(key); assert (removedCollection != null);
return removedCollection; return removedCollection;
} }
}
protected kelondroRowSet getdelete(kelondroRow.Entry indexrow, boolean remove, boolean deleteIfEmpty) throws IOException { protected kelondroRowSet getdelete(kelondroRow.Entry indexrow, boolean remove) throws IOException {
// call this only within a synchronized(index) environment // call this only within a synchronized(index) environment
// read values // read values
@ -431,10 +428,10 @@ public class kelondroCollectionIndex {
assert(partitionnumber >= arrayIndex(chunkcount)); assert(partitionnumber >= arrayIndex(chunkcount));
int serialnumber = 0; int serialnumber = 0;
return getwithparams(indexrow, chunksize, chunkcount, partitionnumber, rownumber, serialnumber, remove, deleteIfEmpty); return getwithparams(indexrow, chunksize, chunkcount, partitionnumber, rownumber, serialnumber, remove);
} }
private kelondroRowSet getwithparams(kelondroRow.Entry indexrow, int chunksize, int chunkcount, int clusteridx, int rownumber, int serialnumber, boolean remove, boolean deleteIfEmpty) throws IOException { private kelondroRowSet getwithparams(kelondroRow.Entry indexrow, int chunksize, int chunkcount, int clusteridx, int rownumber, int serialnumber, boolean remove) throws IOException {
// open array entry // open array entry
kelondroFixedWidthArray array = getArray(clusteridx, serialnumber, chunksize); kelondroFixedWidthArray array = getArray(clusteridx, serialnumber, chunksize);
kelondroRow.Entry arrayrow = array.get(rownumber); kelondroRow.Entry arrayrow = array.get(rownumber);
@ -442,9 +439,10 @@ public class kelondroCollectionIndex {
// read the row and define a collection // read the row and define a collection
kelondroRowSet collection = new kelondroRowSet(this.playloadrow, arrayrow.getColBytes(1)); // FIXME: this does not yet work with different rowdef in case of several rowdef.objectsize() kelondroRowSet collection = new kelondroRowSet(this.playloadrow, arrayrow.getColBytes(1)); // FIXME: this does not yet work with different rowdef in case of several rowdef.objectsize()
if (index.order().compare(arrayrow.getColBytes(0), indexrow.getColBytes(idx_col_key)) != 0) { byte[] key = indexrow.getColBytes(idx_col_key);
if (index.order().compare(arrayrow.getColBytes(0), key) != 0) {
// check if we got the right row; this row is wrong. Fix it: // check if we got the right row; this row is wrong. Fix it:
index.remove(indexrow.getColBytes(idx_col_key)); // the wrong row cannot be fixed index.remove(key); // the wrong row cannot be fixed
// store the row number in the index; this may be a double-entry, but better than nothing // store the row number in the index; this may be a double-entry, but better than nothing
kelondroRow.Entry indexEntry = index.row().newEntry(); kelondroRow.Entry indexEntry = index.row().newEntry();
indexEntry.setCol(idx_col_key, arrayrow.getColBytes(0)); indexEntry.setCol(idx_col_key, arrayrow.getColBytes(0));
@ -465,11 +463,11 @@ public class kelondroCollectionIndex {
index.put(indexrow); index.put(indexrow);
array.logFailure("INCONSISTENCY in " + arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString() + ": array has different chunkcount than index: index = " + chunkcount + ", array = " + chunkcountInArray + "; the index has been auto-fixed"); array.logFailure("INCONSISTENCY in " + arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString() + ": array has different chunkcount than index: index = " + chunkcount + ", array = " + chunkcountInArray + "; the index has been auto-fixed");
} }
if ((remove) || ((collection.size() == 0) && (deleteIfEmpty))) array.remove(rownumber); if (remove) array.remove(rownumber); // index is removed in calling method
return collection; return collection;
} }
public Iterator keycollections(byte[] startKey, boolean rot) { public synchronized Iterator keycollections(byte[] startKey, boolean rot) {
// returns an iteration of {byte[], kelondroRowSet} Objects // returns an iteration of {byte[], kelondroRowSet} Objects
try { try {
return new keycollectionIterator(startKey, rot); return new keycollectionIterator(startKey, rot);
@ -494,9 +492,10 @@ public class kelondroCollectionIndex {
public Object next() { public Object next() {
kelondroRow.Entry indexrow = (kelondroRow.Entry) indexRowIterator.next(); kelondroRow.Entry indexrow = (kelondroRow.Entry) indexRowIterator.next();
assert (indexrow != null);
if (indexrow == null) return null; if (indexrow == null) return null;
try { try {
return new Object[]{indexrow.getColBytes(0), getdelete(indexrow, false, false)}; return new Object[]{indexrow.getColBytes(0), getdelete(indexrow, false)};
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
return null; return null;
@ -509,15 +508,13 @@ public class kelondroCollectionIndex {
} }
public void close() throws IOException { public synchronized void close() throws IOException {
synchronized (index) {
this.index.close(); this.index.close();
Iterator i = arrays.values().iterator(); Iterator i = arrays.values().iterator();
while (i.hasNext()) { while (i.hasNext()) {
((kelondroFixedWidthArray) i.next()).close(); ((kelondroFixedWidthArray) i.next()).close();
} }
} }
}
public static void main(String[] args) { public static void main(String[] args) {

1
source/de/anomic/kelondro/kelondroMap.java
Unescape View File

@ -380,6 +380,7 @@ public class kelondroMap {
} }
try { try {
final Map map = get(nextKey); final Map map = get(nextKey);
//assert (map != null) : "nextKey = " + nextKey;
if (map == null) throw new kelondroException("no more elements available"); if (map == null) throw new kelondroException("no more elements available");
map.put("key", nextKey); map.put("key", nextKey);
return map; return map;

7
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -595,6 +595,7 @@ public class kelondroRecords {
protected Node(Handle handle, byte[] bulkchunk, int offset) { protected Node(Handle handle, byte[] bulkchunk, int offset) {
// this initializer is used to create nodes from bulk-read byte arrays // this initializer is used to create nodes from bulk-read byte arrays
this.handle = handle; this.handle = handle;
assert (bulkchunk.length >= offset + headchunksize) : "bulkchunk.length = " + bulkchunk.length + ", offset = " + offset + ", headchunksize = " + headchunksize;
// create empty chunks // create empty chunks
this.headChunk = new byte[headchunksize]; this.headChunk = new byte[headchunksize];
@ -1029,7 +1030,8 @@ public class kelondroRecords {
public Object next() { public Object next() {
try { try {
return row().newEntry(((Node) nodeIterator.next()).getValueRow()); Node n = (Node) nodeIterator.next();
return row().newEntry(n.getValueRow(), n.handle.index);
} catch (IOException e) { } catch (IOException e) {
throw new kelondroException(filename, e.getMessage()); throw new kelondroException(filename, e.getMessage());
} }
@ -1121,7 +1123,7 @@ public class kelondroRecords {
while ((markedDeleted.contains(pos)) && (pos.index < USAGE.allCount())) pos.index++; while ((markedDeleted.contains(pos)) && (pos.index < USAGE.allCount())) pos.index++;
// initialize bulk // initialize bulk
bulksize = Math.min(65536 / recordsize, USAGE.allCount()); bulksize = Math.max(1, Math.min(65536 / recordsize, USAGE.allCount()));
bulkstart = -bulksize; bulkstart = -bulksize;
bulk = new byte[bulksize * recordsize]; bulk = new byte[bulksize * recordsize];
next = (hasNext0()) ? next0() : null; next = (hasNext0()) ? next0() : null;
@ -1148,6 +1150,7 @@ public class kelondroRecords {
byte[] key = nn.getKey(); byte[] key = nn.getKey();
if ((key == null) || if ((key == null) ||
((key.length == 1) && (key[0] == (byte) 0x80)) || // the NUL pointer ('lost' chain terminator) ((key.length == 1) && (key[0] == (byte) 0x80)) || // the NUL pointer ('lost' chain terminator)
((key.length > 3) && (key[0] == (byte) 0x80) && (key[1] == 0) && (key[2] == 0) && (key[3] == 0)) ||
((key.length > 0) && (key[0] == 0)) // a 'lost' pointer within a deleted-chain ((key.length > 0) && (key[0] == 0)) // a 'lost' pointer within a deleted-chain
) { ) {
// this is a deleted node; probably not commited with dispose // this is a deleted node; probably not commited with dispose

37
source/de/anomic/kelondro/kelondroRow.java
Unescape View File

@ -128,6 +128,11 @@ public class kelondroRow {
return new Entry(rowinstance); return new Entry(rowinstance);
} }
public EntryIndex newEntry(byte[] rowinstance, int index) {
if (rowinstance == null) return null;
return new EntryIndex(rowinstance, index);
}
public Entry newEntry(byte[] rowinstance, int start, int length) { public Entry newEntry(byte[] rowinstance, int start, int length) {
if (rowinstance == null) return null; if (rowinstance == null) return null;
return new Entry(rowinstance, start, length); return new Entry(rowinstance, start, length);
@ -142,12 +147,7 @@ public class kelondroRow {
if (external == null) return null; if (external == null) return null;
return new Entry(external); return new Entry(external);
} }
/*
public Entry newEntry(Properties prop) {
if (prop == null) return null;
return new Entry(prop);
}
*/
public class Entry implements Comparable { public class Entry implements Comparable {
private byte[] rowinstance; private byte[] rowinstance;
@ -207,19 +207,7 @@ public class kelondroRow {
} }
} }
} }
/*
public Entry(Properties prop) {
// parse external form
if (nickref == null) genNickRef();
rowinstance = new byte[objectsize];
Iterator i = prop.entrySet().iterator();
Map.Entry entry;
while (i.hasNext()) {
entry = (Map.Entry) i.next();
setCol(((String) entry.getKey()).trim(), ((String) entry.getValue()).trim().getBytes());
}
}
*/
public int compareTo(Object o) { public int compareTo(Object o) {
if (o instanceof Entry) { if (o instanceof Entry) {
return kelondroNaturalOrder.naturalOrder.compare(this.rowinstance, ((Entry) o).rowinstance); return kelondroNaturalOrder.naturalOrder.compare(this.rowinstance, ((Entry) o).rowinstance);
@ -447,6 +435,17 @@ public class kelondroRow {
} }
public final class EntryIndex extends Entry {
private int index;
public EntryIndex(byte[] row, int i) {
super(row);
this.index = i;
}
public int index() {
return index;
}
}
public final static void long2bytes(long x, byte[] b, int offset, int length) { public final static void long2bytes(long x, byte[] b, int offset, int length) {
for (int i = length - 1; i >= 0; i--) { for (int i = length - 1; i >= 0; i--) {
b[offset + i] = (byte) (x & 0XFF); b[offset + i] = (byte) (x & 0XFF);

6
source/de/anomic/kelondro/kelondroRowCollection.java
Unescape View File

@ -202,7 +202,7 @@ public class kelondroRowCollection {
} }
public void addUnique(kelondroRow.Entry row) { public void addUnique(kelondroRow.Entry row) {
add(row.bytes(), 0, row.bytes().length); addUnique(row.bytes(), 0, row.bytes().length);
} }
public void addUnique(kelondroRow.Entry row, Date entryDate) { public void addUnique(kelondroRow.Entry row, Date entryDate) {
@ -210,10 +210,10 @@ public class kelondroRowCollection {
} }
public void add(byte[] a) { public void add(byte[] a) {
add(a, 0, a.length); addUnique(a, 0, a.length);
} }
private final void add(byte[] a, int astart, int alength) { private final void addUnique(byte[] a, int astart, int alength) {
assert (a != null); assert (a != null);
assert (astart >= 0) && (astart < a.length) : " astart = " + a; assert (astart >= 0) && (astart < a.length) : " astart = " + a;
assert (!(serverLog.allZero(a, astart, alength))) : "a = " + serverLog.arrayList(a, astart, alength); assert (!(serverLog.allZero(a, astart, alength))) : "a = " + serverLog.arrayList(a, astart, alength);

21
source/de/anomic/kelondro/kelondroRowSet.java
Unescape View File

@ -24,7 +24,6 @@
package de.anomic.kelondro; package de.anomic.kelondro;
import java.io.IOException;
import java.util.Date; import java.util.Date;
import java.util.Iterator; import java.util.Iterator;
import java.util.Random; import java.util.Random;
@ -83,7 +82,23 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
return entry; return entry;
} }
public kelondroRow.Entry put(kelondroRow.Entry row, Date entryDate) throws IOException { public void addUnique(kelondroRow.Entry row) {
if (removeMarker.size() == 0) {
super.addUnique(row);
} else {
this.put(row);
}
}
public void addUnique(kelondroRow.Entry row, Date entryDate) {
if (removeMarker.size() == 0) {
super.addUnique(row, entryDate);
} else {
this.put(row, entryDate);
}
}
public kelondroRow.Entry put(kelondroRow.Entry row, Date entryDate) {
return put(row); return put(row);
} }
@ -100,7 +115,7 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
set(index, entry); set(index, entry);
removeMarker.remove(new Integer(index)); removeMarker.remove(new Integer(index));
} else if (index < 0) { } else if (index < 0) {
addUnique(entry); super.addUnique(entry);
} else { } else {
oldentry = get(index); oldentry = get(index);
set(index, entry); set(index, entry);

9
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -854,12 +854,14 @@ public final class yacySeedDB {
it = (firstKey == null) ? database.maps(up, rot) : database.maps(up, rot, firstKey); it = (firstKey == null) ? database.maps(up, rot) : database.maps(up, rot, firstKey);
nextSeed = internalNext(); nextSeed = internalNext();
} catch (IOException e) { } catch (IOException e) {
yacyCore.log.logFine("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); e.printStackTrace();
yacyCore.log.logSevere("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile); if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile); if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
it = null; it = null;
} catch (kelondroException e) { } catch (kelondroException e) {
yacyCore.log.logFine("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); e.printStackTrace();
yacyCore.log.logSevere("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile); if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile); if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
it = null; it = null;
@ -872,7 +874,8 @@ public final class yacySeedDB {
it = database.maps(up, field); it = database.maps(up, field);
nextSeed = internalNext(); nextSeed = internalNext();
} catch (kelondroException e) { } catch (kelondroException e) {
yacyCore.log.logFine("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); e.printStackTrace();
yacyCore.log.logSevere("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile); if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile); if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
if (database == seedPotentialDB) seedPotentialDB = resetSeedTable(seedPotentialDB, seedPotentialDBFile); if (database == seedPotentialDB) seedPotentialDB = resetSeedTable(seedPotentialDB, seedPotentialDBFile);

Write Preview
Loading…
Cancel
Save