boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

redesigned remove method in kelondroRowSet

Browse Source 
This should fix also numerous bugs like
http://www.yacy-forum.de/viewtopic.php?p=31077#31077
(java.lang.ArrayIndexOutOfBoundsException in kelondroRowCollection.removeShift)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3476 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 18 years ago
parent 9f929b5438
commit 96b79bf86d
6 changed files with 117 additions and 252 deletions
Show Stats Download Patch File Download Diff File

8
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -597,7 +597,7 @@ public class kelondroCollectionIndex {
// join with new collection
oldcollection.addAllUnique(collection);
oldcollection.shape();
oldcollection.sort();
oldcollection.uniq(); // FIXME: not clear if it would be better to insert the collection with put to avoid double-entries
oldcollection.trim(false);
@ -714,7 +714,7 @@ public class kelondroCollectionIndex {
// join with new collection
oldcollection.addAllUnique(collection);
oldcollection.shape();
oldcollection.sort();
oldcollection.uniq(); // FIXME: not clear if it would be better to insert the collection with put to avoid double-entries
oldcollection.trim(false);
collection = oldcollection;
@ -796,7 +796,7 @@ public class kelondroCollectionIndex {
private void saveCommons(byte[] key, kelondroRowSet collection) {
if (key.length != 12) return;
collection.shape();
collection.sort();
TimeZone GMTTimeZone = TimeZone.getTimeZone("GMT");
Calendar gregorian = new GregorianCalendar(GMTTimeZone);
SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
@ -844,7 +844,7 @@ public class kelondroCollectionIndex {
if ((k instanceof byte[]) && (oldcollection.remove((byte[]) k) != null)) removed++;
if ((k instanceof String) && (oldcollection.remove(((String) k).getBytes()) != null)) removed++;
}
oldcollection.shape();
oldcollection.sort();
oldcollection.trim(false);
if (oldcollection.size() == 0) {

2
source/de/anomic/kelondro/kelondroFlexTable.java
Unescape View File

@ -151,7 +151,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
}
System.out.print(" -ordering- ");
System.out.flush();
ri.shape();
ri.sort();
return ri;
}

2
source/de/anomic/kelondro/kelondroIntBytesMap.java
Unescape View File

@ -109,7 +109,7 @@ public class kelondroIntBytesMap {
public void flush() {
if (index instanceof kelondroRowSet) {
((kelondroRowSet) index).shape();
((kelondroRowSet) index).sort();
((kelondroRowSet) index).trim(true);
}
}

90
source/de/anomic/kelondro/kelondroRowCollection.java
Unescape View File

@ -39,7 +39,7 @@ public class kelondroRowCollection {
public static final double growfactor = 1.4;
protected byte[] chunkcache;
private byte[] chunkcache;
protected int chunkcount;
protected long lastTimeRead, lastTimeWrote;
protected kelondroRow rowdef;
@ -135,7 +135,7 @@ public class kelondroRowCollection {
public static final int exportOverheadSize = 14;
public byte[] exportCollection() {
public synchronized byte[] exportCollection() {
// returns null if the collection is empty
trim(false);
kelondroRow row = exportRow(chunkcache.length);
@ -169,19 +169,25 @@ public class kelondroRowCollection {
newChunkcache = null;
}
public void trim(boolean plusGrowFactor) {
if (chunkcache.length == 0) return;
synchronized (chunkcache) {
public synchronized void trim(boolean plusGrowFactor) {
if (chunkcache.length == 0)
return;
int needed = chunkcount * rowdef.objectsize();
if (plusGrowFactor) needed = (int) (needed * growfactor);
if (needed >= chunkcache.length) return; // in case that the growfactor causes that the cache would grow instead of shrink, simply ignore the growfactor
if (serverMemory.available() + 1000 < needed) return; // if the swap buffer is not available, we must give up. This is not critical. Othervise we provoke a serious problem with OOM
if (plusGrowFactor)
needed = (int) (needed * growfactor);
if (needed >= chunkcache.length)
return; // in case that the growfactor causes that the cache would
// grow instead of shrink, simply ignore the growfactor
if (serverMemory.available() + 1000 < needed)
return; // if the swap buffer is not available, we must give up.
// This is not critical. Othervise we provoke a serious
// problem with OOM
byte[] newChunkcache = new byte[needed];
System.arraycopy(chunkcache, 0, newChunkcache, 0, Math.min(chunkcache.length, newChunkcache.length));
System.arraycopy(chunkcache, 0, newChunkcache, 0, Math.min(
chunkcache.length, newChunkcache.length));
chunkcache = newChunkcache;
newChunkcache = null;
}
}
public final long lastRead() {
return lastTimeRead;
@ -191,16 +197,14 @@ public class kelondroRowCollection {
return lastTimeWrote;
}
public final kelondroRow.Entry get(int index) {
public synchronized final kelondroRow.Entry get(int index) {
assert (index >= 0) : "get: access with index " + index + " is below zero";
assert (index < chunkcount) : "get: access with index " + index + " is above chunkcount " + chunkcount + "; sortBound = " + sortBound;
assert (index * rowdef.objectsize < chunkcache.length);
if (index >= chunkcount) return null;
if (index * rowdef.objectsize() >= chunkcache.length) return null;
byte[] a = new byte[rowdef.objectsize()];
synchronized (chunkcache) {
System.arraycopy(chunkcache, index * rowdef.objectsize(), a, 0, rowdef.objectsize());
}
this.lastTimeRead = System.currentTimeMillis();
return rowdef.newEntry(a);
}
@ -209,15 +213,13 @@ public class kelondroRowCollection {
set(index, a.bytes(), 0, a.bytes().length);
}
public final void set(int index, byte[] a, int astart, int alength) {
public synchronized final void set(int index, byte[] a, int astart, int alength) {
assert (index >= 0) : "get: access with index " + index + " is below zero";
assert (index < chunkcount) : "get: access with index " + index + " is above chunkcount " + chunkcount;
assert (!(bugappearance(a, astart, alength))) : "a = " + serverLog.arrayList(a, astart, alength);
if (bugappearance(a, astart, alength)) return; // TODO: this is temporary; remote peers may still submit bad entries
int l = Math.min(rowdef.objectsize(), Math.min(alength, a.length - astart));
synchronized (chunkcache) {
System.arraycopy(a, astart, chunkcache, index * rowdef.objectsize(), l);
}
this.lastTimeWrote = System.currentTimeMillis();
}
@ -238,7 +240,7 @@ public class kelondroRowCollection {
addUnique(a, 0, a.length);
}
private final void addUnique(byte[] a, int astart, int alength) {
private synchronized final void addUnique(byte[] a, int astart, int alength) {
assert (a != null);
assert (astart >= 0) && (astart < a.length) : " astart = " + a;
assert (!(serverLog.allZero(a, astart, alength))) : "a = " + serverLog.arrayList(a, astart, alength);
@ -250,11 +252,9 @@ public class kelondroRowCollection {
}
assert (!(bugappearance(a, astart, alength))) : "a = " + serverLog.arrayList(a, astart, alength);
int l = Math.min(rowdef.objectsize(), Math.min(alength, a.length - astart));
synchronized (chunkcache) {
ensureSize(chunkcount + 1);
System.arraycopy(a, astart, chunkcache, rowdef.objectsize() * chunkcount, l);
chunkcount++;
}
this.lastTimeWrote = System.currentTimeMillis();
}
@ -270,15 +270,13 @@ public class kelondroRowCollection {
return false;
}
public final void addAllUnique(kelondroRowCollection c) {
public synchronized final void addAllUnique(kelondroRowCollection c) {
if (c == null) return;
assert(rowdef.objectsize() == c.rowdef.objectsize());
synchronized(chunkcache) {
ensureSize(chunkcount + c.size());
System.arraycopy(c.chunkcache, 0, chunkcache, rowdef.objectsize() * chunkcount, rowdef.objectsize() * c.size());
chunkcount += c.size();
}
}
protected final void removeShift(int pos, int dist, int upBound) {
assert ((pos + dist) * rowdef.objectsize() >= 0) : "pos = " + pos + ", dist = " + dist + ", rowdef.objectsize() = " + rowdef.objectsize;
@ -296,24 +294,19 @@ public class kelondroRowCollection {
System.arraycopy(chunkcache, this.rowdef.objectsize() * (chunkcount - 1), chunkcache, this.rowdef.objectsize() * i, this.rowdef.objectsize());
}
protected final void removeRow(int p) {
protected synchronized final void removeRow(int p) {
assert ((p >= 0) && (p < chunkcount) && (chunkcount > 0)) : "p = " + p + ", chunkcount = " + chunkcount;
synchronized (chunkcache) {
if (p < sortBound) {
sortBound--;
removeShift(p, 1, chunkcount);
chunkcount--;
sortBound--;
} else {
copytop(p);
chunkcount--;
}
}
chunkcount--;
this.lastTimeWrote = System.currentTimeMillis();
}
public kelondroRow.Entry removeOne() {
synchronized (chunkcache) {
public synchronized kelondroRow.Entry removeOne() {
if (chunkcount == 0) return null;
kelondroRow.Entry r = get(chunkcount - 1);
if (chunkcount == sortBound) sortBound--;
@ -321,9 +314,8 @@ public class kelondroRowCollection {
this.lastTimeWrote = System.currentTimeMillis();
return r;
}
}
public void clear() {
public synchronized void clear() {
this.chunkcache = new byte[0];
this.chunkcount = 0;
this.sortBound = 0;
@ -334,7 +326,7 @@ public class kelondroRowCollection {
return chunkcount;
}
public Iterator rows() {
public synchronized Iterator rows() {
// iterates kelondroRow.Entry - type entries
return new rowIterator();
}
@ -361,10 +353,9 @@ public class kelondroRowCollection {
}
}
public void select(Set keys) {
public synchronized void select(Set keys) {
// removes all entries but the ones given by urlselection
if ((keys == null) || (keys.size() == 0)) return;
synchronized (this) {
Iterator i = rows();
kelondroRow.Entry row;
while (i.hasNext()) {
@ -372,9 +363,8 @@ public class kelondroRowCollection {
if (!(keys.contains(row.getColString(0, null)))) i.remove();
}
}
}
protected final void sort() {
public synchronized final void sort() {
assert (this.rowdef.objectOrder != null);
if (this.sortBound == this.chunkcount) return; // this is already sorted
//System.out.println("SORT(chunkcount=" + this.chunkcount + ", sortBound=" + this.sortBound + ")");
@ -474,23 +464,20 @@ public class kelondroRowCollection {
if (i == p) return j; else if (j == p) return i; else return p;
}
public void uniq() {
public synchronized void uniq() {
assert (this.rowdef.objectOrder != null);
// removes double-occurrences of chunks
// this works only if the collection was ordered with sort before
synchronized (chunkcache) {
if (chunkcount <= 1) return;
int i = 0;
while (i < chunkcount - 1) {
if (compare(i, i + 1) == 0) {
//System.out.println("DOUBLE: " + new String(this.chunkcache, this.chunksize * i, this.chunksize));
removeRow(i);
} else {
i++;
}
}
}
}
public String toString() {
StringBuffer s = new StringBuffer();
@ -521,6 +508,25 @@ public class kelondroRowCollection {
return c;
}
protected int compare(byte[] a, int astart, int alength, int chunknumber) {
assert (chunknumber < chunkcount);
int l = Math.min(this.rowdef.width(rowdef.primaryKey), Math.min(a.length - astart, alength));
return rowdef.objectOrder.compare(a, astart, l, chunkcache, chunknumber * this.rowdef.objectsize() + this.rowdef.colstart[rowdef.primaryKey], this.rowdef.width(rowdef.primaryKey));
}
protected boolean match(byte[] a, int astart, int alength, int chunknumber) {
if (chunknumber >= chunkcount) return false;
int i = 0;
int p = chunknumber * this.rowdef.objectsize() + this.rowdef.colstart[rowdef.primaryKey];
final int len = Math.min(this.rowdef.width(rowdef.primaryKey), Math.min(alength, a.length - astart));
while (i < len) if (a[astart + i++] != chunkcache[p++]) return false;
return ((len == this.rowdef.width(rowdef.primaryKey)) || (chunkcache[len] == 0)) ;
}
public synchronized void close() {
chunkcache = null;
}
public static void main(String[] args) {
System.out.println(new java.util.Date(10957 * day));
System.out.println(new java.util.Date(0));

183
source/de/anomic/kelondro/kelondroRowSet.java
Unescape View File

@ -29,87 +29,49 @@ import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
import java.util.TreeMap;
import de.anomic.server.logging.serverLog;
public class kelondroRowSet extends kelondroRowCollection implements kelondroIndex {
private static final int collectionReSortLimit = 90;
private static final int removeMaxSize = 100;
private kelondroProfile profile;
private TreeMap removeMarker;
public kelondroRowSet(kelondroRow rowdef, int objectCount, byte[] cache, int sortBound) {
super(rowdef, objectCount, cache, sortBound);
this.removeMarker = new TreeMap();
this.profile = new kelondroProfile();
}
public kelondroRowSet(kelondroRowSet rs) {
super(rs);
this.profile = rs.profile;
this.removeMarker = rs.removeMarker;
}
public kelondroRowSet(kelondroRow rowdef, int objectCount) {
super(rowdef, objectCount);
this.removeMarker = new TreeMap();
this.profile = new kelondroProfile();
}
public kelondroRowSet(kelondroRow rowdef, byte[] exportedCollectionRowinstance) {
super(rowdef, exportedCollectionRowinstance);
this.removeMarker = new TreeMap();
this.profile = new kelondroProfile();
}
public boolean has(byte[] key) throws IOException {
public synchronized boolean has(byte[] key) throws IOException {
return (get(key) != null);
}
public kelondroRow.Entry get(byte[] key) {
public synchronized kelondroRow.Entry get(byte[] key) {
return get(key, 0, key.length);
}
private kelondroRow.Entry get(byte[] key, int astart, int alength) {
long handle = profile.startRead();
kelondroRow.Entry entry = null;
synchronized (chunkcache) {
int index = find(key, astart, alength);
if ((index >= 0) && (!(isMarkedRemoved(index)))) entry = get(index);
}
kelondroRow.Entry entry = (index >= 0) ? get(index) : null;
profile.stopRead(handle);
return entry;
}
public void addUnique(kelondroRow.Entry row) {
// add an entry without doing a double-occurrence test
if (removeMarker.size() == 0) {
super.addUnique(row);
} else {
this.put(row);
}
}
public void addUnique(kelondroRow.Entry row, Date entryDate) {
if (removeMarker.size() == 0) {
super.addUnique(row, entryDate);
} else {
this.put(row, entryDate);
}
}
public void addUniqueMultiple(List rows, Date entryDate) throws IOException {
if (removeMarker.size() == 0) {
super.addUniqueMultiple(rows, entryDate);
} else {
Iterator i = rows.iterator();
while (i.hasNext()) addUnique((kelondroRow.Entry) i.next(), entryDate);
}
}
public synchronized void putMultiple(List rows, Date entryDate) throws IOException {
Iterator i = rows.iterator();
while (i.hasNext()) put((kelondroRow.Entry) i.next(), entryDate);
@ -119,116 +81,34 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
return put(row);
}
public kelondroRow.Entry put(kelondroRow.Entry entry) {
public synchronized kelondroRow.Entry put(kelondroRow.Entry entry) {
assert (entry != null);
assert (entry.getColBytes(rowdef.primaryKey) != null);
//assert (!(serverLog.allZero(entry.getColBytes(super.sortColumn))));
long handle = profile.startWrite();
int index = -1;
kelondroRow.Entry oldentry = null;
synchronized (chunkcache) {
index = find(entry.bytes(), super.rowdef.colstart[rowdef.primaryKey], super.rowdef.width(rowdef.primaryKey));
if (isMarkedRemoved(index)) {
set(index, entry);
removeMarker.remove(new Integer(index));
} else if (index < 0) {
if (index < 0) {
super.addUnique(entry);
} else {
oldentry = get(index);
set(index, entry);
}
}
profile.stopWrite(handle);
return oldentry;
}
public int size() {
return super.size() - removeMarker.size();
}
public kelondroRow.Entry remove(byte[] a) {
return removeMarked(a, 0, a.length);
}
private kelondroRow.Entry removeMarked(byte[] a, int astart, int alength) {
if (chunkcount == 0) return null;
long handle = profile.startDelete();
// check if it is contained in chunkcache
kelondroRow.Entry entry = null;
synchronized(chunkcache) {
int p = find(a, astart, alength);
if (p < 0) {
// the entry is not there
profile.stopDelete(handle);
return null;
} else {
// there is an entry
entry = get(p);
if (p < sortBound) {
// mark entry as to-be-deleted
removeMarker.put(new Integer(p), entry.getColBytes(rowdef.primaryKey));
if (removeMarker.size() > removeMaxSize) resolveMarkedRemoved();
} else {
// remove directly by swap
super.copytop(p);
chunkcount--;
}
profile.stopDelete(handle);
private synchronized kelondroRow.Entry remove(byte[] a, int start, int length) {
int index = find(a, start, length);
if (index < 0) return null;
kelondroRow.Entry entry = super.get(index);
super.removeRow(index);
return entry;
}
}
}
private boolean isMarkedRemoved(int index) {
return removeMarker.containsKey(new Integer(index));
}
public void shape() {
assert (rowdef.objectOrder != null); // we cannot shape without an object order
synchronized (chunkcache) {
try {
resolveMarkedRemoved();
super.sort();
} catch (kelondroException e) {
// bad bug, cannot be fixed. We abandon all data
serverLog.logSevere("kelondroRowSet", "abandoned row");
this.clear();
}
//if (super.rowdef.column(0).cellwidth() == 4) System.out.println("TABLE OF " + super.rowdef.toString() + "\n" + serverLog.table(super.chunkcache, super.rowdef.objectsize, 0)); // DEBUG
}
}
private void resolveMarkedRemoved() {
if (removeMarker.size() == 0) return;
// check case when complete chunkcache is marked as deleted
if (removeMarker.size() == chunkcount) {
this.clear();
removeMarker.clear();
return;
}
Integer nxt = (Integer) removeMarker.firstKey();
removeMarker.remove(nxt);
int idx = nxt.intValue();
assert (idx < sortBound) : "idx = " + idx + ", sortBound = " + sortBound;
int d = 1;
byte[] a;
while (removeMarker.size() > 0) {
nxt = (Integer) removeMarker.firstKey();
a = (byte[]) removeMarker.remove(nxt);
assert kelondroNaturalOrder.compares(a, 0, a.length, get(nxt.intValue()).getColBytes(rowdef.primaryKey), 0, a.length) == 0;
assert (nxt.intValue() < sortBound);
super.removeShift(idx, d, nxt.intValue());
idx = nxt.intValue() - d;
d++;
}
super.removeShift(idx, d, chunkcount);
chunkcount -= d;
sortBound -= d; // there are all markers below the sortBound
removeMarker.clear();
public kelondroRow.Entry remove(byte[] a) {
return remove(a, 0, a.length);
}
public void removeMarkedAll(kelondroRowCollection c) {
@ -237,7 +117,7 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
kelondroRow.Entry entry;
while (i.hasNext()) {
entry = (kelondroRow.Entry) i.next();
removeMarked(entry.bytes(), 0, entry.bytes().length);
remove(entry.bytes(), 0, entry.bytes().length);
}
profile.stopDelete(handle);
}
@ -246,9 +126,7 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
if ((rowdef.objectOrder == null) ||
(!(rowdef.objectOrder.signature().equals(newOrder.signature()))) ||
(newColumn != rowdef.primaryKey)) {
resolveMarkedRemoved();
rowdef.setOrdering(newOrder, newColumn);
assert (removeMarker.size() == 0);
this.sortBound = 0;
}
}
@ -259,7 +137,7 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
if (rowdef.objectOrder == null) return iterativeSearch(a, astart, alength, 0, this.chunkcount);
// check if a re-sorting make sense
if ((this.chunkcount - this.sortBound) > collectionReSortLimit) shape();
if ((this.chunkcount - this.sortBound) > collectionReSortLimit) sort();
// first try to find in sorted area
int p = binarySearch(a, astart, alength);
@ -323,31 +201,16 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
return l;
}
private int compare(byte[] a, int astart, int alength, int chunknumber) {
assert (chunknumber < chunkcount);
int l = Math.min(this.rowdef.width(rowdef.primaryKey), Math.min(a.length - astart, alength));
return rowdef.objectOrder.compare(a, astart, l, chunkcache, chunknumber * this.rowdef.objectsize() + this.rowdef.colstart[rowdef.primaryKey], this.rowdef.width(rowdef.primaryKey));
}
private boolean match(byte[] a, int astart, int alength, int chunknumber) {
if (chunknumber >= chunkcount) return false;
int i = 0;
int p = chunknumber * this.rowdef.objectsize() + this.rowdef.colstart[rowdef.primaryKey];
final int len = Math.min(this.rowdef.width(rowdef.primaryKey), Math.min(alength, a.length - astart));
while (i < len) if (a[astart + i++] != chunkcache[p++]) return false;
return ((len == this.rowdef.width(rowdef.primaryKey)) || (chunkcache[len] == 0)) ;
}
public kelondroProfile profile() {
return profile;
}
public Iterator rows() {
shape();
public synchronized Iterator rows() {
sort();
return super.rows();
}
public kelondroCloneableIterator rows(boolean up, byte[] firstKey) {
public synchronized kelondroCloneableIterator rows(boolean up, byte[] firstKey) {
return new rowIterator(up, firstKey);
}
@ -359,7 +222,7 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
public rowIterator(boolean up, byte[] firstKey) {
// see that all elements are sorted
shape();
sort();
this.up = up;
this.first = firstKey;
this.bound = sortBound;
@ -393,10 +256,6 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
}
}
public void close() {
// just for compatibility with kelondroIndex interface; do nothing
}
public static void main(String[] args) {
/*
String[] test = { "eins", "zwei", "drei", "vier", "fuenf", "sechs", "sieben", "acht", "neun", "zehn" };
@ -502,18 +361,18 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
c.put(c.rowdef.newEntry(new byte[][]{key, key}));
if (i % 1000 == 0) {
for (int j = 0; j < delkeys.length; j++) c.remove(delkeys[j]);
c.shape();
c.sort();
}
}
for (int j = 0; j < delkeys.length; j++) c.remove(delkeys[j]);
c.shape();
c.sort();
random = new Random(0);
for (int i = 0; i < testsize; i++) {
key = randomHash(random);
if (i % 5 == 0) continue;
if (c.get(key) == null) System.out.println("missing entry " + new String(key));
}
c.shape();
c.sort();
System.out.println("RESULT SIZE: " + c.size());
System.out.println("Time: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
}

2
source/de/anomic/plasma/plasmaSearchEvent.java
Unescape View File

@ -377,7 +377,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
long pst = System.currentTimeMillis();
searchResult.addAllUnique(rcLocal);
searchResult.addAllUnique(rcContainers);
searchResult.shape();
searchResult.sort();
searchResult.uniq();
preorderTime = preorderTime - (System.currentTimeMillis() - pst);
if (preorderTime < 0) preorderTime = 200;

Write Preview
Loading…
Cancel
Save