boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

redesigned remove method in kelondroRowSet

Browse Source 
This should fix also numerous bugs like
http://www.yacy-forum.de/viewtopic.php?p=31077#31077
(java.lang.ArrayIndexOutOfBoundsException in kelondroRowCollection.removeShift)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3476 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 18 years ago
parent 9f929b5438
commit 96b79bf86d
6 changed files with 117 additions and 252 deletions
Show Stats Download Patch File Download Diff File

8
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -597,7 +597,7 @@ public class kelondroCollectionIndex {
// join with new collection // join with new collection
oldcollection.addAllUnique(collection); oldcollection.addAllUnique(collection);
oldcollection.shape(); oldcollection.sort();
oldcollection.uniq(); // FIXME: not clear if it would be better to insert the collection with put to avoid double-entries oldcollection.uniq(); // FIXME: not clear if it would be better to insert the collection with put to avoid double-entries
oldcollection.trim(false); oldcollection.trim(false);
@ -714,7 +714,7 @@ public class kelondroCollectionIndex {
// join with new collection // join with new collection
oldcollection.addAllUnique(collection); oldcollection.addAllUnique(collection);
oldcollection.shape(); oldcollection.sort();
oldcollection.uniq(); // FIXME: not clear if it would be better to insert the collection with put to avoid double-entries oldcollection.uniq(); // FIXME: not clear if it would be better to insert the collection with put to avoid double-entries
oldcollection.trim(false); oldcollection.trim(false);
collection = oldcollection; collection = oldcollection;
@ -796,7 +796,7 @@ public class kelondroCollectionIndex {
private void saveCommons(byte[] key, kelondroRowSet collection) { private void saveCommons(byte[] key, kelondroRowSet collection) {
if (key.length != 12) return; if (key.length != 12) return;
collection.shape(); collection.sort();
TimeZone GMTTimeZone = TimeZone.getTimeZone("GMT"); TimeZone GMTTimeZone = TimeZone.getTimeZone("GMT");
Calendar gregorian = new GregorianCalendar(GMTTimeZone); Calendar gregorian = new GregorianCalendar(GMTTimeZone);
SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss"); SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
@ -844,7 +844,7 @@ public class kelondroCollectionIndex {
if ((k instanceof byte[]) && (oldcollection.remove((byte[]) k) != null)) removed++; if ((k instanceof byte[]) && (oldcollection.remove((byte[]) k) != null)) removed++;
if ((k instanceof String) && (oldcollection.remove(((String) k).getBytes()) != null)) removed++; if ((k instanceof String) && (oldcollection.remove(((String) k).getBytes()) != null)) removed++;
} }
oldcollection.shape(); oldcollection.sort();
oldcollection.trim(false); oldcollection.trim(false);
if (oldcollection.size() == 0) { if (oldcollection.size() == 0) {

2
source/de/anomic/kelondro/kelondroFlexTable.java
Unescape View File

@ -151,7 +151,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
} }
System.out.print(" -ordering- "); System.out.print(" -ordering- ");
System.out.flush(); System.out.flush();
ri.shape(); ri.sort();
return ri; return ri;
} }

2
source/de/anomic/kelondro/kelondroIntBytesMap.java
Unescape View File

@ -109,7 +109,7 @@ public class kelondroIntBytesMap {
public void flush() { public void flush() {
if (index instanceof kelondroRowSet) { if (index instanceof kelondroRowSet) {
((kelondroRowSet) index).shape(); ((kelondroRowSet) index).sort();
((kelondroRowSet) index).trim(true); ((kelondroRowSet) index).trim(true);
} }
} }

158
source/de/anomic/kelondro/kelondroRowCollection.java
Unescape View File

@ -39,7 +39,7 @@ public class kelondroRowCollection {
public static final double growfactor = 1.4; public static final double growfactor = 1.4;
protected byte[] chunkcache; private byte[] chunkcache;
protected int chunkcount; protected int chunkcount;
protected long lastTimeRead, lastTimeWrote; protected long lastTimeRead, lastTimeWrote;
protected kelondroRow rowdef; protected kelondroRow rowdef;
@ -135,7 +135,7 @@ public class kelondroRowCollection {
public static final int exportOverheadSize = 14; public static final int exportOverheadSize = 14;
public byte[] exportCollection() { public synchronized byte[] exportCollection() {
// returns null if the collection is empty // returns null if the collection is empty
trim(false); trim(false);
kelondroRow row = exportRow(chunkcache.length); kelondroRow row = exportRow(chunkcache.length);
@ -169,18 +169,24 @@ public class kelondroRowCollection {
newChunkcache = null; newChunkcache = null;
} }
public void trim(boolean plusGrowFactor) { public synchronized void trim(boolean plusGrowFactor) {
if (chunkcache.length == 0) return; if (chunkcache.length == 0)
synchronized (chunkcache) { return;
int needed = chunkcount * rowdef.objectsize(); int needed = chunkcount * rowdef.objectsize();
if (plusGrowFactor) needed = (int) (needed * growfactor); if (plusGrowFactor)
if (needed >= chunkcache.length) return; // in case that the growfactor causes that the cache would grow instead of shrink, simply ignore the growfactor needed = (int) (needed * growfactor);
if (serverMemory.available() + 1000 < needed) return; // if the swap buffer is not available, we must give up. This is not critical. Othervise we provoke a serious problem with OOM if (needed >= chunkcache.length)
byte[] newChunkcache = new byte[needed]; return; // in case that the growfactor causes that the cache would
System.arraycopy(chunkcache, 0, newChunkcache, 0, Math.min(chunkcache.length, newChunkcache.length)); // grow instead of shrink, simply ignore the growfactor
chunkcache = newChunkcache; if (serverMemory.available() + 1000 < needed)
newChunkcache = null; return; // if the swap buffer is not available, we must give up.
} // This is not critical. Othervise we provoke a serious
// problem with OOM
byte[] newChunkcache = new byte[needed];
System.arraycopy(chunkcache, 0, newChunkcache, 0, Math.min(
chunkcache.length, newChunkcache.length));
chunkcache = newChunkcache;
newChunkcache = null;
} }
public final long lastRead() { public final long lastRead() {
@ -191,16 +197,14 @@ public class kelondroRowCollection {
return lastTimeWrote; return lastTimeWrote;
} }
public final kelondroRow.Entry get(int index) { public synchronized final kelondroRow.Entry get(int index) {
assert (index >= 0) : "get: access with index " + index + " is below zero"; assert (index >= 0) : "get: access with index " + index + " is below zero";
assert (index < chunkcount) : "get: access with index " + index + " is above chunkcount " + chunkcount + "; sortBound = " + sortBound; assert (index < chunkcount) : "get: access with index " + index + " is above chunkcount " + chunkcount + "; sortBound = " + sortBound;
assert (index * rowdef.objectsize < chunkcache.length); assert (index * rowdef.objectsize < chunkcache.length);
if (index >= chunkcount) return null; if (index >= chunkcount) return null;
if (index * rowdef.objectsize() >= chunkcache.length) return null; if (index * rowdef.objectsize() >= chunkcache.length) return null;
byte[] a = new byte[rowdef.objectsize()]; byte[] a = new byte[rowdef.objectsize()];
synchronized (chunkcache) { System.arraycopy(chunkcache, index * rowdef.objectsize(), a, 0, rowdef.objectsize());
System.arraycopy(chunkcache, index * rowdef.objectsize(), a, 0, rowdef.objectsize());
}
this.lastTimeRead = System.currentTimeMillis(); this.lastTimeRead = System.currentTimeMillis();
return rowdef.newEntry(a); return rowdef.newEntry(a);
} }
@ -209,15 +213,13 @@ public class kelondroRowCollection {
set(index, a.bytes(), 0, a.bytes().length); set(index, a.bytes(), 0, a.bytes().length);
} }
public final void set(int index, byte[] a, int astart, int alength) { public synchronized final void set(int index, byte[] a, int astart, int alength) {
assert (index >= 0) : "get: access with index " + index + " is below zero"; assert (index >= 0) : "get: access with index " + index + " is below zero";
assert (index < chunkcount) : "get: access with index " + index + " is above chunkcount " + chunkcount; assert (index < chunkcount) : "get: access with index " + index + " is above chunkcount " + chunkcount;
assert (!(bugappearance(a, astart, alength))) : "a = " + serverLog.arrayList(a, astart, alength); assert (!(bugappearance(a, astart, alength))) : "a = " + serverLog.arrayList(a, astart, alength);
if (bugappearance(a, astart, alength)) return; // TODO: this is temporary; remote peers may still submit bad entries if (bugappearance(a, astart, alength)) return; // TODO: this is temporary; remote peers may still submit bad entries
int l = Math.min(rowdef.objectsize(), Math.min(alength, a.length - astart)); int l = Math.min(rowdef.objectsize(), Math.min(alength, a.length - astart));
synchronized (chunkcache) { System.arraycopy(a, astart, chunkcache, index * rowdef.objectsize(), l);
System.arraycopy(a, astart, chunkcache, index * rowdef.objectsize(), l);
}
this.lastTimeWrote = System.currentTimeMillis(); this.lastTimeWrote = System.currentTimeMillis();
} }
@ -238,7 +240,7 @@ public class kelondroRowCollection {
addUnique(a, 0, a.length); addUnique(a, 0, a.length);
} }
private final void addUnique(byte[] a, int astart, int alength) { private synchronized final void addUnique(byte[] a, int astart, int alength) {
assert (a != null); assert (a != null);
assert (astart >= 0) && (astart < a.length) : " astart = " + a; assert (astart >= 0) && (astart < a.length) : " astart = " + a;
assert (!(serverLog.allZero(a, astart, alength))) : "a = " + serverLog.arrayList(a, astart, alength); assert (!(serverLog.allZero(a, astart, alength))) : "a = " + serverLog.arrayList(a, astart, alength);
@ -250,11 +252,9 @@ public class kelondroRowCollection {
} }
assert (!(bugappearance(a, astart, alength))) : "a = " + serverLog.arrayList(a, astart, alength); assert (!(bugappearance(a, astart, alength))) : "a = " + serverLog.arrayList(a, astart, alength);
int l = Math.min(rowdef.objectsize(), Math.min(alength, a.length - astart)); int l = Math.min(rowdef.objectsize(), Math.min(alength, a.length - astart));
synchronized (chunkcache) { ensureSize(chunkcount + 1);
ensureSize(chunkcount + 1); System.arraycopy(a, astart, chunkcache, rowdef.objectsize() * chunkcount, l);
System.arraycopy(a, astart, chunkcache, rowdef.objectsize() * chunkcount, l); chunkcount++;
chunkcount++;
}
this.lastTimeWrote = System.currentTimeMillis(); this.lastTimeWrote = System.currentTimeMillis();
} }
@ -270,14 +270,12 @@ public class kelondroRowCollection {
return false; return false;
} }
public final void addAllUnique(kelondroRowCollection c) { public synchronized final void addAllUnique(kelondroRowCollection c) {
if (c == null) return; if (c == null) return;
assert(rowdef.objectsize() == c.rowdef.objectsize()); assert(rowdef.objectsize() == c.rowdef.objectsize());
synchronized(chunkcache) { ensureSize(chunkcount + c.size());
ensureSize(chunkcount + c.size()); System.arraycopy(c.chunkcache, 0, chunkcache, rowdef.objectsize() * chunkcount, rowdef.objectsize() * c.size());
System.arraycopy(c.chunkcache, 0, chunkcache, rowdef.objectsize() * chunkcount, rowdef.objectsize() * c.size()); chunkcount += c.size();
chunkcount += c.size();
}
} }
protected final void removeShift(int pos, int dist, int upBound) { protected final void removeShift(int pos, int dist, int upBound) {
@ -296,34 +294,28 @@ public class kelondroRowCollection {
System.arraycopy(chunkcache, this.rowdef.objectsize() * (chunkcount - 1), chunkcache, this.rowdef.objectsize() * i, this.rowdef.objectsize()); System.arraycopy(chunkcache, this.rowdef.objectsize() * (chunkcount - 1), chunkcache, this.rowdef.objectsize() * i, this.rowdef.objectsize());
} }
protected final void removeRow(int p) { protected synchronized final void removeRow(int p) {
assert ((p >= 0) && (p < chunkcount) && (chunkcount > 0)) : "p = " + p + ", chunkcount = " + chunkcount; assert ((p >= 0) && (p < chunkcount) && (chunkcount > 0)) : "p = " + p + ", chunkcount = " + chunkcount;
synchronized (chunkcache) { if (p < sortBound) {
if (p < sortBound) { removeShift(p, 1, chunkcount);
sortBound--; sortBound--;
removeShift(p, 1, chunkcount); } else {
chunkcount--; copytop(p);
} else {
copytop(p);
chunkcount--;
}
} }
chunkcount--;
this.lastTimeWrote = System.currentTimeMillis(); this.lastTimeWrote = System.currentTimeMillis();
} }
public kelondroRow.Entry removeOne() { public synchronized kelondroRow.Entry removeOne() {
synchronized (chunkcache) { if (chunkcount == 0) return null;
if (chunkcount == 0) return null; kelondroRow.Entry r = get(chunkcount - 1);
kelondroRow.Entry r = get(chunkcount - 1); if (chunkcount == sortBound) sortBound--;
if (chunkcount == sortBound) sortBound--; chunkcount--;
chunkcount--; this.lastTimeWrote = System.currentTimeMillis();
this.lastTimeWrote = System.currentTimeMillis(); return r;
return r;
}
} }
public void clear() { public synchronized void clear() {
this.chunkcache = new byte[0]; this.chunkcache = new byte[0];
this.chunkcount = 0; this.chunkcount = 0;
this.sortBound = 0; this.sortBound = 0;
@ -334,7 +326,7 @@ public class kelondroRowCollection {
return chunkcount; return chunkcount;
} }
public Iterator rows() { public synchronized Iterator rows() {
// iterates kelondroRow.Entry - type entries // iterates kelondroRow.Entry - type entries
return new rowIterator(); return new rowIterator();
} }
@ -361,20 +353,18 @@ public class kelondroRowCollection {
} }
} }
public void select(Set keys) { public synchronized void select(Set keys) {
// removes all entries but the ones given by urlselection // removes all entries but the ones given by urlselection
if ((keys == null) || (keys.size() == 0)) return; if ((keys == null) || (keys.size() == 0)) return;
synchronized (this) { Iterator i = rows();
Iterator i = rows(); kelondroRow.Entry row;
kelondroRow.Entry row; while (i.hasNext()) {
while (i.hasNext()) { row = (kelondroRow.Entry) i.next();
row = (kelondroRow.Entry) i.next(); if (!(keys.contains(row.getColString(0, null)))) i.remove();
if (!(keys.contains(row.getColString(0, null)))) i.remove();
}
} }
} }
protected final void sort() { public synchronized final void sort() {
assert (this.rowdef.objectOrder != null); assert (this.rowdef.objectOrder != null);
if (this.sortBound == this.chunkcount) return; // this is already sorted if (this.sortBound == this.chunkcount) return; // this is already sorted
//System.out.println("SORT(chunkcount=" + this.chunkcount + ", sortBound=" + this.sortBound + ")"); //System.out.println("SORT(chunkcount=" + this.chunkcount + ", sortBound=" + this.sortBound + ")");
@ -474,20 +464,17 @@ public class kelondroRowCollection {
if (i == p) return j; else if (j == p) return i; else return p; if (i == p) return j; else if (j == p) return i; else return p;
} }
public void uniq() { public synchronized void uniq() {
assert (this.rowdef.objectOrder != null); assert (this.rowdef.objectOrder != null);
// removes double-occurrences of chunks // removes double-occurrences of chunks
// this works only if the collection was ordered with sort before // this works only if the collection was ordered with sort before
synchronized (chunkcache) { if (chunkcount <= 1) return;
if (chunkcount <= 1) return; int i = 0;
int i = 0; while (i < chunkcount - 1) {
while (i < chunkcount - 1) { if (compare(i, i + 1) == 0) {
if (compare(i, i + 1) == 0) { removeRow(i);
//System.out.println("DOUBLE: " + new String(this.chunkcache, this.chunksize * i, this.chunksize)); } else {
removeRow(i); i++;
} else {
i++;
}
} }
} }
} }
@ -521,6 +508,25 @@ public class kelondroRowCollection {
return c; return c;
} }
protected int compare(byte[] a, int astart, int alength, int chunknumber) {
assert (chunknumber < chunkcount);
int l = Math.min(this.rowdef.width(rowdef.primaryKey), Math.min(a.length - astart, alength));
return rowdef.objectOrder.compare(a, astart, l, chunkcache, chunknumber * this.rowdef.objectsize() + this.rowdef.colstart[rowdef.primaryKey], this.rowdef.width(rowdef.primaryKey));
}
protected boolean match(byte[] a, int astart, int alength, int chunknumber) {
if (chunknumber >= chunkcount) return false;
int i = 0;
int p = chunknumber * this.rowdef.objectsize() + this.rowdef.colstart[rowdef.primaryKey];
final int len = Math.min(this.rowdef.width(rowdef.primaryKey), Math.min(alength, a.length - astart));
while (i < len) if (a[astart + i++] != chunkcache[p++]) return false;
return ((len == this.rowdef.width(rowdef.primaryKey)) || (chunkcache[len] == 0)) ;
}
public synchronized void close() {
chunkcache = null;
}
public static void main(String[] args) { public static void main(String[] args) {
System.out.println(new java.util.Date(10957 * day)); System.out.println(new java.util.Date(10957 * day));
System.out.println(new java.util.Date(0)); System.out.println(new java.util.Date(0));

195
source/de/anomic/kelondro/kelondroRowSet.java
Unescape View File

@ -29,87 +29,49 @@ import java.util.Date;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Random; import java.util.Random;
import java.util.TreeMap;
import de.anomic.server.logging.serverLog;
public class kelondroRowSet extends kelondroRowCollection implements kelondroIndex { public class kelondroRowSet extends kelondroRowCollection implements kelondroIndex {
private static final int collectionReSortLimit = 90; private static final int collectionReSortLimit = 90;
private static final int removeMaxSize = 100;
private kelondroProfile profile; private kelondroProfile profile;
private TreeMap removeMarker;
public kelondroRowSet(kelondroRow rowdef, int objectCount, byte[] cache, int sortBound) { public kelondroRowSet(kelondroRow rowdef, int objectCount, byte[] cache, int sortBound) {
super(rowdef, objectCount, cache, sortBound); super(rowdef, objectCount, cache, sortBound);
this.removeMarker = new TreeMap();
this.profile = new kelondroProfile(); this.profile = new kelondroProfile();
} }
public kelondroRowSet(kelondroRowSet rs) { public kelondroRowSet(kelondroRowSet rs) {
super(rs); super(rs);
this.profile = rs.profile; this.profile = rs.profile;
this.removeMarker = rs.removeMarker;
} }
public kelondroRowSet(kelondroRow rowdef, int objectCount) { public kelondroRowSet(kelondroRow rowdef, int objectCount) {
super(rowdef, objectCount); super(rowdef, objectCount);
this.removeMarker = new TreeMap();
this.profile = new kelondroProfile(); this.profile = new kelondroProfile();
} }
public kelondroRowSet(kelondroRow rowdef, byte[] exportedCollectionRowinstance) { public kelondroRowSet(kelondroRow rowdef, byte[] exportedCollectionRowinstance) {
super(rowdef, exportedCollectionRowinstance); super(rowdef, exportedCollectionRowinstance);
this.removeMarker = new TreeMap();
this.profile = new kelondroProfile(); this.profile = new kelondroProfile();
} }
public boolean has(byte[] key) throws IOException { public synchronized boolean has(byte[] key) throws IOException {
return (get(key) != null); return (get(key) != null);
} }
public kelondroRow.Entry get(byte[] key) { public synchronized kelondroRow.Entry get(byte[] key) {
return get(key, 0, key.length); return get(key, 0, key.length);
} }
private kelondroRow.Entry get(byte[] key, int astart, int alength) { private kelondroRow.Entry get(byte[] key, int astart, int alength) {
long handle = profile.startRead(); long handle = profile.startRead();
kelondroRow.Entry entry = null; int index = find(key, astart, alength);
synchronized (chunkcache) { kelondroRow.Entry entry = (index >= 0) ? get(index) : null;
int index = find(key, astart, alength);
if ((index >= 0) && (!(isMarkedRemoved(index)))) entry = get(index);
}
profile.stopRead(handle); profile.stopRead(handle);
return entry; return entry;
} }
public void addUnique(kelondroRow.Entry row) {
// add an entry without doing a double-occurrence test
if (removeMarker.size() == 0) {
super.addUnique(row);
} else {
this.put(row);
}
}
public void addUnique(kelondroRow.Entry row, Date entryDate) {
if (removeMarker.size() == 0) {
super.addUnique(row, entryDate);
} else {
this.put(row, entryDate);
}
}
public void addUniqueMultiple(List rows, Date entryDate) throws IOException {
if (removeMarker.size() == 0) {
super.addUniqueMultiple(rows, entryDate);
} else {
Iterator i = rows.iterator();
while (i.hasNext()) addUnique((kelondroRow.Entry) i.next(), entryDate);
}
}
public synchronized void putMultiple(List rows, Date entryDate) throws IOException { public synchronized void putMultiple(List rows, Date entryDate) throws IOException {
Iterator i = rows.iterator(); Iterator i = rows.iterator();
while (i.hasNext()) put((kelondroRow.Entry) i.next(), entryDate); while (i.hasNext()) put((kelondroRow.Entry) i.next(), entryDate);
@ -119,116 +81,34 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
return put(row); return put(row);
} }
public kelondroRow.Entry put(kelondroRow.Entry entry) { public synchronized kelondroRow.Entry put(kelondroRow.Entry entry) {
assert (entry != null); assert (entry != null);
assert (entry.getColBytes(rowdef.primaryKey) != null); assert (entry.getColBytes(rowdef.primaryKey) != null);
//assert (!(serverLog.allZero(entry.getColBytes(super.sortColumn)))); //assert (!(serverLog.allZero(entry.getColBytes(super.sortColumn))));
long handle = profile.startWrite(); long handle = profile.startWrite();
int index = -1; int index = -1;
kelondroRow.Entry oldentry = null; kelondroRow.Entry oldentry = null;
synchronized (chunkcache) { index = find(entry.bytes(), super.rowdef.colstart[rowdef.primaryKey], super.rowdef.width(rowdef.primaryKey));
index = find(entry.bytes(), super.rowdef.colstart[rowdef.primaryKey], super.rowdef.width(rowdef.primaryKey)); if (index < 0) {
if (isMarkedRemoved(index)) { super.addUnique(entry);
set(index, entry); } else {
removeMarker.remove(new Integer(index)); oldentry = get(index);
} else if (index < 0) { set(index, entry);
super.addUnique(entry);
} else {
oldentry = get(index);
set(index, entry);
}
} }
profile.stopWrite(handle); profile.stopWrite(handle);
return oldentry; return oldentry;
} }
public int size() { private synchronized kelondroRow.Entry remove(byte[] a, int start, int length) {
return super.size() - removeMarker.size(); int index = find(a, start, length);
if (index < 0) return null;
kelondroRow.Entry entry = super.get(index);
super.removeRow(index);
return entry;
} }
public kelondroRow.Entry remove(byte[] a) { public kelondroRow.Entry remove(byte[] a) {
return removeMarked(a, 0, a.length); return remove(a, 0, a.length);
}
private kelondroRow.Entry removeMarked(byte[] a, int astart, int alength) {
if (chunkcount == 0) return null;
long handle = profile.startDelete();
// check if it is contained in chunkcache
kelondroRow.Entry entry = null;
synchronized(chunkcache) {
int p = find(a, astart, alength);
if (p < 0) {
// the entry is not there
profile.stopDelete(handle);
return null;
} else {
// there is an entry
entry = get(p);
if (p < sortBound) {
// mark entry as to-be-deleted
removeMarker.put(new Integer(p), entry.getColBytes(rowdef.primaryKey));
if (removeMarker.size() > removeMaxSize) resolveMarkedRemoved();
} else {
// remove directly by swap
super.copytop(p);
chunkcount--;
}
profile.stopDelete(handle);
return entry;
}
}
}
private boolean isMarkedRemoved(int index) {
return removeMarker.containsKey(new Integer(index));
}
public void shape() {
assert (rowdef.objectOrder != null); // we cannot shape without an object order
synchronized (chunkcache) {
try {
resolveMarkedRemoved();
super.sort();
} catch (kelondroException e) {
// bad bug, cannot be fixed. We abandon all data
serverLog.logSevere("kelondroRowSet", "abandoned row");
this.clear();
}
//if (super.rowdef.column(0).cellwidth() == 4) System.out.println("TABLE OF " + super.rowdef.toString() + "\n" + serverLog.table(super.chunkcache, super.rowdef.objectsize, 0)); // DEBUG
}
}
private void resolveMarkedRemoved() {
if (removeMarker.size() == 0) return;
// check case when complete chunkcache is marked as deleted
if (removeMarker.size() == chunkcount) {
this.clear();
removeMarker.clear();
return;
}
Integer nxt = (Integer) removeMarker.firstKey();
removeMarker.remove(nxt);
int idx = nxt.intValue();
assert (idx < sortBound) : "idx = " + idx + ", sortBound = " + sortBound;
int d = 1;
byte[] a;
while (removeMarker.size() > 0) {
nxt = (Integer) removeMarker.firstKey();
a = (byte[]) removeMarker.remove(nxt);
assert kelondroNaturalOrder.compares(a, 0, a.length, get(nxt.intValue()).getColBytes(rowdef.primaryKey), 0, a.length) == 0;
assert (nxt.intValue() < sortBound);
super.removeShift(idx, d, nxt.intValue());
idx = nxt.intValue() - d;
d++;
}
super.removeShift(idx, d, chunkcount);
chunkcount -= d;
sortBound -= d; // there are all markers below the sortBound
removeMarker.clear();
} }
public void removeMarkedAll(kelondroRowCollection c) { public void removeMarkedAll(kelondroRowCollection c) {
@ -237,7 +117,7 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
kelondroRow.Entry entry; kelondroRow.Entry entry;
while (i.hasNext()) { while (i.hasNext()) {
entry = (kelondroRow.Entry) i.next(); entry = (kelondroRow.Entry) i.next();
removeMarked(entry.bytes(), 0, entry.bytes().length); remove(entry.bytes(), 0, entry.bytes().length);
} }
profile.stopDelete(handle); profile.stopDelete(handle);
} }
@ -246,9 +126,7 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
if ((rowdef.objectOrder == null) || if ((rowdef.objectOrder == null) ||
(!(rowdef.objectOrder.signature().equals(newOrder.signature()))) || (!(rowdef.objectOrder.signature().equals(newOrder.signature()))) ||
(newColumn != rowdef.primaryKey)) { (newColumn != rowdef.primaryKey)) {
resolveMarkedRemoved();
rowdef.setOrdering(newOrder, newColumn); rowdef.setOrdering(newOrder, newColumn);
assert (removeMarker.size() == 0);
this.sortBound = 0; this.sortBound = 0;
} }
} }
@ -259,7 +137,7 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
if (rowdef.objectOrder == null) return iterativeSearch(a, astart, alength, 0, this.chunkcount); if (rowdef.objectOrder == null) return iterativeSearch(a, astart, alength, 0, this.chunkcount);
// check if a re-sorting make sense // check if a re-sorting make sense
if ((this.chunkcount - this.sortBound) > collectionReSortLimit) shape(); if ((this.chunkcount - this.sortBound) > collectionReSortLimit) sort();
// first try to find in sorted area // first try to find in sorted area
int p = binarySearch(a, astart, alength); int p = binarySearch(a, astart, alength);
@ -323,31 +201,16 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
return l; return l;
} }
private int compare(byte[] a, int astart, int alength, int chunknumber) {
assert (chunknumber < chunkcount);
int l = Math.min(this.rowdef.width(rowdef.primaryKey), Math.min(a.length - astart, alength));
return rowdef.objectOrder.compare(a, astart, l, chunkcache, chunknumber * this.rowdef.objectsize() + this.rowdef.colstart[rowdef.primaryKey], this.rowdef.width(rowdef.primaryKey));
}
private boolean match(byte[] a, int astart, int alength, int chunknumber) {
if (chunknumber >= chunkcount) return false;
int i = 0;
int p = chunknumber * this.rowdef.objectsize() + this.rowdef.colstart[rowdef.primaryKey];
final int len = Math.min(this.rowdef.width(rowdef.primaryKey), Math.min(alength, a.length - astart));
while (i < len) if (a[astart + i++] != chunkcache[p++]) return false;
return ((len == this.rowdef.width(rowdef.primaryKey)) || (chunkcache[len] == 0)) ;
}
public kelondroProfile profile() { public kelondroProfile profile() {
return profile; return profile;
} }
public Iterator rows() { public synchronized Iterator rows() {
shape(); sort();
return super.rows(); return super.rows();
} }
public kelondroCloneableIterator rows(boolean up, byte[] firstKey) { public synchronized kelondroCloneableIterator rows(boolean up, byte[] firstKey) {
return new rowIterator(up, firstKey); return new rowIterator(up, firstKey);
} }
@ -359,7 +222,7 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
public rowIterator(boolean up, byte[] firstKey) { public rowIterator(boolean up, byte[] firstKey) {
// see that all elements are sorted // see that all elements are sorted
shape(); sort();
this.up = up; this.up = up;
this.first = firstKey; this.first = firstKey;
this.bound = sortBound; this.bound = sortBound;
@ -393,10 +256,6 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
} }
} }
public void close() {
// just for compatibility with kelondroIndex interface; do nothing
}
public static void main(String[] args) { public static void main(String[] args) {
/* /*
String[] test = { "eins", "zwei", "drei", "vier", "fuenf", "sechs", "sieben", "acht", "neun", "zehn" }; String[] test = { "eins", "zwei", "drei", "vier", "fuenf", "sechs", "sieben", "acht", "neun", "zehn" };
@ -502,18 +361,18 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
c.put(c.rowdef.newEntry(new byte[][]{key, key})); c.put(c.rowdef.newEntry(new byte[][]{key, key}));
if (i % 1000 == 0) { if (i % 1000 == 0) {
for (int j = 0; j < delkeys.length; j++) c.remove(delkeys[j]); for (int j = 0; j < delkeys.length; j++) c.remove(delkeys[j]);
c.shape(); c.sort();
} }
} }
for (int j = 0; j < delkeys.length; j++) c.remove(delkeys[j]); for (int j = 0; j < delkeys.length; j++) c.remove(delkeys[j]);
c.shape(); c.sort();
random = new Random(0); random = new Random(0);
for (int i = 0; i < testsize; i++) { for (int i = 0; i < testsize; i++) {
key = randomHash(random); key = randomHash(random);
if (i % 5 == 0) continue; if (i % 5 == 0) continue;
if (c.get(key) == null) System.out.println("missing entry " + new String(key)); if (c.get(key) == null) System.out.println("missing entry " + new String(key));
} }
c.shape(); c.sort();
System.out.println("RESULT SIZE: " + c.size()); System.out.println("RESULT SIZE: " + c.size());
System.out.println("Time: " + ((System.currentTimeMillis() - start) / 1000) + " seconds"); System.out.println("Time: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
} }

2
source/de/anomic/plasma/plasmaSearchEvent.java
Unescape View File

@ -377,7 +377,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
long pst = System.currentTimeMillis(); long pst = System.currentTimeMillis();
searchResult.addAllUnique(rcLocal); searchResult.addAllUnique(rcLocal);
searchResult.addAllUnique(rcContainers); searchResult.addAllUnique(rcContainers);
searchResult.shape(); searchResult.sort();
searchResult.uniq(); searchResult.uniq();
preorderTime = preorderTime - (System.currentTimeMillis() - pst); preorderTime = preorderTime - (System.currentTimeMillis() - pst);
if (preorderTime < 0) preorderTime = 200; if (preorderTime < 0) preorderTime = 200;

Write Preview
Loading…
Cancel
Save