boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

optimized memory allocation in kelondroRow.Entry

Browse Source 
such an entry cannot be instantiated without allocation of new byte[]; instead
it can re-use memory from other kelondroRow.Entry objects.
during bugfixing also other bugs may have been solved, maybe the INCONSISTENCY problem
could have been solved. One cause can be missing synchronization during bulk storage
when a R/W-path optimization is done. To test this case, the optimization is currently
switched off.
More memory enhancements can be done after this initial change to the allocation scheme.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3536 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 18 years ago
parent 24ea4ca631
commit ba2c307ab3
19 changed files with 387 additions and 297 deletions
Show Stats Download Patch File Download Diff File

3
htroot/yacysearch.java
Unescape View File

@ -178,7 +178,8 @@ public class yacysearch {
// SEARCH
final boolean indexDistributeGranted = sb.getConfig("allowDistributeIndex", "true").equals("true");
final boolean indexReceiveGranted = sb.getConfig("allowReceiveIndex", "true").equals("true");
if (!indexDistributeGranted || !indexReceiveGranted) { global = false; }
final boolean offline = yacyCore.seedDB.mySeed.isVirgin();
if (offline || !indexDistributeGranted || !indexReceiveGranted) { global = false; }
// find search domain
int contentdomCode = plasmaSearchQuery.CONTENTDOM_TEXT;

3
source/de/anomic/index/indexCollectionRI.java
Unescape View File

@ -185,7 +185,8 @@ public class indexCollectionRI implements indexRI {
public synchronized void addMultipleEntries(List /*of indexContainer*/ containerList) {
try {
collectionIndex.mergeMultiple(containerList);
for (int i = 0; i < containerList.size(); i++) collectionIndex.merge((indexContainer) containerList.get(i));
//synchronized (containerList) {collectionIndex.mergeMultiple(containerList);}
} catch (kelondroOutOfLimitsException e) {
e.printStackTrace();
} catch (IOException e) {

4
source/de/anomic/index/indexRAMRI.java
Unescape View File

@ -204,11 +204,11 @@ public final class indexRAMRI implements indexRI {
String wordHash;
//long creationTime;
indexRWIEntry wordEntry;
kelondroRow.Entry row;
kelondroRow.EntryIndex row;
//Runtime rt = Runtime.getRuntime();
while (i.hasNext()) {
// get out one entry
row = (kelondroRow.Entry) i.next();
row = (kelondroRow.EntryIndex) i.next();
if ((row == null) || (row.empty(0)) || (row.empty(3))) continue;
wordHash = row.getColString(0, "UTF-8");
//creationTime = kelondroRecords.bytes2long(row[2]);

20
source/de/anomic/kelondro/kelondroAbstractIOChunks.java
Unescape View File

@ -65,7 +65,7 @@ public abstract class kelondroAbstractIOChunks {
// derived methods:
public void readFully(long pos, byte[] b, int off, int len) throws IOException {
public synchronized void readFully(long pos, byte[] b, int off, int len) throws IOException {
long handle = profile.startRead();
if (len < 0) throw new IndexOutOfBoundsException("length is negative:" + len);
if (b.length < off + len) throw new IndexOutOfBoundsException("bounds do not fit: b.length=" + b.length + ", off=" + off + ", len=" + len);
@ -80,37 +80,37 @@ public abstract class kelondroAbstractIOChunks {
profile.stopRead(handle);
}
public byte readByte(long pos) throws IOException {
public synchronized byte readByte(long pos) throws IOException {
byte[] b = new byte[1];
this.readFully(pos, b, 0, 1);
return b[0];
}
public void writeByte(long pos, final int v) throws IOException {
public synchronized void writeByte(long pos, final int v) throws IOException {
long handle = profile.startWrite();
this.write(pos, new byte[]{(byte) (v & 0xFF)});
profile.stopWrite(handle);
}
public short readShort(long pos) throws IOException {
public synchronized short readShort(long pos) throws IOException {
byte[] b = new byte[2];
this.readFully(pos, b, 0, 2);
return (short) ((((int) b[0] & 0xFF) << 8) | (((int) b[1] & 0xFF) << 0));
}
public void writeShort(long pos, final int v) throws IOException {
public synchronized void writeShort(long pos, final int v) throws IOException {
long handle = profile.startWrite();
this.write(pos, new byte[]{(byte) ((v >>> 8) & 0xFF), (byte) ((v >>> 0) & 0xFF)});
profile.stopWrite(handle);
}
public int readInt(long pos) throws IOException {
public synchronized int readInt(long pos) throws IOException {
byte[] b = new byte[4];
this.readFully(pos, b, 0, 4);
return (((int) b[0] & 0xFF) << 24) | (((int) b[1] & 0xFF) << 16) | (((int) b[2] & 0xFF) << 8) | ((int) b[3] & 0xFF);
}
public void writeInt(long pos, final int v) throws IOException {
public synchronized void writeInt(long pos, final int v) throws IOException {
long handle = profile.startWrite();
this.write(pos, new byte[]{
(byte) ((v >>> 24) & 0xFF),
@ -121,13 +121,13 @@ public abstract class kelondroAbstractIOChunks {
profile.stopWrite(handle);
}
public long readLong(long pos) throws IOException {
public synchronized long readLong(long pos) throws IOException {
byte[] b = new byte[8];
this.readFully(pos, b, 0, 8);
return (((long) b[0] & 0xFF) << 56) | (((long) b[1] & 0xFF) << 48) | (((long) b[2]) << 40) | (((long) b[3] & 0xFF) << 32) | (((long) b[4] & 0xFF) << 24) | (((long) b[5] & 0xFF) << 16) | (((long) b[6] & 0xFF) << 8) | ((long) b[7] & 0xFF);
}
public void writeLong(long pos, final long v) throws IOException {
public synchronized void writeLong(long pos, final long v) throws IOException {
long handle = profile.startWrite();
this.write(pos, new byte[]{
(byte) ((v >>> 56) & 0xFF),
@ -142,7 +142,7 @@ public abstract class kelondroAbstractIOChunks {
profile.stopWrite(handle);
}
public void write(long pos, final byte[] b) throws IOException {
public synchronized void write(long pos, final byte[] b) throws IOException {
long handle = profile.startWrite();
this.write(pos, b, 0, b.length);
profile.stopWrite(handle);

8
source/de/anomic/kelondro/kelondroBufferedIOChunks.java
Unescape View File

@ -74,7 +74,7 @@ public final class kelondroBufferedIOChunks extends kelondroAbstractIOChunks imp
return ra.length();
}
public int read(long pos, byte[] b, int off, int len) throws IOException {
public synchronized int read(long pos, byte[] b, int off, int len) throws IOException {
assert (b.length >= off + len): "read pos=" + pos + ", b.length=" + b.length + ", off=" + off + ", len=" + len;
// check commit time
@ -106,7 +106,7 @@ public final class kelondroBufferedIOChunks extends kelondroAbstractIOChunks imp
}
}
public void write(long pos, byte[] b, int off, int len) throws IOException {
public synchronized void write(long pos, byte[] b, int off, int len) throws IOException {
assert (b.length >= off + len): "write pos=" + pos + ", b.length=" + b.length + ", b='" + new String(b) + "', off=" + off + ", len=" + len;
//if (len > 10) System.out.println("WRITE(" + name + ", " + pos + ", " + b.length + ", " + off + ", " + len + ")");
@ -127,7 +127,7 @@ public final class kelondroBufferedIOChunks extends kelondroAbstractIOChunks imp
}
}
public void commit() throws IOException {
public synchronized void commit() throws IOException {
synchronized (buffer) {
if (buffer.size() == 0) return;
Iterator i = buffer.entrySet().iterator();
@ -170,7 +170,7 @@ public final class kelondroBufferedIOChunks extends kelondroAbstractIOChunks imp
}
}
public void close() throws IOException {
public synchronized void close() throws IOException {
if (this.ra != null) {
commit();
this.ra.close();

40
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -177,7 +177,7 @@ public class kelondroCollectionIndex {
ientry = irow.newEntry();
ientry.setCol(idx_col_key, key);
ientry.setCol(idx_col_chunksize, chunksize);
ientry.setCol(idx_col_chunkcount, kelondroRowCollection.sizeOfExportedCollectionRows(this.payloadrow, aentry.getColBytes(1)));
ientry.setCol(idx_col_chunkcount, kelondroRowCollection.sizeOfExportedCollectionRows(aentry, 1));
ientry.setCol(idx_col_clusteridx, (byte) partitionNumber);
ientry.setCol(idx_col_flags, (byte) 0);
ientry.setCol(idx_col_indexpos, aentry.index());
@ -207,7 +207,7 @@ public class kelondroCollectionIndex {
if (propfile.exists()) {
props = serverFileUtils.loadHashMap(propfile);
String stored_rowdef = (String) props.get("rowdef");
if ((stored_rowdef == null) || (!(rowdef.subsumes(new kelondroRow(stored_rowdef, null, 0))))) {
if ((stored_rowdef == null) || (!(rowdef.subsumes(new kelondroRow(stored_rowdef, rowdef.objectOrder, 0))))) {
System.out.println("FATAL ERROR: stored rowdef '" + stored_rowdef + "' does not match with new rowdef '" +
rowdef + "' for array cluster '" + path + "/" + filenameStub + "'");
System.exit(-1);
@ -414,11 +414,12 @@ public class kelondroCollectionIndex {
array.set(rowNumber, arrayEntry);
// update the index entry
indexrow.setCol(idx_col_chunkcount, collection.size());
final int collectionsize = collection.size(); // extra variable for easier debugging
indexrow.setCol(idx_col_chunkcount, collectionsize);
indexrow.setCol(idx_col_clusteridx, (byte) partitionNumber);
indexrow.setCol(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
// after calling this method there mus be a index.put(indexrow);
// after calling this method there must be a index.put(indexrow);
}
private ArrayList array_replace_multiple(TreeMap array_replace_map, int serialNumber, int chunkSize) throws IOException {
@ -515,6 +516,10 @@ public class kelondroCollectionIndex {
newPartitionNumber, serialNumber, this.payloadrow.objectsize()); // modifies indexrow
}
arrayResolveRemoved(); // remove all to-be-removed marked entries
if ((int) indexrow.getColLong(idx_col_chunkcount) != collection.size())
serverLog.logSevere("kelondroCollectionIndex", "UPDATE (put) ERROR: array has different chunkcount than index after merge: index = " + (int) indexrow.getColLong(idx_col_chunkcount) + ", collection.size() = " + collection.size());
index.put(indexrow); // write modified indexrow
}
@ -567,6 +572,7 @@ public class kelondroCollectionIndex {
ArrayList actionList;
TreeMap actionMap;
boolean madegc = false;
System.out.println("DEBUG existingContainer: " + existingContainer.toString());
while (existingContainer.size() > 0) {
oldPartitionNumber1 = ((Integer) existingContainer.lastKey()).intValue();
containerMap = (TreeMap) existingContainer.remove(new Integer(oldPartitionNumber1));
@ -586,8 +592,8 @@ public class kelondroCollectionIndex {
int oldchunkcount = (int) indexrow.getColLong(idx_col_chunkcount); // the number if rows in the collection
int oldrownumber = (int) indexrow.getColLong(idx_col_indexpos); // index of the entry in array
int oldPartitionNumber = (int) indexrow.getColByte(idx_col_clusteridx); // points to array file
assert oldPartitionNumber1 == oldPartitionNumber;
assert oldrownumber1 == oldrownumber;
assert oldPartitionNumber1 == oldPartitionNumber : "oldPartitionNumber1 = " + oldPartitionNumber1 + ", oldPartitionNumber = " + oldPartitionNumber + ", containerMap = " + containerMap + ", existingContainer: " + existingContainer.toString();
assert oldrownumber1 == oldrownumber : "oldrownumber1 = " + oldrownumber1 + ", oldrownumber = " + oldrownumber + ", containerMap = " + containerMap + ", existingContainer: " + existingContainer.toString();
assert (oldPartitionNumber >= arrayIndex(oldchunkcount));
int oldSerialNumber = 0;
@ -705,7 +711,7 @@ public class kelondroCollectionIndex {
int oldchunkcount = (int) indexrow.getColLong(idx_col_chunkcount); // the number if rows in the collection
int oldrownumber = (int) indexrow.getColLong(idx_col_indexpos); // index of the entry in array
int oldPartitionNumber = (int) indexrow.getColByte(idx_col_clusteridx); // points to array file
assert (oldPartitionNumber >= arrayIndex(oldchunkcount));
assert (oldPartitionNumber >= arrayIndex(oldchunkcount)) : "oldPartitionNumber = " + oldPartitionNumber + ", arrayIndex(oldchunkcount) = " + arrayIndex(oldchunkcount);
int oldSerialNumber = 0;
// load the old collection and join it
@ -717,7 +723,7 @@ public class kelondroCollectionIndex {
oldcollection.uniq(); // FIXME: not clear if it would be better to insert the collection with put to avoid double-entries
oldcollection.trim(false);
collection = oldcollection;
// check for size of collection:
// if necessary shrink the collection and dump a part of that collection
// to avoid that this grows too big
@ -747,6 +753,12 @@ public class kelondroCollectionIndex {
newPartitionNumber, oldSerialNumber, this.payloadrow.objectsize()); // modifies indexrow
}
arrayResolveRemoved(); // remove all to-be-removed marked entries
final int collectionsize = collection.size(); // extra variable for easier debugging
final int indexrowcount = (int) indexrow.getColLong(idx_col_chunkcount);
if (indexrowcount != collectionsize)
serverLog.logSevere("kelondroCollectionIndex", "UPDATE (merge) ERROR: array has different chunkcount than index after merge: index = " + indexrowcount + ", collection.size() = " + collectionsize);
index.put(indexrow); // write modified indexrow
}
}
@ -911,13 +923,13 @@ public class kelondroCollectionIndex {
int chunkcount = (int) indexrow.getColLong(idx_col_chunkcount);
int rownumber = (int) indexrow.getColLong(idx_col_indexpos);
int partitionnumber = (int) indexrow.getColByte(idx_col_clusteridx);
assert(partitionnumber >= arrayIndex(chunkcount));
assert(partitionnumber >= arrayIndex(chunkcount)) : "partitionnumber = " + partitionnumber + ", arrayIndex(chunkcount) = " + arrayIndex(chunkcount);
int serialnumber = 0;
return getwithparams(indexrow, chunksize, chunkcount, partitionnumber, rownumber, serialnumber, remove);
}
private kelondroRowSet getwithparams(kelondroRow.Entry indexrow, int chunksize, int chunkcount, int clusteridx, int rownumber, int serialnumber, boolean remove) throws IOException {
private synchronized kelondroRowSet getwithparams(kelondroRow.Entry indexrow, int chunksize, int chunkcount, int clusteridx, int rownumber, int serialnumber, boolean remove) throws IOException {
// open array entry
kelondroFixedWidthArray array = getArray(clusteridx, serialnumber, chunksize);
kelondroRow.Entry arrayrow = array.get(rownumber);
@ -933,7 +945,7 @@ public class kelondroCollectionIndex {
serverLog.logSevere("kelondroCollectionIndex." + array.filename, "lost a RowCollection because of a bad arraykey");
return new kelondroRowSet(this.payloadrow, 0);
}
kelondroRowSet collection = new kelondroRowSet(this.payloadrow, arrayrow.getColBytes(1)); // FIXME: this does not yet work with different rowdef in case of several rowdef.objectsize()
kelondroRowSet collection = new kelondroRowSet(this.payloadrow, arrayrow, 1); // FIXME: this does not yet work with different rowdef in case of several rowdef.objectsize()
if ((!(index.row().objectOrder.wellformed(indexkey))) || (index.row().objectOrder.compare(arraykey, indexkey) != 0)) {
// check if we got the right row; this row is wrong. Fix it:
index.remove(indexkey); // the wrong row cannot be fixed
@ -955,7 +967,7 @@ public class kelondroCollectionIndex {
// fix the entry in index
indexrow.setCol(idx_col_chunkcount, chunkcountInArray);
index.put(indexrow);
array.logFailure("INCONSISTENCY in " + arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString() + ": array has different chunkcount than index: index = " + chunkcount + ", array = " + chunkcountInArray + "; the index has been auto-fixed");
array.logFailure("INCONSISTENCY (get) in " + arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString() + ": array has different chunkcount than index: index = " + chunkcount + ", array = " + chunkcountInArray + "; the index has been auto-fixed");
}
if (remove) array.remove(rownumber, false); // index is removed in calling method
return collection;
@ -1031,7 +1043,7 @@ public class kelondroCollectionIndex {
collection.addUnique(rowdef.newEntry(new byte[][]{"abc".getBytes(), "efg".getBytes()}));
collectionIndex.put("erstes".getBytes(), collection);
for (int i = 0; i <= 17; i++) {
for (int i = 0; i <= 170; i++) {
collection = new kelondroRowSet(rowdef, 0);
for (int j = 0; j < i; j++) {
collection.addUnique(rowdef.newEntry(new byte[][]{("abc" + j).getBytes(), "xxx".getBytes()}));
@ -1041,7 +1053,7 @@ public class kelondroCollectionIndex {
}
// extend collections with more values
for (int i = 0; i <= 17; i++) {
for (int i = 0; i <= 170; i++) {
collection = new kelondroRowSet(rowdef, 0);
for (int j = 0; j < i; j++) {
collection.addUnique(rowdef.newEntry(new byte[][]{("def" + j).getBytes(), "xxx".getBytes()}));

8
source/de/anomic/kelondro/kelondroFixedWidthArray.java
Unescape View File

@ -124,9 +124,15 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
set(((Integer) entry.getKey()).intValue(), (kelondroRow.Entry) entry.getValue());
}
}
public synchronized kelondroRow.Entry getIfValid(int index) throws IOException {
byte[] b = getNode(new Handle(index), true).getValueRow();
if (b[0] == 0) return null;
return row().newEntry(b);
}
public synchronized kelondroRow.Entry get(int index) throws IOException {
return row().newEntry(getNode(new Handle(index), true).getValueRow());
return row().newEntry(getNode(new Handle(index), true).getValueRow());
}
protected synchronized int seti(int index, int value) throws IOException {

4
source/de/anomic/kelondro/kelondroFlexSplitTable.java
Unescape View File

@ -189,7 +189,7 @@ public class kelondroFlexSplitTable implements kelondroIndex {
}
public synchronized kelondroRow.Entry put(kelondroRow.Entry row, Date entryDate) throws IOException {
assert row.bytes().length <= this.rowdef.objectsize;
assert row.objectsize() <= this.rowdef.objectsize;
Object[] keeper = keeperOf(row.getColBytes(0));
if (keeper != null) return ((kelondroIndex) keeper[0]).put(row);
if ((entryDate == null) || (entryDate.after(new Date()))) entryDate = new Date(); // fix date
@ -222,7 +222,7 @@ public class kelondroFlexSplitTable implements kelondroIndex {
}
public synchronized void addUnique(kelondroRow.Entry row, Date entryDate) throws IOException {
assert row.bytes().length <= this.rowdef.objectsize;
assert row.objectsize() <= this.rowdef.objectsize;
if ((entryDate == null) || (entryDate.after(new Date()))) entryDate = new Date(); // fix date
String suffix = dateSuffix(entryDate);
if (suffix == null) return;

7
source/de/anomic/kelondro/kelondroFlexTable.java
Unescape View File

@ -228,7 +228,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
public synchronized kelondroRow.Entry put(kelondroRow.Entry row) throws IOException {
assert (row != null);
assert (!(serverLog.allZero(row.getColBytes(0))));
assert row.bytes().length <= this.rowdef.objectsize;
assert row.objectsize() <= this.rowdef.objectsize;
byte[] key = row.getColBytes(0);
int pos = RWindex.geti(key);
if ((pos < 0) && (ROindex != null)) pos = ROindex.geti(key);
@ -246,7 +246,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
}
public synchronized void addUnique(kelondroRow.Entry row) throws IOException {
assert row.bytes().length == this.rowdef.objectsize;
assert row.objectsize() == this.rowdef.objectsize;
RWindex.addi(row.getColBytes(0), super.add(row));
}
@ -390,8 +390,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
serverLog.logWarning("kelondroFlexTable", "close(): file '" + this.filename + "' was not tracked with record tracker.");
}
if (ROindex != null) {ROindex.close(); ROindex = null;}
RWindex.close();
RWindex = null;
if (RWindex != null) {RWindex.close(); RWindex = null;}
super.close();
}

57
source/de/anomic/kelondro/kelondroFlexWidthArray.java
Unescape View File

@ -70,7 +70,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
if (propfile.exists()) {
props = serverFileUtils.loadHashMap(propfile);
String stored_rowdef = (String) props.get("rowdef");
if ((stored_rowdef == null) || (!(rowdef.subsumes(new kelondroRow(stored_rowdef, null, 0))))) {
if ((stored_rowdef == null) || (!(rowdef.subsumes(new kelondroRow(stored_rowdef, rowdef.objectOrder, 0))))) {
System.out.println("FATAL ERROR: stored rowdef '" + stored_rowdef + "' does not match with new rowdef '" +
rowdef + "' for flex table '" + path + "', table " + tablename);
System.exit(-1);
@ -88,7 +88,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
kelondroColumn columns[] = new kelondroColumn[colend - colstart + 1];
for (int j = colstart; j <= colend; j++) columns[j-colstart] = rowdef.column(j);
col[colstart] = new kelondroFixedWidthArray(new File(tabledir, files[i]), new kelondroRow(columns, null, 0), 16);
col[colstart] = new kelondroFixedWidthArray(new File(tabledir, files[i]), new kelondroRow(columns, (colstart == 0) ? rowdef.objectOrder : kelondroNaturalOrder.naturalOrder, 0), 16);
for (int j = colstart; j <= colend; j++) check = check.substring(0, j) + "X" + check.substring(j + 1);
}
}
@ -107,7 +107,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
columns[j - p] = rowdef.column(j);
check = check.substring(0, j) + "X" + check.substring(j + 1);
}
col[p] = new kelondroFixedWidthArray(new File(tabledir, colfilename(p, q)), new kelondroRow(columns, null, 0), 16);
col[p] = new kelondroFixedWidthArray(new File(tabledir, colfilename(p, q)), new kelondroRow(columns, (p == 0) ? rowdef.objectOrder : kelondroNaturalOrder.naturalOrder, 0), 16);
}
}
@ -196,22 +196,18 @@ public class kelondroFlexWidthArray implements kelondroArray {
Iterator i;
Map.Entry entry;
kelondroRow.Entry rowentry, e;
int c = 0, index, lastcol;
int c = 0, index;
synchronized (col) {
// go across each file
while (c < rowdef.columns()) {
i = entries.entrySet().iterator();
lastcol = c + col[c].row().columns() - 1;
while (i.hasNext()) {
entry = (Map.Entry) i.next();
index = ((Integer) entry.getKey()).intValue();
rowentry = (kelondroRow.Entry) entry.getValue();
assert rowentry.bytes().length == this.rowdef.objectsize;
assert rowentry.objectsize() == this.rowdef.objectsize;
e = col[c].row().newEntry(
rowentry.bytes(),
rowdef.colstart[c],
rowdef.colstart[lastcol] - rowdef.colstart[c] + rowdef.width(lastcol));
e = col[c].row().newEntry(rowentry.bytes(), rowdef.colstart[c]);
col[c].set(index, e);
}
c = c + col[c].row().columns();
@ -220,15 +216,13 @@ public class kelondroFlexWidthArray implements kelondroArray {
}
public void set(int index, kelondroRow.Entry rowentry) throws IOException {
assert rowentry.bytes().length == this.rowdef.objectsize;
assert rowentry.objectsize() == this.rowdef.objectsize;
int c = 0;
kelondroRow.Entry e;
synchronized (col) {
byte[] reb = rowentry.bytes();
while (c < rowdef.columns()) {
e = col[c].row().newEntry(
rowentry.bytes(),
rowdef.colstart[c],
col[c].row().objectsize());
e = col[c].row().newEntry(reb, rowdef.colstart[c]);
col[c].set(index, e);
c = c + col[c].row().columns();
}
@ -236,21 +230,17 @@ public class kelondroFlexWidthArray implements kelondroArray {
}
public int add(kelondroRow.Entry rowentry) throws IOException {
assert rowentry.bytes().length == this.rowdef.objectsize;
assert rowentry.objectsize() == this.rowdef.objectsize;
kelondroRow.Entry e;
int index = -1;
int lastcol;
byte[] reb = rowentry.bytes();
synchronized (col) {
e = col[0].row().newEntry(rowentry.bytes(), 0, rowdef.width(0));
e = col[0].row().newEntry(reb, 0);
index = col[0].add(e);
int c = col[0].row().columns();
while (c < rowdef.columns()) {
lastcol = c + col[c].row().columns() - 1;
e = col[c].row().newEntry(
rowentry.bytes(),
rowdef.colstart[c],
rowdef.colstart[lastcol] + rowdef.width(lastcol) - rowdef.colstart[c]);
e = col[c].row().newEntry(reb, rowdef.colstart[c]);
col[c].set(index, e);
c = c + col[c].row().columns();
}
@ -272,22 +262,18 @@ public class kelondroFlexWidthArray implements kelondroArray {
i = rows.iterator();
while (i.hasNext()) {
rowentry = (kelondroRow.Entry) i.next();
assert rowentry.bytes().length == this.rowdef.objectsize;
assert rowentry.objectsize() == this.rowdef.objectsize;
kelondroRow.Entry e;
int index = -1;
int lastcol;
byte[] reb = rowentry.bytes();
synchronized (col) {
e = col[0].row().newEntry(rowentry.bytes(), 0, rowdef.width(0));
e = col[0].row().newEntry(reb, 0);
index = col[0].add(e);
int c = col[0].row().columns();
while (c < rowdef.columns()) {
lastcol = c + col[c].row().columns() - 1;
e = col[c].row().newEntry(
rowentry.bytes(),
rowdef.colstart[c],
rowdef.colstart[lastcol] + rowdef.width(lastcol) - rowdef.colstart[c]);
e = col[c].row().newEntry(reb, rowdef.colstart[c]);
// remember write to column, but do not write directly
colm[c].put(new Integer(index), e); // col[c].set(index,e);
c = c + col[c].row().columns();
@ -309,9 +295,14 @@ public class kelondroFlexWidthArray implements kelondroArray {
p = rowdef.newEntry();
synchronized (col) {
while (r < rowdef.columns()) {
e = col[r].get(index);
if (r == 0) {
e = col[r].getIfValid(index);
if (e == null) return null; // probably a deleted entry
} else {
e = col[r].get(index);
}
for (int i = 0; i < col[r].row().columns(); i++)
p.setCol(r + i, e.getColBytes(i));
p.setCol(r + i, e.getColBytes(i));
r = r + col[r].row().columns();
}
}

32
source/de/anomic/kelondro/kelondroRAIOChunks.java
Unescape View File

@ -52,30 +52,26 @@ public final class kelondroRAIOChunks extends kelondroAbstractIOChunks implement
this.ra = ra;
}
public long length() throws IOException {
public synchronized long length() throws IOException {
return ra.length();
}
public int read(long pos, byte[] b, int off, int len) throws IOException {
public synchronized int read(long pos, byte[] b, int off, int len) throws IOException {
if (len == 0) return 0;
synchronized (this.ra) {
this.ra.seek(pos);
long available = ra.available();
if (available >= len) {
return ra.read(b, off, len);
} else if (available == 0) {
return -1;
} else {
return ra.read(b, off, (int) available);
}
this.ra.seek(pos);
long available = ra.available();
if (available >= len) {
return ra.read(b, off, len);
} else if (available == 0) {
return -1;
} else {
return ra.read(b, off, (int) available);
}
}
public void write(long pos, byte[] b, int off, int len) throws IOException {
synchronized (this.ra) {
this.ra.seek(pos);
this.ra.write(b, off, len);
}
public synchronized void write(long pos, byte[] b, int off, int len) throws IOException {
this.ra.seek(pos);
this.ra.write(b, off, len);
}
public void commit() throws IOException {
@ -83,7 +79,7 @@ public final class kelondroRAIOChunks extends kelondroAbstractIOChunks implement
// this method is used to flush write-buffers
}
public void close() throws IOException {
public synchronized void close() throws IOException {
if (this.ra != null) this.ra.close();
this.ra = null;
}

193
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -204,7 +204,8 @@ public class kelondroRecords {
// needs to run up and own all the way between the beginning and the end of the
// file for each record. We check consistency beteen file size and
if (finalwrite) synchronized (entryFile) {
entryFile.writeInt(POS_USEDC, USEDC);
assert this.USEDC >= 0 : "this.USEDC = " + this.USEDC;
entryFile.writeInt(POS_USEDC, this.USEDC);
entryFile.commit();
}
}
@ -221,6 +222,7 @@ public class kelondroRecords {
private synchronized void readused() throws IOException {
synchronized (entryFile) {
this.USEDC = entryFile.readInt(POS_USEDC);
assert this.USEDC >= 0 : "this.USEDC = " + this.USEDC + ", filename = " + filename;
}
}
@ -245,20 +247,22 @@ public class kelondroRecords {
// delete element with handle h
// this element is then connected to the deleted-chain and can be
// re-used change counter
assert (h.index >= 0);
assert (h.index >= 0);
assert (h.index != NUL);
assert (h.index < USEDC + FREEC) : "USEDC = " + USEDC + ", FREEC = " + FREEC + ", h.index = " + h.index;
long sp = seekpos(h);
assert (sp <= entryFile.length() + ROW.objectsize) : h.index + "/" + sp + " exceeds file size " + entryFile.length();
synchronized (USAGE) {
USEDC--;
FREEC++;
// change pointer
entryFile.writeInt(sp, FREEH.index); // extend free-list
// write new FREEH Handle link
FREEH = h;
writefree();
writeused(false);
synchronized (entryFile) {
assert (h.index < USEDC + FREEC) : "USEDC = " + USEDC + ", FREEC = " + FREEC + ", h.index = " + h.index;
long sp = seekpos(h);
assert (sp <= entryFile.length() + ROW.objectsize) : h.index + "/" + sp + " exceeds file size " + entryFile.length();
USEDC--;
FREEC++;
// change pointer
entryFile.writeInt(sp, FREEH.index); // extend free-list
// write new FREEH Handle link
FREEH = h;
writefree();
writeused(false);
}
}
}
@ -271,46 +275,48 @@ public class kelondroRecords {
}
assert (chunk.length == ROW.objectsize()) : "chunk.length = " + chunk.length + ", ROW.objectsize() = " + ROW.objectsize();
synchronized (USAGE) {
if (USAGE.FREEC == 0) {
// generate new entry
int index = USAGE.allCount();
entryFile.write(seekpos(index) + overhead, chunk, 0, ROW.objectsize()); // occupy space, othervise the USAGE computaton does not work
USAGE.USEDC++;
writeused(false);
return index;
} else {
// re-use record from free-list
USAGE.USEDC++;
USAGE.FREEC--;
// take link
int index;
if (USAGE.FREEH.index == NUL) {
serverLog.logSevere("kelondroRecords/" + filename, "INTERNAL ERROR (DATA INCONSISTENCY): re-use of records failed, lost " + (USAGE.FREEC + 1) + " records.");
// try to heal..
USAGE.USEDC = USAGE.allCount() + 1;
USAGE.FREEC = 0;
index = USAGE.USEDC - 1;
} else {
index = USAGE.FREEH.index;
//System.out.println("*DEBUG* ALLOCATED DELETED INDEX " + index);
// check for valid seek position
long seekp = seekpos(USAGE.FREEH);
if (seekp > entryFile.length()) {
// this is a severe inconsistency. try to heal..
serverLog.logSevere("kelondroRecords/" + filename, "new Handle: lost " + USAGE.FREEC + " marked nodes; seek position " + seekp + "/" + USAGE.FREEH.index + " out of file size " + entryFile.length() + "/" + ((entryFile.length() - POS_NODES) / recordsize));
index = USAGE.allCount(); // a place at the end of the file
USAGE.USEDC += USAGE.FREEC; // to avoid that non-empty records at the end are overwritten
USAGE.FREEC = 0; // discard all possible empty nodes
USAGE.FREEH.index = NUL;
} else {
// read link to next element of FREEH chain
USAGE.FREEH.index = entryFile.readInt(seekp);
}
}
USAGE.writeused(false);
USAGE.writefree();
entryFile.write(seekpos(index) + overhead, chunk, 0, ROW.objectsize()); // overwrite space
return index;
synchronized (entryFile) {
if (USAGE.FREEC == 0) {
// generate new entry
int index = USAGE.allCount();
entryFile.write(seekpos(index) + overhead, chunk, 0, ROW.objectsize()); // occupy space, othervise the USAGE computaton does not work
USAGE.USEDC++;
writeused(false);
return index;
} else {
// re-use record from free-list
USAGE.USEDC++;
USAGE.FREEC--;
// take link
int index;
if (USAGE.FREEH.index == NUL) {
serverLog.logSevere("kelondroRecords/" + filename, "INTERNAL ERROR (DATA INCONSISTENCY): re-use of records failed, lost " + (USAGE.FREEC + 1) + " records.");
// try to heal..
USAGE.USEDC = USAGE.allCount() + 1;
USAGE.FREEC = 0;
index = USAGE.USEDC - 1;
} else {
index = USAGE.FREEH.index;
//System.out.println("*DEBUG* ALLOCATED DELETED INDEX " + index);
// check for valid seek position
long seekp = seekpos(USAGE.FREEH);
if (seekp > entryFile.length()) {
// this is a severe inconsistency. try to heal..
serverLog.logSevere("kelondroRecords/" + filename, "new Handle: lost " + USAGE.FREEC + " marked nodes; seek position " + seekp + "/" + USAGE.FREEH.index + " out of file size " + entryFile.length() + "/" + ((entryFile.length() - POS_NODES) / recordsize));
index = USAGE.allCount(); // a place at the end of the file
USAGE.USEDC += USAGE.FREEC; // to avoid that non-empty records at the end are overwritten
USAGE.FREEC = 0; // discard all possible empty nodes
USAGE.FREEH.index = NUL;
} else {
// read link to next element of FREEH chain
USAGE.FREEH.index = entryFile.readInt(seekp);
}
}
USAGE.writeused(false);
USAGE.writefree();
entryFile.write(seekpos(index) + overhead, chunk, 0, ROW.objectsize()); // overwrite space
return index;
}
}
}
}
@ -325,48 +331,49 @@ public class kelondroRecords {
}
//assert (chunk.length == ROW.objectsize()) : "chunk.length = " + chunk.length + ", ROW.objectsize() = " + ROW.objectsize();
synchronized (USAGE) {
if (index < USAGE.allCount()) {
// write within the file
// this can be critical, if we simply overwrite fields that are marked
// as deleted. This case should be avoided. There is no other way to check
// that the field is not occupied than looking at the FREEC counter
assert (USAGE.FREEC == 0) : "FREEC = " + USAGE.FREEC;
// simply overwrite the cell
entryFile.write(seekpos(index), bulkchunk, offset, recordsize);
// no changes of counter necessary
} else {
// write beyond the end of the file
// records that are in between are marked as deleted
Handle h;
while (index > USAGE.allCount()) {
h = new Handle(USAGE.allCount());
USAGE.FREEC++;
entryFile.write(seekpos(h), spaceChunk); // occupy space, othervise the USAGE computaton does not work
entryFile.writeInt(seekpos(h), USAGE.FREEH.index);
USAGE.FREEH = h;
USAGE.writefree();
entryFile.commit();
}
assert (index <= USAGE.allCount());
synchronized (entryFile) {
if (index < USAGE.allCount()) {
// write within the file
// this can be critical, if we simply overwrite fields that are marked
// as deleted. This case should be avoided. There is no other way to check
// that the field is not occupied than looking at the FREEC counter
assert (USAGE.FREEC == 0) : "FREEC = " + USAGE.FREEC;
// simply overwrite the cell
entryFile.write(seekpos(index), bulkchunk, offset, recordsize);
// no changes of counter necessary
} else {
// write beyond the end of the file
// records that are in between are marked as deleted
Handle h;
while (index > USAGE.allCount()) {
h = new Handle(USAGE.allCount());
USAGE.FREEC++;
entryFile.write(seekpos(h), spaceChunk); // occupy space, othervise the USAGE computaton does not work
entryFile.writeInt(seekpos(h), USAGE.FREEH.index);
USAGE.FREEH = h;
USAGE.writefree();
entryFile.commit();
}
assert (index <= USAGE.allCount());
// adopt USAGE.USEDC
if (USAGE.allCount() == index) {
entryFile.write(seekpos(index), bulkchunk, offset, recordsize); // write chunk and occupy space
USAGE.USEDC++;
USAGE.writeused(false);
entryFile.commit();
// adopt USAGE.USEDC
if (USAGE.allCount() == index) {
entryFile.write(seekpos(index), bulkchunk, offset, recordsize); // write chunk and occupy space
USAGE.USEDC++;
USAGE.writeused(false);
entryFile.commit();
}
}
}
}
}
private synchronized void checkConsistency() {
if (debugmode) try { // in debug mode
long efl = entryFile.length();
assert ((efl - POS_NODES) % ((long) recordsize)) == 0 : "rest = " + ((entryFile.length() - POS_NODES) % ((long) recordsize)) + ", USEDC = " + this.USEDC + ", FREEC = " + this.FREEC + ", recordsize = " + recordsize + ", file = " + filename;
long calculated_used = (efl - POS_NODES) / ((long) recordsize);
assert calculated_used == this.USEDC + this.FREEC : "calculated_used = " + calculated_used + ", USEDC = " + this.USEDC + ", FREEC = " + this.FREEC + ", recordsize = " + recordsize + ", file = " + filename;
if (calculated_used != this.USEDC + this.FREEC) logFailure("INCONSISTENCY in USED computation: calculated_used = " + calculated_used + ", USEDC = " + this.USEDC + ", FREEC = " + this.FREEC + ", recordsize = " + recordsize + ", file = " + filename);
} catch (IOException e) {
assert false;
}
@ -911,7 +918,7 @@ public class kelondroRecords {
if (cacheHeaders == null) {
if (fillTail) {
// read complete record
byte[] chunkbuffer = new byte[recordsize];
byte[] chunkbuffer = new byte[recordsize];
entryFile.readFully(seekpos(this.handle), chunkbuffer, 0, recordsize);
this.headChunk = new byte[headchunksize];
this.tailChunk = new byte[tailchunksize];
@ -1031,6 +1038,12 @@ public class kelondroRecords {
return result; // return previous value
}
public boolean valid() {
// returns true if the key starts with non-zero byte
// this may help to detect deleted entries
return headChunk[overhead] != 0;
}
public byte[] getKey() {
// read key
return trimCopy(headChunk, overhead, ROW.width(0));
@ -1071,8 +1084,10 @@ public class kelondroRecords {
boolean doCommit = this.headChanged || this.tailChanged;
// save head
synchronized (entryFile) {
if (this.headChanged) {
//System.out.println("WRITEH(" + filename + ", " + seekpos(this.handle) + ", " + this.headChunk.length + ")");
assert (headChunk == null) || (headChunk.length == headchunksize);
entryFile.write(seekpos(this.handle), (this.headChunk == null) ? new byte[headchunksize] : this.headChunk);
updateNodeCache(cachePriority);
this.headChanged = false;
@ -1081,11 +1096,13 @@ public class kelondroRecords {
// save tail
if ((this.tailChunk != null) && (this.tailChanged)) {
//System.out.println("WRITET(" + filename + ", " + (seekpos(this.handle) + headchunksize) + ", " + this.tailChunk.length + ")");
assert (tailChunk == null) || (tailChunk.length == tailchunksize);
entryFile.write(seekpos(this.handle) + headchunksize, (this.tailChunk == null) ? new byte[tailchunksize] : this.tailChunk);
this.tailChanged = false;
}
if (doCommit) entryFile.commit();
}
}
public synchronized void collapse() {
@ -1283,7 +1300,9 @@ public class kelondroRecords {
// Returns the number of key-value mappings in this map.
public int size() {
return USAGE.used();
synchronized (entryFile) {
return USAGE.used();
}
}
protected final int free() {
@ -1311,7 +1330,7 @@ public class kelondroRecords {
public Object next() {
try {
Node n = (Node) nodeIterator.next();
return row().newEntry(n.getValueRow(), n.handle.index);
return row().newEntryIndex(n.getValueRow(), n.handle.index);
} catch (IOException e) {
throw new kelondroException(filename, e.getMessage());
}

199
source/de/anomic/kelondro/kelondroRow.java
Unescape View File

@ -46,6 +46,7 @@ public class kelondroRow {
public kelondroRow(kelondroColumn[] row, kelondroOrder objectOrder, int primaryKey) {
this.row = row;
assert (objectOrder != null);
this.objectOrder = objectOrder;
this.primaryKey = primaryKey;
this.colstart = new int[row.length];
@ -57,7 +58,8 @@ public class kelondroRow {
}
public kelondroRow(String structure, kelondroOrder objectOrder, int primaryKey) {
this.objectOrder = objectOrder;
assert (objectOrder != null);
this.objectOrder = objectOrder;
this.primaryKey = primaryKey;
// define row with row syntax
// example:
@ -88,7 +90,8 @@ public class kelondroRow {
}
public void setOrdering(kelondroOrder objectOrder, int primaryKey) {
this.objectOrder = objectOrder;
assert (objectOrder != null);
this.objectOrder = objectOrder;
this.primaryKey = primaryKey;
}
@ -138,51 +141,36 @@ public class kelondroRow {
return new String(s);
}
public long getColLong(byte[] rowinstance, int column) {
// uses the column definition to choose the right encoding
return getColLong(rowinstance, row[column].encoder(), colstart[column], row[column].cellwidth());
}
protected static final long getColLong(byte[] rowinstance, int encoder, int offset, int length) {
// start - fix for badly stored parameters
if ((length >= 3) && (rowinstance[offset] == '[') && (rowinstance[offset + 1] == 'B') && (rowinstance[offset + 2] == '@')) return 0;
if ((length == 2) && (rowinstance[offset] == '[') && (rowinstance[offset + 1] == 'B')) return 0;
if ((length == 1) && (rowinstance[offset] == '[')) return 0;
// stop - fix for badly stored parameters
switch (encoder) {
case kelondroColumn.encoder_none:
throw new kelondroException("ROW", "getColLong has celltype none, no encoder given");
case kelondroColumn.encoder_b64e:
// start - fix for badly stored parameters
boolean maxvalue = true;
for (int i = 0; i < length; i++) if (rowinstance[offset + i] != '_') {maxvalue = false; break;}
if (maxvalue) return 0;
// stop - fix for badly stored parameters
return kelondroBase64Order.enhancedCoder.decodeLong(rowinstance, offset, length);
case kelondroColumn.encoder_b256:
return kelondroNaturalOrder.decodeLong(rowinstance, offset, length);
case kelondroColumn.encoder_bytes:
throw new kelondroException("ROW", "getColLong of celltype bytes not applicable");
}
throw new kelondroException("ROW", "getColLong did not find appropriate encoding");
}
public Entry newEntry() {
return new Entry();
}
public Entry newEntry(byte[] rowinstance) {
if (rowinstance == null) return null;
//assert (rowinstance[0] != 0);
assert (this.objectOrder.wellformed(rowinstance, 0, row[0].cellwidth()));
if (!(this.objectOrder.wellformed(rowinstance, 0, row[0].cellwidth()))) return null;
return new Entry(rowinstance);
}
public Entry newEntry(byte[] rowinstance, int start, int length) {
public Entry newEntry(Entry oldrow, int fromColumn) {
if (oldrow == null) return null;
assert (oldrow.getColBytes(0)[0] != 0);
assert (this.objectOrder.wellformed(oldrow.getColBytes(0), 0, row[0].cellwidth()));
return new Entry(oldrow, fromColumn);
}
public Entry newEntry(byte[] rowinstance, int start) {
if (rowinstance == null) return null;
return new Entry(rowinstance, start, length);
//assert (rowinstance[0] != 0);
assert (this.objectOrder.wellformed(rowinstance, start, row[0].cellwidth()));
return new Entry(rowinstance, start);
}
public Entry newEntry(byte[][] cells) {
if (cells == null) return null;
assert (cells[0][0] != 0);
assert (this.objectOrder.wellformed(cells[0], 0, row[0].cellwidth()));
return new Entry(cells);
}
@ -190,36 +178,48 @@ public class kelondroRow {
if (external == null) return null;
return new Entry(external, decimalCardinal);
}
public EntryIndex newEntry(byte[] rowinstance, int index) {
if (rowinstance == null) return null;
public EntryIndex newEntryIndex(byte[] rowinstance, int index) {
if (rowinstance == null) return null;
assert (rowinstance[0] != 0);
assert (this.objectOrder.wellformed(rowinstance, 0, row[0].cellwidth()));
return new EntryIndex(rowinstance, index);
}
public class Entry implements Comparable {
private byte[] rowinstance;
private int offset; // the offset where the row starts within rowinstance
public Entry() {
rowinstance = new byte[objectsize];
for (int i = 0; i < objectsize; i++) this.rowinstance[i] = 0;
offset = 0;
}
public Entry(byte[] newrow) {
this(newrow, 0, newrow.length);
this(newrow, 0);
}
public Entry(byte[] newrow, int start, int length) {
assert newrow.length >= (length + start) : "objectsize = " + objectsize + ", start = " + start + ", length = " + length;
assert objectsize == length : "objectsize = " + objectsize + ", start = " + start + ", length = " + length;
this.rowinstance = new byte[objectsize];
System.arraycopy(newrow, start, this.rowinstance, 0, objectsize);
public Entry(Entry oldrow, int fromColumn) {
this(oldrow.rowinstance, oldrow.offset + oldrow.colstart(fromColumn));
}
public Entry(byte[] newrow, int start) {
if (newrow.length - start >= objectsize) {
this.rowinstance = newrow;
} else {
this.rowinstance = new byte[objectsize];
System.arraycopy(newrow, start, this.rowinstance, 0, newrow.length - start);
}
this.offset = start;
//for (int i = ll; i < objectsize; i++) this.rowinstance[i] = 0;
}
public Entry(byte[][] cols) {
assert row.length == cols.length : "cols.length = " + cols.length + ", row.length = " + row.length;
rowinstance = new byte[objectsize];
this.rowinstance = new byte[objectsize];
this.offset = 0;
int ll;
int cs, cw;
for (int i = 0; i < row.length; i++) {
@ -243,7 +243,8 @@ public class kelondroRow {
if (nickref == null) genNickRef();
String nick;
int p;
rowinstance = new byte[objectsize];
this.rowinstance = new byte[objectsize];
this.offset = 0;
for (int i = 0; i < elts.length; i++) {
p = elts[i].indexOf('=');
if (p > 0) {
@ -274,6 +275,14 @@ public class kelondroRow {
}
}
protected int colstart(int column) {
return colstart[column];
}
protected int cellwidth(int column) {
return row[column].cellwidth();
}
public int compareTo(Object o) {
if (objectOrder == null) throw new kelondroException("objects cannot be compared, no order given");
if (o instanceof Entry) {
@ -283,7 +292,19 @@ public class kelondroRow {
}
public byte[] bytes() {
return rowinstance;
if ((offset == 0) && (rowinstance.length == objectsize)) {
return rowinstance;
} else {
byte[] tmp = new byte[objectsize];
System.arraycopy(rowinstance, offset, tmp, 0, objectsize);
return tmp;
}
}
public void writeToArray(byte[] target, int targetOffset) {
// this method shall replace the byte()s where possible, bacause it may reduce the number of new byte[] allocations
assert (targetOffset + objectsize <= target.length) : "targetOffset = " + targetOffset + ", target.length = " + target.length + ", objectsize = " + objectsize;
System.arraycopy(rowinstance, offset, target, targetOffset, objectsize);
}
public int columns() {
@ -295,14 +316,14 @@ public class kelondroRow {
}
public boolean empty(int column) {
return rowinstance[colstart[column]] == 0;
return rowinstance[offset + colstart[column]] == 0;
}
public void setCol(String nickname, char c) {
if (nickref == null) genNickRef();
Object[] ref = (Object[]) nickref.get(nickname);
if (ref == null) return;
rowinstance[((Integer) ref[1]).intValue()] = (byte) c;
rowinstance[offset + ((Integer) ref[1]).intValue()] = (byte) c;
}
public void setCol(String nickname, byte[] cell) {
@ -318,27 +339,27 @@ public class kelondroRow {
}
public void setCol(int column, char[] cell) {
int offset = colstart[column];
for (int i = 0; i < cell.length; i++) rowinstance[offset + i] = (byte) cell[i];
for (int i = cell.length; i < row[column].cellwidth(); i++) rowinstance[offset + i] = 0;
int clstrt = colstart[column];
for (int i = 0; i < cell.length; i++) rowinstance[offset + clstrt + i] = (byte) cell[i];
for (int i = cell.length; i < row[column].cellwidth(); i++) rowinstance[offset + clstrt + i] = 0;
}
private void setCol(int encoding, int offset, int length, byte[] cell) {
private void setCol(int encoding, int clstrt, int length, byte[] cell) {
if (cell == null) {
while (length-- > 0) rowinstance[offset + length] = 0;
while (length-- > 0) rowinstance[offset + clstrt + length] = 0;
} else {
if (cell.length < length) {
System.arraycopy(cell, 0, rowinstance, offset, cell.length);
while (length-- > cell.length) rowinstance[offset + length] = 0;
System.arraycopy(cell, 0, rowinstance, offset + clstrt, cell.length);
while (length-- > cell.length) rowinstance[offset + clstrt + length] = 0;
} else {
//assert cell.length == length;
System.arraycopy(cell, 0, rowinstance, offset, length);
System.arraycopy(cell, 0, rowinstance, offset + clstrt, length);
}
}
}
public void setCol(int column, byte c) {
rowinstance[colstart[column]] = c;
rowinstance[offset + colstart[column]] = c;
}
public void setCol(int column, String cell, String encoding) {
@ -368,12 +389,12 @@ public class kelondroRow {
Object[] ref = (Object[]) nickref.get(nickname);
if (ref == null) return;
kelondroColumn col = (kelondroColumn) ref[0];
setCol(col.encoder(), ((Integer) ref[1]).intValue(), col.cellwidth(), cell);
setCol(col.encoder(), offset + ((Integer) ref[1]).intValue(), col.cellwidth(), cell);
}
public void setCol(int column, long cell) {
// uses the column definition to choose the right encoding
setCol(row[column].encoder(), colstart[column], row[column].cellwidth(), cell);
setCol(row[column].encoder(), offset + colstart[column], row[column].cellwidth(), cell);
}
private void setCol(int encoder, int offset, int length, long cell) {
@ -397,7 +418,7 @@ public class kelondroRow {
if (ref == null) return dflt;
kelondroColumn col = (kelondroColumn) ref[0];
byte[] cell = new byte[col.cellwidth()];
System.arraycopy(rowinstance, ((Integer) ref[1]).intValue(), cell, 0, cell.length);
System.arraycopy(rowinstance, offset + ((Integer) ref[1]).intValue(), cell, 0, cell.length);
return cell;
}
@ -413,16 +434,16 @@ public class kelondroRow {
return getColString(row[column].encoder(), colstart[column], row[column].cellwidth(), encoding);
}
private String getColString(int encoder, int offset, int length, String encoding) {
if (rowinstance[offset] == 0) return null;
if (length > rowinstance.length - offset) length = rowinstance.length - offset;
while ((length > 0) && (rowinstance[offset + length - 1] == 0)) length--;
private String getColString(int encoder, int clstrt, int length, String encoding) {
if (rowinstance[offset + clstrt] == 0) return null;
if (length > rowinstance.length - offset - clstrt) length = rowinstance.length - offset - clstrt;
while ((length > 0) && (rowinstance[offset + clstrt + length - 1] == 0)) length--;
if (length == 0) return null;
try {
if ((encoding == null) || (encoding.length() == 0))
return new String(rowinstance, offset, length);
return new String(rowinstance, offset + clstrt, length);
else
return new String(rowinstance, offset, length, encoding);
return new String(rowinstance, offset + clstrt, length, encoding);
} catch (UnsupportedEncodingException e) {
return "";
}
@ -433,38 +454,68 @@ public class kelondroRow {
Object[] ref = (Object[]) nickref.get(nickname);
if (ref == null) return dflt;
kelondroColumn col = (kelondroColumn) ref[0];
int colstart = ((Integer) ref[1]).intValue();
return kelondroRow.getColLong(rowinstance, col.encoder(), colstart, col.cellwidth());
int clstrt = ((Integer) ref[1]).intValue();
return getColLong(col.encoder(), clstrt, col.cellwidth());
}
public long getColLong(int column) {
// uses the column definition to choose the right encoding
return kelondroRow.getColLong(rowinstance, row[column].encoder(), colstart[column], row[column].cellwidth());
return getColLong(row[column].encoder(), colstart[column], row[column].cellwidth());
}
protected final long getColLong(int encoder, int clstrt, int length) {
// start - fix for badly stored parameters
if ((length >= 3) && (rowinstance[offset + clstrt] == '[') && (rowinstance[offset + clstrt + 1] == 'B') && (rowinstance[offset + clstrt + 2] == '@')) return 0;
if ((length == 2) && (rowinstance[offset + clstrt] == '[') && (rowinstance[offset + clstrt + 1] == 'B')) return 0;
if ((length == 1) && (rowinstance[offset + clstrt] == '[')) return 0;
// stop - fix for badly stored parameters
switch (encoder) {
case kelondroColumn.encoder_none:
throw new kelondroException("ROW", "getColLong has celltype none, no encoder given");
case kelondroColumn.encoder_b64e:
// start - fix for badly stored parameters
boolean maxvalue = true;
for (int i = 0; i < length; i++) if (rowinstance[offset + clstrt + i] != '_') {maxvalue = false; break;}
if (maxvalue) return 0;
// stop - fix for badly stored parameters
return kelondroBase64Order.enhancedCoder.decodeLong(rowinstance, offset + clstrt, length);
case kelondroColumn.encoder_b256:
return kelondroNaturalOrder.decodeLong(rowinstance, offset + clstrt, length);
case kelondroColumn.encoder_bytes:
throw new kelondroException("ROW", "getColLong of celltype bytes not applicable");
}
throw new kelondroException("ROW", "getColLong did not find appropriate encoding");
}
public byte getColByte(String nickname, byte dflt) {
if (nickref == null) genNickRef();
Object[] ref = (Object[]) nickref.get(nickname);
if (ref == null) return dflt;
return rowinstance[((Integer) ref[1]).intValue()];
return rowinstance[offset + ((Integer) ref[1]).intValue()];
}
public byte getColByte(int column) {
return rowinstance[colstart[column]];
return rowinstance[offset + colstart[column]];
}
public byte[] getColBytes(int column) {
byte[] c = new byte[row[column].cellwidth()];
System.arraycopy(rowinstance, colstart[column], c, 0, row[column].cellwidth());
System.arraycopy(rowinstance, offset + colstart[column], c, 0, row[column].cellwidth());
return c;
}
public char[] getColChars(int column) {
char[] c = new char[row[column].cellwidth()];
System.arraycopy(rowinstance, colstart[column], c, 0, row[column].cellwidth());
System.arraycopy(rowinstance, offset + colstart[column], c, 0, row[column].cellwidth());
return c;
}
public void writeToArray(int column, byte[] target, int targetOffset) {
// this method shall replace the getColBytes where possible, bacause it may reduce the number of new byte[] allocations
assert (targetOffset + row[column].cellwidth() <= target.length) : "targetOffset = " + targetOffset + ", target.length = " + target.length + ", row[column].cellwidth() = " + row[column].cellwidth();
System.arraycopy(rowinstance, offset + colstart[column], target, targetOffset, row[column].cellwidth());
}
public String toPropertyForm(boolean includeBraces, boolean decimalCardinal, boolean longname) {
serverByteBuffer bb = new serverByteBuffer();
if (includeBraces) bb.append('{');
@ -479,7 +530,7 @@ public class kelondroRow {
assert row[i].cellwidth() == 1;
bb.append(Integer.toString((int) (0xff & getColByte(i))));
} else {
bb.append(rowinstance, colstart[i], row[i].cellwidth());
bb.append(rowinstance, offset + colstart[i], row[i].cellwidth());
}
if (i < row.length - 1) {
bb.append(',');

30
source/de/anomic/kelondro/kelondroRowCollection.java
Unescape View File

@ -80,10 +80,10 @@ public class kelondroRowCollection {
this.lastTimeWrote = System.currentTimeMillis();
}
public kelondroRowCollection(kelondroRow rowdef, byte[] exportedCollectionRowinstance) {
public kelondroRowCollection(kelondroRow rowdef, kelondroRow.Entry exportedCollectionRowEnvironment, int columnInEnvironment) {
this.rowdef = rowdef;
int chunkcachelength = exportedCollectionRowinstance.length - exportOverheadSize;
kelondroRow.Entry exportedCollection = exportRow(chunkcachelength).newEntry(exportedCollectionRowinstance);
int chunkcachelength = exportedCollectionRowEnvironment.cellwidth(columnInEnvironment) - exportOverheadSize;
kelondroRow.Entry exportedCollection = exportRow(chunkcachelength).newEntry(exportedCollectionRowEnvironment, columnInEnvironment);
this.chunkcount = (int) exportedCollection.getColLong(exp_chunkcount);
//assert (this.chunkcount <= chunkcachelength / rowdef.objectsize) : "chunkcount = " + this.chunkcount + ", chunkcachelength = " + chunkcachelength + ", rowdef.objectsize = " + rowdef.objectsize;
if ((this.chunkcount > chunkcachelength / rowdef.objectsize)) {
@ -114,9 +114,10 @@ public class kelondroRowCollection {
}
private static final kelondroRow exportMeasureRow = exportRow(0 /* no relevance */);
protected static final int sizeOfExportedCollectionRows(kelondroRow rowdef, byte[] exportedCollectionRowinstance) {
int chunkcount = (int) exportMeasureRow.getColLong(exportedCollectionRowinstance, exp_chunkcount);
protected static final int sizeOfExportedCollectionRows(kelondroRow.Entry exportedCollectionRowEnvironment, int columnInEnvironment) {
kelondroRow.Entry exportedCollectionEntry = exportMeasureRow.newEntry(exportedCollectionRowEnvironment, columnInEnvironment);
int chunkcount = (int) exportedCollectionEntry.getColLong(exp_chunkcount);
return chunkcount;
}
@ -136,7 +137,7 @@ public class kelondroRowCollection {
"short ordercol-2 {b256}," +
"short orderbound-2 {b256}," +
"byte[] collection-" + chunkcachelength,
null, 0
kelondroNaturalOrder.naturalOrder, 0
);
}
@ -214,23 +215,16 @@ public class kelondroRowCollection {
assert (index * rowdef.objectsize < chunkcache.length);
if (index >= chunkcount) return null;
if (index * rowdef.objectsize() >= chunkcache.length) return null;
byte[] a = new byte[rowdef.objectsize()];
System.arraycopy(chunkcache, index * rowdef.objectsize(), a, 0, rowdef.objectsize());
this.lastTimeRead = System.currentTimeMillis();
return rowdef.newEntry(a);
return rowdef.newEntry(chunkcache, index * rowdef.objectsize());
}
public final void set(int index, kelondroRow.Entry a) {
set(index, a.bytes(), 0, a.bytes().length);
}
public synchronized final void set(int index, byte[] a, int astart, int alength) {
assert (index >= 0) : "get: access with index " + index + " is below zero";
assert (index < chunkcount) : "get: access with index " + index + " is above chunkcount " + chunkcount;
assert (!(bugappearance(a, astart, alength))) : "a = " + serverLog.arrayList(a, astart, alength);
if (bugappearance(a, astart, alength)) return; // TODO: this is temporary; remote peers may still submit bad entries
int l = Math.min(rowdef.objectsize(), Math.min(alength, a.length - astart));
System.arraycopy(a, astart, chunkcache, index * rowdef.objectsize(), l);
//assert (!(bugappearance(a, astart, alength))) : "a = " + serverLog.arrayList(a, astart, alength);
//if (bugappearance(a, astart, alength)) return; // TODO: this is temporary; remote peers may still submit bad entries
a.writeToArray(chunkcache, index * rowdef.objectsize());
this.lastTimeWrote = System.currentTimeMillis();
}

15
source/de/anomic/kelondro/kelondroRowSet.java
Unescape View File

@ -51,8 +51,8 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
this.profile = new kelondroProfile();
}
public kelondroRowSet(kelondroRow rowdef, byte[] exportedCollectionRowinstance) {
super(rowdef, exportedCollectionRowinstance);
public kelondroRowSet(kelondroRow rowdef, kelondroRow.Entry exportedCollectionRowEnvironment, int columnInEnvironment) {
super(rowdef, exportedCollectionRowEnvironment, columnInEnvironment);
this.profile = new kelondroProfile();
}
@ -111,17 +111,6 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
return remove(a, 0, a.length);
}
public void removeMarkedAll(kelondroRowCollection c) {
long handle = profile.startDelete();
Iterator i = c.rows();
kelondroRow.Entry entry;
while (i.hasNext()) {
entry = (kelondroRow.Entry) i.next();
remove(entry.bytes(), 0, entry.bytes().length);
}
profile.stopDelete(handle);
}
public void setOrdering(kelondroOrder newOrder, int newColumn) {
if ((rowdef.objectOrder == null) ||
(!(rowdef.objectOrder.signature().equals(newOrder.signature()))) ||

2
source/de/anomic/kelondro/kelondroTree.java
Unescape View File

@ -341,7 +341,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
assert (newrow != null);
assert (newrow.columns() == row().columns());
assert (!(serverLog.allZero(newrow.getColBytes(super.row().primaryKey))));
assert newrow.bytes().length <= super.row().objectsize;
assert newrow.objectsize() <= super.row().objectsize;
// Associates the specified value with the specified key in this map
kelondroRow.Entry result = null;
//writeLock.stay(2000, 1000);

6
source/de/anomic/plasma/plasmaCrawlLURL.java
Unescape View File

@ -67,6 +67,7 @@ import de.anomic.index.indexURLEntry;
import de.anomic.kelondro.kelondroBitfield;
import de.anomic.kelondro.kelondroCache;
import de.anomic.kelondro.kelondroCloneableIterator;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroFlexSplitTable;
import de.anomic.kelondro.kelondroIndex;
import de.anomic.kelondro.kelondroRow;
@ -204,8 +205,11 @@ public final class plasmaCrawlLURL {
}
public synchronized indexURLEntry newEntry(String propStr) {
if (propStr != null && propStr.startsWith("{") && propStr.endsWith("}")) {
if (propStr != null && propStr.startsWith("{") && propStr.endsWith("}")) try {
return new indexURLEntry(serverCodings.s2p(propStr.substring(1, propStr.length() - 1)));
} catch (kelondroException e) {
// wrong format
return null;
} else {
return null;
}

49
source/de/anomic/plasma/plasmaWordIndex.java
Unescape View File

@ -199,8 +199,12 @@ public final class plasmaWordIndex implements indexRI {
}
public void flushCacheSome() {
flushCache(dhtOutCache, (dhtOutCache.size() > 3 * flushsize) ? flushsize : Math.min(flushsize, Math.max(1, dhtOutCache.size() / lowcachedivisor)));
flushCache(dhtInCache, (dhtInCache.size() > 3 * flushsize) ? flushsize : Math.min(flushsize, Math.max(1, dhtInCache.size() / lowcachedivisor)));
synchronized (dhtOutCache) {
flushCache(dhtOutCache, (dhtOutCache.size() > 3 * flushsize) ? flushsize : Math.min(flushsize, Math.max(1, dhtOutCache.size() / lowcachedivisor)));
}
synchronized (dhtInCache) {
flushCache(dhtInCache, (dhtInCache.size() > 3 * flushsize) ? flushsize : Math.min(flushsize, Math.max(1, dhtInCache.size() / lowcachedivisor)));
}
}
private void flushCache(indexRAMRI ram, int count) {
@ -394,32 +398,48 @@ public final class plasmaWordIndex implements indexRI {
public indexContainer deleteContainer(String wordHash) {
indexContainer c = new indexContainer(wordHash, indexRWIEntry.urlEntryRow);
c.addAllUnique(dhtInCache.deleteContainer(wordHash));
c.addAllUnique(dhtOutCache.deleteContainer(wordHash));
synchronized (dhtInCache) {
c.addAllUnique(dhtInCache.deleteContainer(wordHash));
}
synchronized (dhtOutCache) {
c.addAllUnique(dhtOutCache.deleteContainer(wordHash));
}
c.addAllUnique(collections.deleteContainer(wordHash));
return c;
}
public boolean removeEntry(String wordHash, String urlHash) {
boolean removed = false;
removed = removed | (dhtInCache.removeEntry(wordHash, urlHash));
removed = removed | (dhtOutCache.removeEntry(wordHash, urlHash));
synchronized (dhtInCache) {
removed = removed | (dhtInCache.removeEntry(wordHash, urlHash));
}
synchronized (dhtOutCache) {
removed = removed | (dhtOutCache.removeEntry(wordHash, urlHash));
}
removed = removed | (collections.removeEntry(wordHash, urlHash));
return removed;
}
public int removeEntries(String wordHash, Set urlHashes) {
int removed = 0;
removed += dhtInCache.removeEntries(wordHash, urlHashes);
removed += dhtOutCache.removeEntries(wordHash, urlHashes);
synchronized (dhtInCache) {
removed += dhtInCache.removeEntries(wordHash, urlHashes);
}
synchronized (dhtOutCache) {
removed += dhtOutCache.removeEntries(wordHash, urlHashes);
}
removed += collections.removeEntries(wordHash, urlHashes);
return removed;
}
public String removeEntriesExpl(String wordHash, Set urlHashes) {
String removed = "";
removed += dhtInCache.removeEntries(wordHash, urlHashes) + ", ";
removed += dhtOutCache.removeEntries(wordHash, urlHashes) + ", ";
synchronized (dhtOutCache) {
removed += dhtInCache.removeEntries(wordHash, urlHashes) + ", ";
}
synchronized (dhtOutCache) {
removed += dhtOutCache.removeEntries(wordHash, urlHashes) + ", ";
}
removed += collections.removeEntries(wordHash, urlHashes);
return removed;
}
@ -453,8 +473,13 @@ public final class plasmaWordIndex implements indexRI {
// urlHash assigned. This can only work if the entry is really fresh
// and can be found in the RAM cache
// this returns the number of deletion that had been possible
int d = dhtInCache.tryRemoveURLs(urlHash);
if (d > 0) return d; else return dhtOutCache.tryRemoveURLs(urlHash);
int d = 0;
synchronized (dhtInCache) {
d = dhtInCache.tryRemoveURLs(urlHash);
}
synchronized (dhtOutCache) {
if (d > 0) return d; else return dhtOutCache.tryRemoveURLs(urlHash);
}
}
public TreeSet indexContainerSet(String startHash, boolean ram, boolean rot, int count) {

4
source/de/anomic/server/serverInstantThread.java
Unescape View File

@ -118,7 +118,9 @@ public final class serverInstantThread extends serverAbstractThread implements s
this.terminate(false);
} catch (InvocationTargetException e) {
String targetException = e.getTargetException().getMessage();
if (targetException.indexOf("heap space") > 0) e.getTargetException().printStackTrace();
e.getTargetException().printStackTrace();
e.printStackTrace();
if ((targetException.indexOf("heap space") > 0) || (targetException.indexOf("NullPointerException") > 0)) e.getTargetException().printStackTrace();
serverLog.logSevere("SERVER", "Runtime Error in serverInstantThread.job, thread '" + this.getName() + "': " + e.getMessage() + "; target exception: " + targetException, e.getTargetException());
e.getTargetException().printStackTrace();
} catch (OutOfMemoryError e) {

Write Preview
Loading…
Cancel
Save