boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

some storage process enhancements (write without preceding read)

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3348 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 18 years ago
parent faad869865
commit daf2e15f59
9 changed files with 69 additions and 20 deletions
Show Stats Download Patch File Download Diff File

2
source/de/anomic/index/indexRAMRI.java
Unescape View File

@ -139,7 +139,7 @@ public final class indexRAMRI implements indexRI {
row.setCol(1, occ);
row.setCol(2, time);
row.setCol(3, iEntry.toKelondroEntry().bytes());
dumpArray.set((int) urlcount++, row);
dumpArray.overwrite((int) urlcount++, row);
}
}
wordcount++;

3
source/de/anomic/kelondro/kelondroArray.java
Unescape View File

@ -32,7 +32,8 @@ public interface kelondroArray {
public kelondroRow row();
public kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException;
public kelondroRow.Entry replace(int index, kelondroRow.Entry rowentry) throws IOException;
public void overwrite(int index, kelondroRow.Entry rowentry) throws IOException;
public kelondroRow.Entry get(int index) throws IOException;

2
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -377,7 +377,7 @@ public class kelondroCollectionIndex {
arrayEntry.setCol(1, collection.exportCollection());
// overwrite entry in this array
array.set(oldrownumber, arrayEntry);
array.overwrite(oldrownumber, arrayEntry);
// update the index entry
indexrow.setCol(idx_col_chunkcount, collection.size());

21
source/de/anomic/kelondro/kelondroFixedWidthArray.java
Unescape View File

@ -85,7 +85,7 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
}
}
public synchronized kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException {
public synchronized kelondroRow.Entry replace(int index, kelondroRow.Entry rowentry) throws IOException {
// make room for element
Node n;
@ -100,10 +100,25 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
// write the row
byte[] before = n.setValueRow((rowentry == null) ? null : rowentry.bytes());
n.commit(CP_NONE);
return row().newEntry(before);
}
public synchronized void overwrite(int index, kelondroRow.Entry rowentry) throws IOException {
// this writes a row without reading the row from the file system first
// make room for element
Node n;
while (super.USAGE.allCount() <= index) {
n = newNode();
n.commit(CP_NONE);
}
// create a node at position index with rowentry
n = newNode(new Handle(index), (rowentry == null) ? null : rowentry.bytes(), 0);
n.commit(CP_NONE);
}
public synchronized kelondroRow.Entry get(int index) throws IOException {
return row().newEntry(getNode(new Handle(index)).getValueRow());
}
@ -162,7 +177,7 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
System.out.println("erster Test");
f.delete();
kelondroFixedWidthArray k = new kelondroFixedWidthArray(f, rowdef, 6);
k.set(3, k.row().newEntry(new byte[][]{
k.overwrite(3, k.row().newEntry(new byte[][]{
"test123".getBytes(), "abcd".getBytes()}));
k.add(k.row().newEntry(new byte[][]{
"test456".getBytes(), "efgh".getBytes()}));

2
source/de/anomic/kelondro/kelondroFlexTable.java
Unescape View File

@ -193,7 +193,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
index.puti(row.getColBytes(0), super.add(row));
return null;
}
return super.set(i, row);
return super.replace(i, row);
}
public synchronized void addUnique(kelondroRow.Entry row, Date entryDate) throws IOException {

26
source/de/anomic/kelondro/kelondroFlexWidthArray.java
Unescape View File

@ -176,7 +176,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
return col[0].size();
}
public kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException {
public kelondroRow.Entry replace(int index, kelondroRow.Entry rowentry) throws IOException {
assert rowentry.bytes().length == this.rowdef.objectsize;
int c = 0;
kelondroRow.Entry e0, e1, p;
@ -189,7 +189,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
rowentry.bytes(),
rowdef.colstart[c],
rowdef.colstart[lastcol] - rowdef.colstart[c] + rowdef.width(lastcol));
e1 = col[c].set(index, e0);
e1 = col[c].replace(index, e0);
for (int i = 0; i < col[c].row().columns(); i++) {
p.setCol(c + i, e1.getColBytes(i));
}
@ -199,6 +199,24 @@ public class kelondroFlexWidthArray implements kelondroArray {
return p;
}
public void overwrite(int index, kelondroRow.Entry rowentry) throws IOException {
assert rowentry.bytes().length == this.rowdef.objectsize;
int c = 0;
kelondroRow.Entry e0;
int lastcol;
synchronized (col) {
while (c < rowdef.columns()) {
lastcol = c + col[c].row().columns() - 1;
e0 = col[c].row().newEntry(
rowentry.bytes(),
rowdef.colstart[c],
rowdef.colstart[lastcol] - rowdef.colstart[c] + rowdef.width(lastcol));
col[c].overwrite(index, e0);
c = c + col[c].row().columns();
}
}
}
public int add(kelondroRow.Entry rowentry) throws IOException {
assert rowentry.bytes().length == this.rowdef.objectsize;
kelondroRow.Entry e;
@ -215,7 +233,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
rowentry.bytes(),
rowdef.colstart[c],
rowdef.colstart[lastcol] + rowdef.width(lastcol) - rowdef.colstart[c]);
col[c].set(index,e);
col[c].overwrite(index,e);
c = c + col[c].row().columns();
}
}
@ -247,7 +265,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
// the other columns will be blanked out only
while (r < rowdef.columns()) {
col[r].set(index, null);
col[r].overwrite(index, null);
r = r + col[r].row().columns();
}
}

4
source/de/anomic/kelondro/kelondroHashtable.java
Unescape View File

@ -208,13 +208,13 @@ public class kelondroHashtable {
}
// make space
while (rowNumber >= hashArray.size()) hashArray.set(hashArray.size(), dummyRow);
while (rowNumber >= hashArray.size()) hashArray.overwrite(hashArray.size(), dummyRow);
// write row
kelondroRow.Entry newhkrow = hashArray.row().newEntry();
newhkrow.setCol(0, hash.key());
newhkrow.setCol(1, rowentry.bytes());
hashArray.set(rowNumber, newhkrow);
hashArray.overwrite(rowNumber, newhkrow);
return hashArray.row().newEntry(oldhkrow.getColBytes(1));
}

22
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -583,6 +583,10 @@ public class kelondroRecords {
return new Node();
}
protected final Node newNode(Handle handle, byte[] bulkchunk, int offset) {
return new Node(handle, bulkchunk, offset, true);
}
protected final Node getNode(Handle handle) throws IOException {
return getNode(handle, null, 0);
}
@ -649,7 +653,7 @@ public class kelondroRecords {
for (int i = tailchunksize - 1; i >= 0; i--) this.tailChunk[i] = 0;
}
protected Node(Handle handle, byte[] bulkchunk, int offset) {
protected Node(Handle handle, byte[] bulkchunk, int offset, boolean setChanged) {
// this initializer is used to create nodes from bulk-read byte arrays
this.handle = handle;
assert (bulkchunk.length >= offset + headchunksize) : "bulkchunk.length = " + bulkchunk.length + ", offset = " + offset + ", headchunksize = " + headchunksize;
@ -657,8 +661,18 @@ public class kelondroRecords {
// create empty chunks
this.headChunk = new byte[headchunksize];
this.tailChunk = new byte[tailchunksize];
System.arraycopy(bulkchunk, offset, this.headChunk, 0, headchunksize);
System.arraycopy(bulkchunk, offset + headchunksize, this.tailChunk, 0, tailchunksize);
// write content to chunks
if (bulkchunk != null) {
System.arraycopy(bulkchunk, offset, this.headChunk, 0, headchunksize);
System.arraycopy(bulkchunk, offset + headchunksize, this.tailChunk, 0, tailchunksize);
}
// mark chunks as changed
// if the head/tail chunks come from a file system read, setChanged should be false
// if the chunks come from a overwrite attempt, it should be true
this.headChanged = setChanged;
this.tailChanged = setChanged;
}
protected Node(Handle handle, Node parentNode, int referenceInParent) throws IOException {
@ -1246,7 +1260,7 @@ public class kelondroRecords {
}
// read node from bulk
Node n = new Node(new Handle(pos.index), bulk, (pos.index - bulkstart) * recordsize);
Node n = new Node(new Handle(pos.index), bulk, (pos.index - bulkstart) * recordsize, false);
pos.index++;
while ((markedDeleted.contains(pos)) && (pos.index < USAGE.allCount())) pos.index++;
return n;

7
source/de/anomic/plasma/plasmaWordIndex.java
Unescape View File

@ -6,7 +6,7 @@
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
// $LastChangedBy: $
//
// LICENSE
//
@ -190,8 +190,9 @@ public final class plasmaWordIndex implements indexRI {
private void flushCacheSome(indexRAMRI ram, boolean busy) {
int flushCount = (busy) ? ram.size() / busyDivisor : ram.size() / idleDivisor;
if (flushCount > 100) flushCount = 100;
if (flushCount < 1) flushCount = Math.min(1, ram.size());
flushCache(ram, flushCount);
if (flushCount >= 1) {
flushCache(ram, flushCount);
}
while (ram.maxURLinCache() >= 2040) flushCache(ram, 1);
}

Write Preview
Loading…
Cancel
Save