boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

some storage process enhancements (write without preceding read)

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3348 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 18 years ago
parent faad869865
commit daf2e15f59
9 changed files with 69 additions and 20 deletions
Show Stats Download Patch File Download Diff File

2
source/de/anomic/index/indexRAMRI.java
Unescape View File

@ -139,7 +139,7 @@ public final class indexRAMRI implements indexRI {
row.setCol(1, occ); row.setCol(1, occ);
row.setCol(2, time); row.setCol(2, time);
row.setCol(3, iEntry.toKelondroEntry().bytes()); row.setCol(3, iEntry.toKelondroEntry().bytes());
dumpArray.set((int) urlcount++, row); dumpArray.overwrite((int) urlcount++, row);
} }
} }
wordcount++; wordcount++;

3
source/de/anomic/kelondro/kelondroArray.java
Unescape View File

@ -32,7 +32,8 @@ public interface kelondroArray {
public kelondroRow row(); public kelondroRow row();
public kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException; public kelondroRow.Entry replace(int index, kelondroRow.Entry rowentry) throws IOException;
public void overwrite(int index, kelondroRow.Entry rowentry) throws IOException;
public kelondroRow.Entry get(int index) throws IOException; public kelondroRow.Entry get(int index) throws IOException;

2
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -377,7 +377,7 @@ public class kelondroCollectionIndex {
arrayEntry.setCol(1, collection.exportCollection()); arrayEntry.setCol(1, collection.exportCollection());
// overwrite entry in this array // overwrite entry in this array
array.set(oldrownumber, arrayEntry); array.overwrite(oldrownumber, arrayEntry);
// update the index entry // update the index entry
indexrow.setCol(idx_col_chunkcount, collection.size()); indexrow.setCol(idx_col_chunkcount, collection.size());

21
source/de/anomic/kelondro/kelondroFixedWidthArray.java
Unescape View File

@ -85,7 +85,7 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
} }
} }
public synchronized kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException { public synchronized kelondroRow.Entry replace(int index, kelondroRow.Entry rowentry) throws IOException {
// make room for element // make room for element
Node n; Node n;
@ -100,10 +100,25 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
// write the row // write the row
byte[] before = n.setValueRow((rowentry == null) ? null : rowentry.bytes()); byte[] before = n.setValueRow((rowentry == null) ? null : rowentry.bytes());
n.commit(CP_NONE); n.commit(CP_NONE);
return row().newEntry(before); return row().newEntry(before);
} }
public synchronized void overwrite(int index, kelondroRow.Entry rowentry) throws IOException {
// this writes a row without reading the row from the file system first
// make room for element
Node n;
while (super.USAGE.allCount() <= index) {
n = newNode();
n.commit(CP_NONE);
}
// create a node at position index with rowentry
n = newNode(new Handle(index), (rowentry == null) ? null : rowentry.bytes(), 0);
n.commit(CP_NONE);
}
public synchronized kelondroRow.Entry get(int index) throws IOException { public synchronized kelondroRow.Entry get(int index) throws IOException {
return row().newEntry(getNode(new Handle(index)).getValueRow()); return row().newEntry(getNode(new Handle(index)).getValueRow());
} }
@ -162,7 +177,7 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
System.out.println("erster Test"); System.out.println("erster Test");
f.delete(); f.delete();
kelondroFixedWidthArray k = new kelondroFixedWidthArray(f, rowdef, 6); kelondroFixedWidthArray k = new kelondroFixedWidthArray(f, rowdef, 6);
k.set(3, k.row().newEntry(new byte[][]{ k.overwrite(3, k.row().newEntry(new byte[][]{
"test123".getBytes(), "abcd".getBytes()})); "test123".getBytes(), "abcd".getBytes()}));
k.add(k.row().newEntry(new byte[][]{ k.add(k.row().newEntry(new byte[][]{
"test456".getBytes(), "efgh".getBytes()})); "test456".getBytes(), "efgh".getBytes()}));

2
source/de/anomic/kelondro/kelondroFlexTable.java
Unescape View File

@ -193,7 +193,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
index.puti(row.getColBytes(0), super.add(row)); index.puti(row.getColBytes(0), super.add(row));
return null; return null;
} }
return super.set(i, row); return super.replace(i, row);
} }
public synchronized void addUnique(kelondroRow.Entry row, Date entryDate) throws IOException { public synchronized void addUnique(kelondroRow.Entry row, Date entryDate) throws IOException {

26
source/de/anomic/kelondro/kelondroFlexWidthArray.java
Unescape View File

@ -176,7 +176,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
return col[0].size(); return col[0].size();
} }
public kelondroRow.Entry set(int index, kelondroRow.Entry rowentry) throws IOException { public kelondroRow.Entry replace(int index, kelondroRow.Entry rowentry) throws IOException {
assert rowentry.bytes().length == this.rowdef.objectsize; assert rowentry.bytes().length == this.rowdef.objectsize;
int c = 0; int c = 0;
kelondroRow.Entry e0, e1, p; kelondroRow.Entry e0, e1, p;
@ -189,7 +189,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
rowentry.bytes(), rowentry.bytes(),
rowdef.colstart[c], rowdef.colstart[c],
rowdef.colstart[lastcol] - rowdef.colstart[c] + rowdef.width(lastcol)); rowdef.colstart[lastcol] - rowdef.colstart[c] + rowdef.width(lastcol));
e1 = col[c].set(index, e0); e1 = col[c].replace(index, e0);
for (int i = 0; i < col[c].row().columns(); i++) { for (int i = 0; i < col[c].row().columns(); i++) {
p.setCol(c + i, e1.getColBytes(i)); p.setCol(c + i, e1.getColBytes(i));
} }
@ -199,6 +199,24 @@ public class kelondroFlexWidthArray implements kelondroArray {
return p; return p;
} }
public void overwrite(int index, kelondroRow.Entry rowentry) throws IOException {
assert rowentry.bytes().length == this.rowdef.objectsize;
int c = 0;
kelondroRow.Entry e0;
int lastcol;
synchronized (col) {
while (c < rowdef.columns()) {
lastcol = c + col[c].row().columns() - 1;
e0 = col[c].row().newEntry(
rowentry.bytes(),
rowdef.colstart[c],
rowdef.colstart[lastcol] - rowdef.colstart[c] + rowdef.width(lastcol));
col[c].overwrite(index, e0);
c = c + col[c].row().columns();
}
}
}
public int add(kelondroRow.Entry rowentry) throws IOException { public int add(kelondroRow.Entry rowentry) throws IOException {
assert rowentry.bytes().length == this.rowdef.objectsize; assert rowentry.bytes().length == this.rowdef.objectsize;
kelondroRow.Entry e; kelondroRow.Entry e;
@ -215,7 +233,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
rowentry.bytes(), rowentry.bytes(),
rowdef.colstart[c], rowdef.colstart[c],
rowdef.colstart[lastcol] + rowdef.width(lastcol) - rowdef.colstart[c]); rowdef.colstart[lastcol] + rowdef.width(lastcol) - rowdef.colstart[c]);
col[c].set(index,e); col[c].overwrite(index,e);
c = c + col[c].row().columns(); c = c + col[c].row().columns();
} }
} }
@ -247,7 +265,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
// the other columns will be blanked out only // the other columns will be blanked out only
while (r < rowdef.columns()) { while (r < rowdef.columns()) {
col[r].set(index, null); col[r].overwrite(index, null);
r = r + col[r].row().columns(); r = r + col[r].row().columns();
} }
} }

4
source/de/anomic/kelondro/kelondroHashtable.java
Unescape View File

@ -208,13 +208,13 @@ public class kelondroHashtable {
} }
// make space // make space
while (rowNumber >= hashArray.size()) hashArray.set(hashArray.size(), dummyRow); while (rowNumber >= hashArray.size()) hashArray.overwrite(hashArray.size(), dummyRow);
// write row // write row
kelondroRow.Entry newhkrow = hashArray.row().newEntry(); kelondroRow.Entry newhkrow = hashArray.row().newEntry();
newhkrow.setCol(0, hash.key()); newhkrow.setCol(0, hash.key());
newhkrow.setCol(1, rowentry.bytes()); newhkrow.setCol(1, rowentry.bytes());
hashArray.set(rowNumber, newhkrow); hashArray.overwrite(rowNumber, newhkrow);
return hashArray.row().newEntry(oldhkrow.getColBytes(1)); return hashArray.row().newEntry(oldhkrow.getColBytes(1));
} }

22
source/de/anomic/kelondro/kelondroRecords.java
Unescape View File

@ -583,6 +583,10 @@ public class kelondroRecords {
return new Node(); return new Node();
} }
protected final Node newNode(Handle handle, byte[] bulkchunk, int offset) {
return new Node(handle, bulkchunk, offset, true);
}
protected final Node getNode(Handle handle) throws IOException { protected final Node getNode(Handle handle) throws IOException {
return getNode(handle, null, 0); return getNode(handle, null, 0);
} }
@ -649,7 +653,7 @@ public class kelondroRecords {
for (int i = tailchunksize - 1; i >= 0; i--) this.tailChunk[i] = 0; for (int i = tailchunksize - 1; i >= 0; i--) this.tailChunk[i] = 0;
} }
protected Node(Handle handle, byte[] bulkchunk, int offset) { protected Node(Handle handle, byte[] bulkchunk, int offset, boolean setChanged) {
// this initializer is used to create nodes from bulk-read byte arrays // this initializer is used to create nodes from bulk-read byte arrays
this.handle = handle; this.handle = handle;
assert (bulkchunk.length >= offset + headchunksize) : "bulkchunk.length = " + bulkchunk.length + ", offset = " + offset + ", headchunksize = " + headchunksize; assert (bulkchunk.length >= offset + headchunksize) : "bulkchunk.length = " + bulkchunk.length + ", offset = " + offset + ", headchunksize = " + headchunksize;
@ -657,8 +661,18 @@ public class kelondroRecords {
// create empty chunks // create empty chunks
this.headChunk = new byte[headchunksize]; this.headChunk = new byte[headchunksize];
this.tailChunk = new byte[tailchunksize]; this.tailChunk = new byte[tailchunksize];
System.arraycopy(bulkchunk, offset, this.headChunk, 0, headchunksize);
System.arraycopy(bulkchunk, offset + headchunksize, this.tailChunk, 0, tailchunksize); // write content to chunks
if (bulkchunk != null) {
System.arraycopy(bulkchunk, offset, this.headChunk, 0, headchunksize);
System.arraycopy(bulkchunk, offset + headchunksize, this.tailChunk, 0, tailchunksize);
}
// mark chunks as changed
// if the head/tail chunks come from a file system read, setChanged should be false
// if the chunks come from a overwrite attempt, it should be true
this.headChanged = setChanged;
this.tailChanged = setChanged;
} }
protected Node(Handle handle, Node parentNode, int referenceInParent) throws IOException { protected Node(Handle handle, Node parentNode, int referenceInParent) throws IOException {
@ -1246,7 +1260,7 @@ public class kelondroRecords {
} }
// read node from bulk // read node from bulk
Node n = new Node(new Handle(pos.index), bulk, (pos.index - bulkstart) * recordsize); Node n = new Node(new Handle(pos.index), bulk, (pos.index - bulkstart) * recordsize, false);
pos.index++; pos.index++;
while ((markedDeleted.contains(pos)) && (pos.index < USAGE.allCount())) pos.index++; while ((markedDeleted.contains(pos)) && (pos.index < USAGE.allCount())) pos.index++;
return n; return n;

7
source/de/anomic/plasma/plasmaWordIndex.java
Unescape View File

@ -6,7 +6,7 @@
// //
// $LastChangedDate$ // $LastChangedDate$
// $LastChangedRevision$ // $LastChangedRevision$
// $LastChangedBy$ // $LastChangedBy: $
// //
// LICENSE // LICENSE
// //
@ -190,8 +190,9 @@ public final class plasmaWordIndex implements indexRI {
private void flushCacheSome(indexRAMRI ram, boolean busy) { private void flushCacheSome(indexRAMRI ram, boolean busy) {
int flushCount = (busy) ? ram.size() / busyDivisor : ram.size() / idleDivisor; int flushCount = (busy) ? ram.size() / busyDivisor : ram.size() / idleDivisor;
if (flushCount > 100) flushCount = 100; if (flushCount > 100) flushCount = 100;
if (flushCount < 1) flushCount = Math.min(1, ram.size()); if (flushCount >= 1) {
flushCache(ram, flushCount); flushCache(ram, flushCount);
}
while (ram.maxURLinCache() >= 2040) flushCache(ram, 1); while (ram.maxURLinCache() >= 2040) flushCache(ram, 1);
} }

Write Preview
Loading…
Cancel
Save