boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

bugfixing for new kelondroFlexTable

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2204 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent 958408ca4a
commit e8848a3532
6 changed files with 104 additions and 64 deletions
Show Stats Download Patch File Download Diff File

32
source/dbtest.java
Unescape View File

@ -28,13 +28,9 @@ public class dbtest {
public final static int valuelength = 223; // sum of all data length as defined in plasmaURL
//public final static long buffer = 0;
public final static long buffer = 8192 * 1024; // 8 MB buffer
public static byte[] dummyvalue1 = new byte[valuelength];
public static byte[] dummyvalue2 = new byte[valuelength];
static {
// fill the dummy value
for (int i = 0; i < valuelength; i++) dummyvalue1[i] = '.';
dummyvalue1[0] = '[';
dummyvalue1[valuelength - 1] = ']';
for (int i = 0; i < valuelength; i++) dummyvalue2[i] = '-';
dummyvalue2[0] = '{';
dummyvalue2[valuelength - 1] = '}';
@ -169,7 +165,7 @@ public class dbtest {
profiler.start();
// create the database access
kelondroRow testRow = new kelondroRow(new int[]{keylength, valuelength, valuelength});
kelondroRow testRow = new kelondroRow(new int[]{keylength, keylength, valuelength});
if (dbe.equals("kelondroTree")) {
File tablefile = new File(tablename + ".kelondro.db");
if (tablefile.exists()) {
@ -213,8 +209,29 @@ public class dbtest {
long count = Long.parseLong(args[3]);
long randomstart = Long.parseLong(args[4]);
Random random = new Random(randomstart);
byte[] key;
for (int i = 0; i < count; i++) {
table.put(table.row().newEntry(new byte[][]{randomHash(random), dummyvalue1, dummyvalue2}));
key = randomHash(random);
table.put(table.row().newEntry(new byte[][]{key, key, dummyvalue2}));
if (i % 500 == 0) {
System.out.println(i + " entries processed so far.");
}
}
}
if (command.equals("read")) {
// read the database and compare with random entries;
// args: <number-of-entries> <random-startpoint>
long count = Long.parseLong(args[3]);
long randomstart = Long.parseLong(args[4]);
Random random = new Random(randomstart);
kelondroRow.Entry entry;
byte[] key;
for (int i = 0; i < count; i++) {
key = randomHash(random);
entry = table.get(key);
if (entry == null) System.out.println("missing value for entry " + new String(key)); else
if (!(new String(entry.getColBytes(1)).equals(new String(key)))) System.out.println("wrong value for entry " + new String(key) + ": " + new String(entry.getColBytes(1)));
if (i % 500 == 0) {
System.out.println(i + " entries processed so far.");
}
@ -267,11 +284,12 @@ public class dbtest {
}
}
long aftercommand = System.currentTimeMillis();
// finally close the database/table
if (table instanceof kelondroTree) ((kelondroTree) table).close();
if (table instanceof kelondroFlexTable) ((kelondroFlexTable) table).close();
if (table instanceof kelondroSplittedTree) ((kelondroSplittedTree) table).close();
if (table instanceof dbTable) ((dbTable)table).closeDatabaseConnection();
long afterclose = System.currentTimeMillis();

3
source/de/anomic/kelondro/kelondroCollection.java
Unescape View File

@ -180,8 +180,7 @@ public class kelondroCollection {
}
private void remove(int p) {
if (chunkcount == 0) return;
if ((p < 0) || (p >= chunkcount)) return; // out of bounds, nothing to delete
assert ((p >= 0) && (p < chunkcount) && (chunkcount > 0));
System.arraycopy(chunkcache, (p + 1) * chunksize, chunkcache, p * chunksize, (chunkcount - p - 1) * chunksize);
chunkcount--;
if (p < sortbound) sortbound--;

31
source/de/anomic/kelondro/kelondroCollectionIntMap.java
Unescape View File

@ -38,14 +38,14 @@ public class kelondroCollectionIntMap extends kelondroCollection {
this.setOrdering(kelondroNaturalOrder.naturalOrder);
}
public void addi(byte[] key, int i) {
public synchronized void addi(byte[] key, int i) {
kelondroRow.Entry indexentry = indexrow.newEntry();
indexentry.setCol(0, key);
indexentry.setColLongB256(1, i);
add(indexentry.bytes());
}
public int puti(byte[] key, int i) {
public synchronized int puti(byte[] key, int i) {
int index = -1;
synchronized (chunkcache) {
index = find(key, key.length);
@ -65,23 +65,26 @@ public class kelondroCollectionIntMap extends kelondroCollection {
}
}
public int geti(byte[] key) {
public synchronized int geti(byte[] key) {
int index = -1;
synchronized (chunkcache) {
index = find(key, key.length);
}
if (index < 0) {
return -1;
} else {
kelondroRow.Entry indexentry = indexrow.newEntry(get(index));
return (int) indexentry.getColLongB256(1);
if (index < 0) {
return -1;
} else {
kelondroRow.Entry indexentry = indexrow.newEntry(get(index));
return (int) indexentry.getColLongB256(1);
}
}
}
public int removei(byte[] key) {
byte[] b = remove(key);
if (b == null) return -1;
kelondroRow.Entry indexentry = indexrow.newEntry(b);
return (int) indexentry.getColLongB256(1);
public synchronized int removei(byte[] key) {
byte[] b;
synchronized (chunkcache) {
b = remove(key);
if (b == null) return -1;
kelondroRow.Entry indexentry = indexrow.newEntry(b);
return (int) indexentry.getColLongB256(1);
}
}
}

45
source/de/anomic/kelondro/kelondroFlexTable.java
Unescape View File

@ -1,4 +1,4 @@
// kelondroFrexTable.java
// kelondroFlexTable.java
// (C) 2006 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany
// first published 01.06.2006 on http://www.anomic.de
//
@ -73,28 +73,37 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
}
*/
public kelondroRow.Entry get(byte[] key) throws IOException {
int i = index.geti(key);
if (i < 0) return null;
return super.get(i);
public synchronized kelondroRow.Entry get(byte[] key) throws IOException {
synchronized (index) {
int i = index.geti(key);
if (i >= this.size()) {
System.out.println("errror");
}
if (i < 0) return null;
return super.get(i);
}
}
public kelondroRow.Entry put(kelondroRow.Entry row) throws IOException {
int i = index.geti(row.getColBytes(0));
if (i < 0) {
index.puti(row.getColBytes(0), super.add(row));
return null;
} else {
return super.set(i, row);
public synchronized kelondroRow.Entry put(kelondroRow.Entry row) throws IOException {
synchronized (index) {
int i = index.geti(row.getColBytes(0));
if (i < 0) {
index.puti(row.getColBytes(0), super.add(row));
return null;
} else {
return super.set(i, row);
}
}
}
public kelondroRow.Entry remove(byte[] key) throws IOException {
int i = index.removei(key);
if (i < 0) return null;
kelondroRow.Entry r = super.get(i);
super.remove(i);
return r;
public synchronized kelondroRow.Entry remove(byte[] key) throws IOException {
synchronized (index) {
int i = index.removei(key);
if (i < 0) return null;
kelondroRow.Entry r = super.get(i);
super.remove(i);
return r;
}
}
}

53
source/de/anomic/kelondro/kelondroFlexWidthArray.java
Unescape View File

@ -44,7 +44,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
check += '_';
}
// open existing files
// check if tabel directory exists
File tabledir = new File(path, tablename + ".table");
if (tabledir.exists()) {
if (!(tabledir.isDirectory())) throw new IOException("path " + tabledir.toString() + " must be a directory");
@ -52,6 +52,8 @@ public class kelondroFlexWidthArray implements kelondroArray {
tabledir.mkdirs();
tabledir.mkdir();
}
// open existing files
String[] files = tabledir.list();
for (int i = 0; i < files.length; i++) {
if ((files[i].startsWith("col.") && (files[i].endsWith(".list")))) {
@ -85,6 +87,10 @@ public class kelondroFlexWidthArray implements kelondroArray {
}
}
public void close() throws IOException {
for (int i = 0; i < col.length; i++) if (col[i] != null) col[i].close();
}
protected static final String colfilename(int start, int end) {
String f = Integer.toString(end);
while (f.length() < 3) f = "0" + f;
@ -126,6 +132,29 @@ public class kelondroFlexWidthArray implements kelondroArray {
return p;
}
public int add(kelondroRow.Entry rowentry) throws IOException {
kelondroRow.Entry e;
int index = -1;
int lastcol;
synchronized (col) {
e = col[0].row().newEntry(rowentry.bytes(), 0, rowdef.width(0));
index = col[0].add(e);
int c = col[0].row().columns();
while (c < rowdef.columns()) {
lastcol = c + col[c].row().columns() - 1;
e = col[c].row().newEntry(
rowentry.bytes(),
rowdef.colstart[c],
rowdef.colstart[lastcol] - rowdef.colstart[c]
+ rowdef.width(lastcol));
col[c].set(index, e);
c = c + col[c].row().columns();
}
}
return index;
}
public kelondroRow.Entry get(int index) throws IOException {
int r = 0;
kelondroRow.Entry e, p;
@ -141,28 +170,6 @@ public class kelondroFlexWidthArray implements kelondroArray {
return p;
}
public int add(kelondroRow.Entry rowentry) throws IOException {
kelondroRow.Entry e;
int index = -1;
synchronized (col) {
e = col[0].row().newEntry(rowentry.bytes(), 0, rowdef.width(0));
index = col[0].add(e);
int r = col[0].row().columns();
while (r < rowdef.columns()) {
e = col[r].row().newEntry(
rowentry.bytes(),
rowdef.colstart[r],
rowdef.colstart[r + col[r].row().columns() - 1]
+ rowdef.width(r + col[r].row().columns() - 1)
- rowdef.colstart[r]);
col[r].set(index, e);
r = r + col[r].row().columns();
}
}
return index;
}
public void remove(int index) throws IOException {
int r = 0;
synchronized (col) {

4
source/de/anomic/kelondro/kelondroSplittedTree.java
Unescape View File

@ -128,6 +128,10 @@ public class kelondroSplittedTree implements kelondroIndex {
}
}
public void close() throws IOException {
for (int i = 0; i < ktfs.length; i++) ktfs[i].close();
}
public kelondroRow row() {
return ktfs[0].row();
}

Write Preview
Loading…
Cancel
Save