boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

integrated kelondroRow into kelondroStack

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2156 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent 5bb565944f
commit 3c3c047d0a
9 changed files with 205 additions and 178 deletions
Show Stats Download Patch File Download Diff File

10
source/de/anomic/kelondro/kelondroBase64Order.java
Unescape View File

@ -134,13 +134,19 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
}
public long decodeLong(String s) {
while (s.endsWith("="))
s = s.substring(0, s.length() - 1);
while (s.endsWith("=")) s = s.substring(0, s.length() - 1);
long c = 0;
for (int i = 0; i < s.length(); i++) c = (c << 6) | ahpla[s.charAt(i)];
return c;
}
public long decodeLong(byte[] s, int offset, int length) {
while ((length > 0) && (s[offset + length - 1] == '=')) length--;
long c = 0;
for (int i = 0; i < length; i++) c = (c << 6) | ahpla[s[offset + i]];
return c;
}
public static long max(int len) {
// computes the maximum number that can be coded with a base64-encoded
// String of base len

2
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -189,7 +189,7 @@ public class kelondroCollectionIndex {
kelondroRow.Entry arrayrow = array[partitionnumber].get(rownumber);
if (arrayrow == null) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, partitionnumber).toString(), "array does not contain expected row");
// read the row and define a collection
int chunkcountInArray = (int) arrayrow.getColLong(1);
int chunkcountInArray = (int) arrayrow.getColLongB256(1);
if (chunkcountInArray != chunkcount) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, partitionnumber).toString(), "array has different chunkcount than index: index = " + chunkcount + ", array = " + chunkcountInArray);
return new kelondroCollection(chunksize, chunkcount, arrayrow.getColString(2, null), arrayrow.getColBytes(3));
}

2
source/de/anomic/kelondro/kelondroHashtable.java
Unescape View File

@ -233,7 +233,7 @@ public class kelondroHashtable {
rowNumber = hash.node();
if (rowNumber >= hashArray.size()) return new Object[]{new Integer(rowNumber), null};
hkrow = hashArray.get(rowNumber);
rowKey = (int) hkrow.getColLong(0);
rowKey = (int) hkrow.getColLongB256(0);
if (rowKey == 0) return new Object[]{new Integer(rowNumber), null};
hash.rehash();
} while (rowKey != hash.key());

11
source/de/anomic/kelondro/kelondroRow.java
Unescape View File

@ -175,6 +175,7 @@ public class kelondroRow {
public String getColString(int column, String encoding) {
int length = row[column].cellwidth();
int offset = colstart[column];
if (rowinstance[offset] == 0) return null;
if (length > rowinstance.length - offset) length = rowinstance.length - offset;
while ((length > 0) && (rowinstance[offset + length - 1] == 0)) length--;
if (length == 0) return null;
@ -188,10 +189,18 @@ public class kelondroRow {
}
}
public long getColLong(int column) {
public long getColLongB256(int column) {
return kelondroNaturalOrder.decodeLong(rowinstance, colstart[column], row[column].cellwidth());
}
public long getColLongB64E(int column) {
return kelondroBase64Order.enhancedCoder.decodeLong(rowinstance, colstart[column], row[column].cellwidth());
}
public byte getColByte(int column) {
return rowinstance[colstart[column]];
}
public byte[] getColBytes(int column) {
byte[] c = new byte[row[column].cellwidth()];
System.arraycopy(rowinstance, colstart[column], c, 0, row[column].cellwidth());

287
source/de/anomic/kelondro/kelondroStack.java
Unescape View File

@ -72,7 +72,7 @@ public final class kelondroStack extends kelondroRecords {
}
public kelondroStack(File file, long buffersize, int[] columns, boolean exitOnFail) {
// this creates a new tree
// this creates a new stack
super(file, buffersize, thisOHBytes, thisOHHandles, columns, thisFHandles, columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
try {
setHandle(root, null); // define the root value
@ -85,15 +85,15 @@ public final class kelondroStack extends kelondroRecords {
}
public kelondroStack(File file, long buffersize) throws IOException{
// this opens a file with an existing tree
super(file, buffersize);
// this opens a file with an existing stack
super(file, buffersize);
if ((getHandle(root) == null) && (getHandle(toor) == null)) clear();
}
public void clear() throws IOException {
super.clear();
setHandle(root, null); // reset the root value
setHandle(toor, null); // reset the toor value
setHandle(toor, null); // reset the toor value
}
public static kelondroStack reset(kelondroStack stack) {
@ -111,118 +111,121 @@ public final class kelondroStack extends kelondroRecords {
}
public class Counter implements Iterator {
Handle nextHandle = null;
public Counter() {
nextHandle = getHandle(root);
}
public boolean hasNext() {
return (nextHandle != null);
}
public Object next() {
Handle ret = nextHandle;
try {
nextHandle = getNode(nextHandle, null, 0).getOHHandle(right);
Handle nextHandle = null;
public Counter() {
nextHandle = getHandle(root);
}
public boolean hasNext() {
return (nextHandle != null);
}
public Object next() {
Handle ret = nextHandle;
try {
nextHandle = getNode(nextHandle, null, 0).getOHHandle(right);
return getNode(ret, null, 0);
} catch (IOException e) {
} catch (IOException e) {
throw new kelondroException(filename, "IO error at Counter:next()");
}
}
public void remove() {
throw new UnsupportedOperationException("no remove here..");
}
}
}
public void remove() {
throw new UnsupportedOperationException("no remove here..");
}
}
public synchronized void push(byte[][] row) throws IOException {
if (row.length != columns()) throw new IllegalArgumentException("push: wrong row length " + row.length + "; must be " + columns());
// check if there is already a stack
if (getHandle(toor) == null) {
if (getHandle(root) != null) throw new RuntimeException("push: internal organisation of root and toor");
// create node
Node n = newNode();
n.setValueCells(row);
n.setOHHandle(left, null);
public synchronized void push(kelondroRow.Entry row) throws IOException {
// check if there is already a stack
if (getHandle(toor) == null) {
if (getHandle(root) != null) throw new RuntimeException("push: internal organisation of root and toor");
// create node
Node n = newNode();
n.setValueRow(row.bytes());
n.setOHHandle(left, null);
n.setOHHandle(right, null);
n.commit(CP_NONE);
// assign handles
setHandle(root, n.handle());
setHandle(toor, n.handle());
// thats it
} else {
// expand the list at the end
Node n = newNode();
n.setValueCells(row);
n.setOHHandle(left, getHandle(toor));
n.commit(CP_NONE);
// assign handles
setHandle(root, n.handle());
setHandle(toor, n.handle());
// thats it
} else {
// expand the list at the end
Node n = newNode();
n.setValueRow(row.bytes());
n.setOHHandle(left, getHandle(toor));
n.setOHHandle(right, null);
Node n1 = getNode(getHandle(toor), null, 0);
Node n1 = getNode(getHandle(toor), null, 0);
n1.setOHHandle(right, n.handle());
n.commit(CP_NONE);
n1.commit(CP_NONE);
// assign handles
setHandle(toor, n.handle());
// thats it
}
n.commit(CP_NONE);
n1.commit(CP_NONE);
// assign handles
setHandle(toor, n.handle());
// thats it
}
}
public synchronized byte[][] pop() throws IOException {
// return row ontop of the stack and shrink stack by one
return pop(0);
public synchronized kelondroRow.Entry pop() throws IOException {
// return row ontop of the stack and shrink stack by one
return pop(0);
}
public synchronized byte[][] pop(int dist) throws IOException {
// return row relative to top of the stack and remove addressed element
Node n = topNode(dist);
public synchronized kelondroRow.Entry pop(int dist) throws IOException {
// return row relative to top of the stack and remove addressed element
Node n = topNode(dist);
if (n == null) return null;
byte[][] ret = n.getValueCells();
kelondroRow.Entry ret = row().newEntry(n.getValueRow());
// remove node
unlinkNode(n);
deleteNode(n.handle());
return ret;
deleteNode(n.handle());
return ret;
}
public synchronized byte[][] top() throws IOException {
// return row ontop of the stack
return top(0);
public synchronized kelondroRow.Entry top() throws IOException {
// return row ontop of the stack
return top(0);
}
public synchronized byte[][] top(int dist) throws IOException {
// return row ontop of the stack
// with dist == 0 this is the same function as with top()
public synchronized kelondroRow.Entry top(int dist) throws IOException {
// return row ontop of the stack
// with dist == 0 this is the same function as with top()
Node n = topNode(dist);
if (n == null) return null;
return n.getValueCells();
return row().newEntry(n.getValueRow());
}
public synchronized byte[][] pot() throws IOException {
// return row on the bottom of the stack and remove record
return pot(0);
public synchronized kelondroRow.Entry pot() throws IOException {
// return row on the bottom of the stack and remove record
return pot(0);
}
public synchronized byte[][] pot(int dist) throws IOException {
// return row relative to the bottom of the stack and remove addressed element
Node n = botNode(dist);
public synchronized kelondroRow.Entry pot(int dist) throws IOException {
// return row relative to the bottom of the stack and remove addressed element
Node n = botNode(dist);
if (n == null) return null;
byte[][] ret = n.getValueCells();
kelondroRow.Entry ret = row().newEntry(n.getValueCells());
// remove node
unlinkNode(n);
deleteNode(n.handle());
return ret;
deleteNode(n.handle());
return ret;
}
public synchronized byte[][] bot() throws IOException {
// return row on the bottom of the stack
return bot(0);
public synchronized kelondroRow.Entry bot() throws IOException {
// return row on the bottom of the stack
return bot(0);
}
public synchronized byte[][] bot(int dist) throws IOException {
// return row on bottom of the stack
// with dist == 0 this is the same function as with bot()
Node n = botNode(dist);
if (n == null) return null;
return n.getValueCells();
public synchronized kelondroRow.Entry bot(int dist) throws IOException {
// return row on bottom of the stack
// with dist == 0 this is the same function as with bot()
Node n = botNode(dist);
if (n == null) return null;
return row().newEntry(n.getValueRow());
}
public synchronized ArrayList botList(int dist) throws IOException {
@ -235,59 +238,51 @@ public final class kelondroStack extends kelondroRecords {
private void unlinkNode(Node n) throws IOException {
// join chaines over node
Handle l = n.getOHHandle(left);
Handle l = n.getOHHandle(left);
Handle r = n.getOHHandle(right);
// look left
if (l == null) {
// reached the root on left side
setHandle(root, r);
} else {
// un-link the previous record
Node k = getNode(l, null, 0);
k.setOHHandle(left, k.getOHHandle(left));
if (l == null) {
// reached the root on left side
setHandle(root, r);
} else {
// un-link the previous record
Node k = getNode(l, null, 0);
k.setOHHandle(left, k.getOHHandle(left));
k.setOHHandle(right, r);
k.commit(CP_NONE);
}
}
// look right
if (r == null) {
// reached the root on right side
setHandle(toor, l);
} else {
// un-link the following record
Node k = getNode(r, null, 0);
k.setOHHandle(left, l);
// reached the root on right side
setHandle(toor, l);
} else {
// un-link the following record
Node k = getNode(r, null, 0);
k.setOHHandle(left, l);
k.setOHHandle(right, k.getOHHandle(right));
k.commit(CP_NONE);
}
}
}
private Node topNode(int dist) throws IOException {
// return node ontop of the stack
// return node ontop of the stack
return queueNode(dist, toor, left);
}
private Node botNode(int dist) throws IOException {
// return node on bottom of the stack
// return node on bottom of the stack
return queueNode(dist, root, right);
}
private Node queueNode(int dist, int side, int dir) throws IOException {
// with dist == 0 this is the same function as with getNode(getHandle(side), null, 0)
Handle h = getHandle(side);
if (h == null) return null;
if (dist >= size()) return null; // that would exceed the stack
while (dist-- > 0) h = getNode(h).getOHHandle(dir); // track through elements
return getNode(h);
}
/*
public synchronized byte[][] seekPop(byte[] key, long maxdepth) throws IOException {
}
public synchronized byte[][] seekPot(byte[] key, long maxdepth) throws IOException {
// with dist == 0 this is the same function as with
// getNode(getHandle(side), null, 0)
Handle h = getHandle(side);
if (h == null) return null;
if (dist >= size()) return null; // that would exceed the stack
while (dist-- > 0) h = getNode(h).getOHHandle(dir); // track through elements
return getNode(h);
}
*/
public Iterator iterator() {
// iterates the elements in an ordered way. returns Node - type Objects
@ -302,7 +297,7 @@ public final class kelondroStack extends kelondroRecords {
String s;
StringTokenizer st;
int recs = 0;
byte[][] buffer = new byte[columns()][];
kelondroRow.Entry buffer = row().newEntry();
int c;
int line = 0;
while ((s = f.readLine()) != null) {
@ -313,7 +308,7 @@ public final class kelondroStack extends kelondroRecords {
// buffer the entry
c = 0;
while ((c < columns()) && (st.hasMoreTokens())) {
buffer[c++] = st.nextToken().trim().getBytes();
buffer.setCol(c++, st.nextToken().trim().getBytes());
}
if ((st.hasMoreTokens()) || (c != columns())) {
System.err.println("inapropriate number of entries in line " + line);
@ -331,28 +326,34 @@ public final class kelondroStack extends kelondroRecords {
}
public String hp(Handle h) {
if (h == null) return "NULL"; else return h.toString();
if (h == null)
return "NULL";
else
return h.toString();
}
public void print() throws IOException {
super.print(false);
Node n;
try {
Iterator it = iterator();
while (it.hasNext()) {
n = (Node) it.next();
//n = getNode(h, null, 0);
System.out.println("> NODE " + hp(n.handle()) +
"; left " + hp(n.getOHHandle(left)) + ", right " + hp(n.getOHHandle(right)));
System.out.print(" KEY:'" + (new String(n.getValueCells()[0])).trim() + "'");
for (int j = 1; j < columns(); j++)
System.out.print(", V[" + j + "]:'" + (new String(n.getValueCells()[j])).trim() + "'");
System.out.println();
}
System.out.println();
} catch (IOException e) {
System.out.println("File error: " + e.getMessage());
}
super.print(false);
Node n;
try {
Iterator it = iterator();
while (it.hasNext()) {
n = (Node) it.next();
// n = getNode(h, null, 0);
System.out.println("> NODE " + hp(n.handle()) + "; left "
+ hp(n.getOHHandle(left)) + ", right "
+ hp(n.getOHHandle(right)));
System.out.print(" KEY:'"
+ (new String(n.getValueCells()[0])).trim() + "'");
for (int j = 1; j < columns(); j++)
System.out.print(", V[" + j + "]:'"
+ (new String(n.getValueCells()[j])).trim() + "'");
System.out.println();
}
System.out.println();
} catch (IOException e) {
System.out.println("File error: " + e.getMessage());
}
}
private static void cmd(String[] args) {
@ -372,8 +373,8 @@ public final class kelondroStack extends kelondroRecords {
ret = null;
} else if (args[0].equals("-g")) {
fm = new kelondroStack(new File(args[1]), 0x100000);
byte[][] ret2 = fm.pop();
ret = ((ret2 == null) ? null : ret2[1]);
kelondroRow.Entry ret2 = fm.pop();
ret = ((ret2 == null) ? null : ret2.getColBytes(1));
fm.close();
}
fm.close();
@ -404,8 +405,8 @@ public final class kelondroStack extends kelondroRecords {
}
} else if (args[0].equals("-g")) {
kelondroStack fm = new kelondroStack(new File(args[2]), 0x100000);
byte[][] ret2 = fm.pop(Integer.parseInt(args[1]));
ret = ((ret2 == null) ? null : ret2[1]);
kelondroRow.Entry ret2 = fm.pop(Integer.parseInt(args[1]));
ret = ((ret2 == null) ? null : ret2.getColBytes(1));
fm.close();
}
} else if (args.length == 4) {
@ -420,7 +421,7 @@ public final class kelondroStack extends kelondroRecords {
fm.close();
} else if (args[0].equals("-p")) {
kelondroStack fm = new kelondroStack(new File(args[3]), 0x100000);
fm.push(new byte[][] {args[1].getBytes(), args[2].getBytes()});
fm.push(fm.row().newEntry(new byte[][] {args[1].getBytes(), args[2].getBytes()}));
fm.close();
}
}

8
source/de/anomic/plasma/plasmaCrawlBalancer.java
Unescape View File

@ -108,7 +108,7 @@ public class plasmaCrawlBalancer {
while (i.hasNext()) {
entry = (Map.Entry) i.next();
list = (ArrayList) entry.getValue();
stack.push(new byte[][]{(byte[]) list.remove(0)});
stack.push(stack.row().newEntry(new byte[][]{(byte[]) list.remove(0)}));
if (list.size() == 0) i.remove();
}
}
@ -141,10 +141,10 @@ public class plasmaCrawlBalancer {
// returns a pair of domain/hash from the stack
// if the domain is unknown, a null/hash is returned
if (stack.size() > 0) {
return new Object[]{null, stack.pop()[0]};
return new Object[]{null, stack.pop().getColBytes(0)};
} else if (domainStacks.size() > 0) {
flushOnce();
return new Object[]{null, stack.pop()[0]};
return new Object[]{null, stack.pop().getColBytes(0)};
} else {
return null;
}
@ -152,7 +152,7 @@ public class plasmaCrawlBalancer {
public synchronized byte[] top(int dist) throws IOException {
flushAll();
return stack.top(dist)[0];
return stack.top(dist).getColBytes(0);
}
public void clear() throws IOException {

10
source/de/anomic/plasma/plasmaCrawlNURL.java
Unescape View File

@ -314,9 +314,9 @@ public class plasmaCrawlNURL extends indexURL {
case STACK_TYPE_LIMIT: limitStack.add(domain, hash.getBytes()); break;
case STACK_TYPE_OVERHANG: overhangStack.add(domain, hash.getBytes()); break;
case STACK_TYPE_REMOTE: remoteStack.add(domain, hash.getBytes()); break;
case STACK_TYPE_IMAGE: imageStack.push(new byte[][] {hash.getBytes()}); break;
case STACK_TYPE_MOVIE: movieStack.push(new byte[][] {hash.getBytes()}); break;
case STACK_TYPE_MUSIC: musicStack.push(new byte[][] {hash.getBytes()}); break;
case STACK_TYPE_IMAGE: imageStack.push(imageStack.row().newEntry(new byte[][] {hash.getBytes()})); break;
case STACK_TYPE_MOVIE: movieStack.push(movieStack.row().newEntry(new byte[][] {hash.getBytes()})); break;
case STACK_TYPE_MUSIC: musicStack.push(musicStack.row().newEntry(new byte[][] {hash.getBytes()})); break;
default: break;
}
stackIndex.add(hash);
@ -389,7 +389,7 @@ public class plasmaCrawlNURL extends indexURL {
private Entry pop(kelondroStack stack) throws IOException {
// this is a filo - pop
if (stack.size() > 0) {
Entry e = new Entry(new String(stack.pop()[0]));
Entry e = new Entry(new String(stack.pop().getColBytes(0)));
stackIndex.remove(e.hash);
return e;
} else {
@ -414,7 +414,7 @@ public class plasmaCrawlNURL extends indexURL {
ArrayList list = new ArrayList(count);
for (int i = 0; i < count; i++) {
try {
byte[] hash = stack.top(i)[0];
byte[] hash = stack.top(i).getColBytes(0);
list.add(new Entry(new String(hash)));
} catch (IOException e) {
continue;

28
source/de/anomic/plasma/plasmaSwitchboardQueue.java
Unescape View File

@ -50,6 +50,7 @@ import de.anomic.index.indexURL;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroStack;
import de.anomic.kelondro.kelondroRow;
import de.anomic.server.logging.serverLog;
import de.anomic.server.serverDate;
import de.anomic.yacy.yacySeedDB;
@ -118,7 +119,7 @@ public class plasmaSwitchboardQueue {
}
public void push(Entry entry) throws IOException {
sbQueueStack.push(new byte[][]{
sbQueueStack.push(sbQueueStack.row().newEntry(new byte[][]{
entry.url.toString().getBytes(),
(entry.referrerHash == null) ? indexURL.dummyHash.getBytes() : entry.referrerHash.getBytes(),
kelondroBase64Order.enhancedCoder.encodeLong((entry.ifModifiedSince == null) ? 0 : entry.ifModifiedSince.getTime(), 11).getBytes(),
@ -127,12 +128,12 @@ public class plasmaSwitchboardQueue {
kelondroBase64Order.enhancedCoder.encodeLong((long) entry.depth, indexURL.urlCrawlDepthLength).getBytes(),
(entry.profileHandle == null) ? indexURL.dummyHash.getBytes() : entry.profileHandle.getBytes(),
(entry.anchorName == null) ? "-".getBytes() : entry.anchorName.getBytes()
});
}));
}
public Entry pop() throws IOException {
if (sbQueueStack.size() == 0) return null;
byte[][] b = sbQueueStack.pot();
kelondroRow.Entry b = sbQueueStack.pot();
if (b == null) return null;
return new Entry(b);
}
@ -223,6 +224,27 @@ public class plasmaSwitchboardQueue {
this.referrerURL = null;
}
public Entry(kelondroRow.Entry row) throws IOException {
long ims = row.getColLongB64E(2);
byte flags = row.getColByte(3);
try {
this.url = new URL(row.getColString(0, "UTF-8"));
} catch (MalformedURLException e) {
this.url = null;
}
this.referrerHash = row.getColString(1, "UTF-8");
this.ifModifiedSince = (ims == 0) ? null : new Date(ims);
this.flags = ((flags & 1) == 1) ? (byte) 1 : (byte) 0;
this.initiator = row.getColString(4, "UTF-8");
this.depth = (int) row.getColLongB64E(5);
this.profileHandle = row.getColString(6, "UTF-8");
this.anchorName = row.getColString(7, "UTF-8");
this.profileEntry = null;
this.responseHeader = null;
this.referrerURL = null;
}
public Entry(byte[][] row) throws IOException {
long ims = (row[2] == null) ? 0 : kelondroBase64Order.enhancedCoder.decodeLong(new String(row[2], "UTF-8"));
byte flags = (row[3] == null) ? 0 : row[3][0];

25
source/de/anomic/yacy/yacyNewsQueue.java
Unescape View File

@ -48,6 +48,7 @@ import java.io.File;
import java.io.IOException;
import java.util.Date;
import de.anomic.kelondro.kelondroRow;
import de.anomic.kelondro.kelondroStack;
import de.anomic.kelondro.kelondroException;
@ -140,26 +141,14 @@ public class yacyNewsQueue {
return null;
}
/*
public synchronized void incDistributedCounter(yacyNewsRecord entry) throws IOException {
// this works only if the entry element lies ontop of the stack
yacyNewsRecord topEntry = top();
if (!(topEntry.id().equals(entry.id()))) throw new IllegalArgumentException("entry is not ontop of the stack");
pop();
entry.incDistribution();
push(entry);
}
*/
private yacyNewsRecord b2r(byte[][] b) throws IOException {
private yacyNewsRecord b2r(kelondroRow.Entry b) throws IOException {
if (b == null) return null;
String id = new String(b[0]);
String id = b.getColString(0, null);
//Date touched = yacyCore.parseUniversalDate(new String(b[1]));
return newsDB.get(id);
}
private byte[][] r2b(yacyNewsRecord r, boolean updateDB) throws IOException {
private kelondroRow.Entry r2b(yacyNewsRecord r, boolean updateDB) throws IOException {
if (r == null) return null;
if (updateDB) {
newsDB.put(r);
@ -167,9 +156,9 @@ public class yacyNewsQueue {
yacyNewsRecord r1 = newsDB.get(r.id());
if (r1 == null) newsDB.put(r);
}
byte[][] b = new byte[2][];
b[0] = r.id().getBytes();
b[1] = yacyCore.universalDateShortString(new Date()).getBytes();
kelondroRow.Entry b = queueStack.row().newEntry(new byte[][]{
r.id().getBytes(),
yacyCore.universalDateShortString(new Date()).getBytes()});
return b;
}

Write Preview
Loading…
Cancel
Save