boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

integrated kelondroRow into kelondroStack

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2156 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent 5bb565944f
commit 3c3c047d0a
9 changed files with 205 additions and 178 deletions
Show Stats Download Patch File Download Diff File

10
source/de/anomic/kelondro/kelondroBase64Order.java
Unescape View File

@ -134,13 +134,19 @@ public class kelondroBase64Order extends kelondroAbstractOrder implements kelond
} }
public long decodeLong(String s) { public long decodeLong(String s) {
while (s.endsWith("=")) while (s.endsWith("=")) s = s.substring(0, s.length() - 1);
s = s.substring(0, s.length() - 1);
long c = 0; long c = 0;
for (int i = 0; i < s.length(); i++) c = (c << 6) | ahpla[s.charAt(i)]; for (int i = 0; i < s.length(); i++) c = (c << 6) | ahpla[s.charAt(i)];
return c; return c;
} }
public long decodeLong(byte[] s, int offset, int length) {
while ((length > 0) && (s[offset + length - 1] == '=')) length--;
long c = 0;
for (int i = 0; i < length; i++) c = (c << 6) | ahpla[s[offset + i]];
return c;
}
public static long max(int len) { public static long max(int len) {
// computes the maximum number that can be coded with a base64-encoded // computes the maximum number that can be coded with a base64-encoded
// String of base len // String of base len

2
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -189,7 +189,7 @@ public class kelondroCollectionIndex {
kelondroRow.Entry arrayrow = array[partitionnumber].get(rownumber); kelondroRow.Entry arrayrow = array[partitionnumber].get(rownumber);
if (arrayrow == null) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, partitionnumber).toString(), "array does not contain expected row"); if (arrayrow == null) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, partitionnumber).toString(), "array does not contain expected row");
// read the row and define a collection // read the row and define a collection
int chunkcountInArray = (int) arrayrow.getColLong(1); int chunkcountInArray = (int) arrayrow.getColLongB256(1);
if (chunkcountInArray != chunkcount) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, partitionnumber).toString(), "array has different chunkcount than index: index = " + chunkcount + ", array = " + chunkcountInArray); if (chunkcountInArray != chunkcount) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, partitionnumber).toString(), "array has different chunkcount than index: index = " + chunkcount + ", array = " + chunkcountInArray);
return new kelondroCollection(chunksize, chunkcount, arrayrow.getColString(2, null), arrayrow.getColBytes(3)); return new kelondroCollection(chunksize, chunkcount, arrayrow.getColString(2, null), arrayrow.getColBytes(3));
} }

2
source/de/anomic/kelondro/kelondroHashtable.java
Unescape View File

@ -233,7 +233,7 @@ public class kelondroHashtable {
rowNumber = hash.node(); rowNumber = hash.node();
if (rowNumber >= hashArray.size()) return new Object[]{new Integer(rowNumber), null}; if (rowNumber >= hashArray.size()) return new Object[]{new Integer(rowNumber), null};
hkrow = hashArray.get(rowNumber); hkrow = hashArray.get(rowNumber);
rowKey = (int) hkrow.getColLong(0); rowKey = (int) hkrow.getColLongB256(0);
if (rowKey == 0) return new Object[]{new Integer(rowNumber), null}; if (rowKey == 0) return new Object[]{new Integer(rowNumber), null};
hash.rehash(); hash.rehash();
} while (rowKey != hash.key()); } while (rowKey != hash.key());

11
source/de/anomic/kelondro/kelondroRow.java
Unescape View File

@ -175,6 +175,7 @@ public class kelondroRow {
public String getColString(int column, String encoding) { public String getColString(int column, String encoding) {
int length = row[column].cellwidth(); int length = row[column].cellwidth();
int offset = colstart[column]; int offset = colstart[column];
if (rowinstance[offset] == 0) return null;
if (length > rowinstance.length - offset) length = rowinstance.length - offset; if (length > rowinstance.length - offset) length = rowinstance.length - offset;
while ((length > 0) && (rowinstance[offset + length - 1] == 0)) length--; while ((length > 0) && (rowinstance[offset + length - 1] == 0)) length--;
if (length == 0) return null; if (length == 0) return null;
@ -188,10 +189,18 @@ public class kelondroRow {
} }
} }
public long getColLong(int column) { public long getColLongB256(int column) {
return kelondroNaturalOrder.decodeLong(rowinstance, colstart[column], row[column].cellwidth()); return kelondroNaturalOrder.decodeLong(rowinstance, colstart[column], row[column].cellwidth());
} }
public long getColLongB64E(int column) {
return kelondroBase64Order.enhancedCoder.decodeLong(rowinstance, colstart[column], row[column].cellwidth());
}
public byte getColByte(int column) {
return rowinstance[colstart[column]];
}
public byte[] getColBytes(int column) { public byte[] getColBytes(int column) {
byte[] c = new byte[row[column].cellwidth()]; byte[] c = new byte[row[column].cellwidth()];
System.arraycopy(rowinstance, colstart[column], c, 0, row[column].cellwidth()); System.arraycopy(rowinstance, colstart[column], c, 0, row[column].cellwidth());

287
source/de/anomic/kelondro/kelondroStack.java
Unescape View File

@ -72,7 +72,7 @@ public final class kelondroStack extends kelondroRecords {
} }
public kelondroStack(File file, long buffersize, int[] columns, boolean exitOnFail) { public kelondroStack(File file, long buffersize, int[] columns, boolean exitOnFail) {
// this creates a new tree // this creates a new stack
super(file, buffersize, thisOHBytes, thisOHHandles, columns, thisFHandles, columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail); super(file, buffersize, thisOHBytes, thisOHHandles, columns, thisFHandles, columns.length /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
try { try {
setHandle(root, null); // define the root value setHandle(root, null); // define the root value
@ -85,15 +85,15 @@ public final class kelondroStack extends kelondroRecords {
} }
public kelondroStack(File file, long buffersize) throws IOException{ public kelondroStack(File file, long buffersize) throws IOException{
// this opens a file with an existing tree // this opens a file with an existing stack
super(file, buffersize); super(file, buffersize);
if ((getHandle(root) == null) && (getHandle(toor) == null)) clear(); if ((getHandle(root) == null) && (getHandle(toor) == null)) clear();
} }
public void clear() throws IOException { public void clear() throws IOException {
super.clear(); super.clear();
setHandle(root, null); // reset the root value setHandle(root, null); // reset the root value
setHandle(toor, null); // reset the toor value setHandle(toor, null); // reset the toor value
} }
public static kelondroStack reset(kelondroStack stack) { public static kelondroStack reset(kelondroStack stack) {
@ -111,118 +111,121 @@ public final class kelondroStack extends kelondroRecords {
} }
public class Counter implements Iterator { public class Counter implements Iterator {
Handle nextHandle = null; Handle nextHandle = null;
public Counter() {
nextHandle = getHandle(root); public Counter() {
} nextHandle = getHandle(root);
public boolean hasNext() { }
return (nextHandle != null);
} public boolean hasNext() {
public Object next() { return (nextHandle != null);
Handle ret = nextHandle; }
try {
nextHandle = getNode(nextHandle, null, 0).getOHHandle(right); public Object next() {
Handle ret = nextHandle;
try {
nextHandle = getNode(nextHandle, null, 0).getOHHandle(right);
return getNode(ret, null, 0); return getNode(ret, null, 0);
} catch (IOException e) { } catch (IOException e) {
throw new kelondroException(filename, "IO error at Counter:next()"); throw new kelondroException(filename, "IO error at Counter:next()");
} }
} }
public void remove() {
throw new UnsupportedOperationException("no remove here.."); public void remove() {
} throw new UnsupportedOperationException("no remove here..");
}
} }
public synchronized void push(byte[][] row) throws IOException { public synchronized void push(kelondroRow.Entry row) throws IOException {
if (row.length != columns()) throw new IllegalArgumentException("push: wrong row length " + row.length + "; must be " + columns()); // check if there is already a stack
// check if there is already a stack if (getHandle(toor) == null) {
if (getHandle(toor) == null) { if (getHandle(root) != null) throw new RuntimeException("push: internal organisation of root and toor");
if (getHandle(root) != null) throw new RuntimeException("push: internal organisation of root and toor"); // create node
// create node Node n = newNode();
Node n = newNode(); n.setValueRow(row.bytes());
n.setValueCells(row); n.setOHHandle(left, null);
n.setOHHandle(left, null);
n.setOHHandle(right, null); n.setOHHandle(right, null);
n.commit(CP_NONE); n.commit(CP_NONE);
// assign handles // assign handles
setHandle(root, n.handle()); setHandle(root, n.handle());
setHandle(toor, n.handle()); setHandle(toor, n.handle());
// thats it // thats it
} else { } else {
// expand the list at the end // expand the list at the end
Node n = newNode(); Node n = newNode();
n.setValueCells(row); n.setValueRow(row.bytes());
n.setOHHandle(left, getHandle(toor)); n.setOHHandle(left, getHandle(toor));
n.setOHHandle(right, null); n.setOHHandle(right, null);
Node n1 = getNode(getHandle(toor), null, 0); Node n1 = getNode(getHandle(toor), null, 0);
n1.setOHHandle(right, n.handle()); n1.setOHHandle(right, n.handle());
n.commit(CP_NONE); n.commit(CP_NONE);
n1.commit(CP_NONE); n1.commit(CP_NONE);
// assign handles // assign handles
setHandle(toor, n.handle()); setHandle(toor, n.handle());
// thats it // thats it
} }
} }
public synchronized byte[][] pop() throws IOException { public synchronized kelondroRow.Entry pop() throws IOException {
// return row ontop of the stack and shrink stack by one // return row ontop of the stack and shrink stack by one
return pop(0); return pop(0);
} }
public synchronized byte[][] pop(int dist) throws IOException { public synchronized kelondroRow.Entry pop(int dist) throws IOException {
// return row relative to top of the stack and remove addressed element // return row relative to top of the stack and remove addressed element
Node n = topNode(dist); Node n = topNode(dist);
if (n == null) return null; if (n == null) return null;
byte[][] ret = n.getValueCells(); kelondroRow.Entry ret = row().newEntry(n.getValueRow());
// remove node // remove node
unlinkNode(n); unlinkNode(n);
deleteNode(n.handle()); deleteNode(n.handle());
return ret; return ret;
} }
public synchronized byte[][] top() throws IOException { public synchronized kelondroRow.Entry top() throws IOException {
// return row ontop of the stack // return row ontop of the stack
return top(0); return top(0);
} }
public synchronized byte[][] top(int dist) throws IOException { public synchronized kelondroRow.Entry top(int dist) throws IOException {
// return row ontop of the stack // return row ontop of the stack
// with dist == 0 this is the same function as with top() // with dist == 0 this is the same function as with top()
Node n = topNode(dist); Node n = topNode(dist);
if (n == null) return null; if (n == null) return null;
return n.getValueCells(); return row().newEntry(n.getValueRow());
} }
public synchronized byte[][] pot() throws IOException { public synchronized kelondroRow.Entry pot() throws IOException {
// return row on the bottom of the stack and remove record // return row on the bottom of the stack and remove record
return pot(0); return pot(0);
} }
public synchronized byte[][] pot(int dist) throws IOException { public synchronized kelondroRow.Entry pot(int dist) throws IOException {
// return row relative to the bottom of the stack and remove addressed element // return row relative to the bottom of the stack and remove addressed element
Node n = botNode(dist); Node n = botNode(dist);
if (n == null) return null; if (n == null) return null;
byte[][] ret = n.getValueCells(); kelondroRow.Entry ret = row().newEntry(n.getValueCells());
// remove node // remove node
unlinkNode(n); unlinkNode(n);
deleteNode(n.handle()); deleteNode(n.handle());
return ret; return ret;
} }
public synchronized byte[][] bot() throws IOException { public synchronized kelondroRow.Entry bot() throws IOException {
// return row on the bottom of the stack // return row on the bottom of the stack
return bot(0); return bot(0);
} }
public synchronized byte[][] bot(int dist) throws IOException { public synchronized kelondroRow.Entry bot(int dist) throws IOException {
// return row on bottom of the stack // return row on bottom of the stack
// with dist == 0 this is the same function as with bot() // with dist == 0 this is the same function as with bot()
Node n = botNode(dist); Node n = botNode(dist);
if (n == null) return null; if (n == null) return null;
return n.getValueCells(); return row().newEntry(n.getValueRow());
} }
public synchronized ArrayList botList(int dist) throws IOException { public synchronized ArrayList botList(int dist) throws IOException {
@ -235,59 +238,51 @@ public final class kelondroStack extends kelondroRecords {
private void unlinkNode(Node n) throws IOException { private void unlinkNode(Node n) throws IOException {
// join chaines over node // join chaines over node
Handle l = n.getOHHandle(left); Handle l = n.getOHHandle(left);
Handle r = n.getOHHandle(right); Handle r = n.getOHHandle(right);
// look left // look left
if (l == null) { if (l == null) {
// reached the root on left side // reached the root on left side
setHandle(root, r); setHandle(root, r);
} else { } else {
// un-link the previous record // un-link the previous record
Node k = getNode(l, null, 0); Node k = getNode(l, null, 0);
k.setOHHandle(left, k.getOHHandle(left)); k.setOHHandle(left, k.getOHHandle(left));
k.setOHHandle(right, r); k.setOHHandle(right, r);
k.commit(CP_NONE); k.commit(CP_NONE);
} }
// look right // look right
if (r == null) { if (r == null) {
// reached the root on right side // reached the root on right side
setHandle(toor, l); setHandle(toor, l);
} else { } else {
// un-link the following record // un-link the following record
Node k = getNode(r, null, 0); Node k = getNode(r, null, 0);
k.setOHHandle(left, l); k.setOHHandle(left, l);
k.setOHHandle(right, k.getOHHandle(right)); k.setOHHandle(right, k.getOHHandle(right));
k.commit(CP_NONE); k.commit(CP_NONE);
} }
} }
private Node topNode(int dist) throws IOException { private Node topNode(int dist) throws IOException {
// return node ontop of the stack // return node ontop of the stack
return queueNode(dist, toor, left); return queueNode(dist, toor, left);
} }
private Node botNode(int dist) throws IOException { private Node botNode(int dist) throws IOException {
// return node on bottom of the stack // return node on bottom of the stack
return queueNode(dist, root, right); return queueNode(dist, root, right);
} }
private Node queueNode(int dist, int side, int dir) throws IOException { private Node queueNode(int dist, int side, int dir) throws IOException {
// with dist == 0 this is the same function as with getNode(getHandle(side), null, 0) // with dist == 0 this is the same function as with
Handle h = getHandle(side); // getNode(getHandle(side), null, 0)
if (h == null) return null; Handle h = getHandle(side);
if (dist >= size()) return null; // that would exceed the stack if (h == null) return null;
while (dist-- > 0) h = getNode(h).getOHHandle(dir); // track through elements if (dist >= size()) return null; // that would exceed the stack
return getNode(h); while (dist-- > 0) h = getNode(h).getOHHandle(dir); // track through elements
} return getNode(h);
/*
public synchronized byte[][] seekPop(byte[] key, long maxdepth) throws IOException {
}
public synchronized byte[][] seekPot(byte[] key, long maxdepth) throws IOException {
} }
*/
public Iterator iterator() { public Iterator iterator() {
// iterates the elements in an ordered way. returns Node - type Objects // iterates the elements in an ordered way. returns Node - type Objects
@ -302,7 +297,7 @@ public final class kelondroStack extends kelondroRecords {
String s; String s;
StringTokenizer st; StringTokenizer st;
int recs = 0; int recs = 0;
byte[][] buffer = new byte[columns()][]; kelondroRow.Entry buffer = row().newEntry();
int c; int c;
int line = 0; int line = 0;
while ((s = f.readLine()) != null) { while ((s = f.readLine()) != null) {
@ -313,7 +308,7 @@ public final class kelondroStack extends kelondroRecords {
// buffer the entry // buffer the entry
c = 0; c = 0;
while ((c < columns()) && (st.hasMoreTokens())) { while ((c < columns()) && (st.hasMoreTokens())) {
buffer[c++] = st.nextToken().trim().getBytes(); buffer.setCol(c++, st.nextToken().trim().getBytes());
} }
if ((st.hasMoreTokens()) || (c != columns())) { if ((st.hasMoreTokens()) || (c != columns())) {
System.err.println("inapropriate number of entries in line " + line); System.err.println("inapropriate number of entries in line " + line);
@ -331,28 +326,34 @@ public final class kelondroStack extends kelondroRecords {
} }
public String hp(Handle h) { public String hp(Handle h) {
if (h == null) return "NULL"; else return h.toString(); if (h == null)
return "NULL";
else
return h.toString();
} }
public void print() throws IOException { public void print() throws IOException {
super.print(false); super.print(false);
Node n; Node n;
try { try {
Iterator it = iterator(); Iterator it = iterator();
while (it.hasNext()) { while (it.hasNext()) {
n = (Node) it.next(); n = (Node) it.next();
//n = getNode(h, null, 0); // n = getNode(h, null, 0);
System.out.println("> NODE " + hp(n.handle()) + System.out.println("> NODE " + hp(n.handle()) + "; left "
"; left " + hp(n.getOHHandle(left)) + ", right " + hp(n.getOHHandle(right))); + hp(n.getOHHandle(left)) + ", right "
System.out.print(" KEY:'" + (new String(n.getValueCells()[0])).trim() + "'"); + hp(n.getOHHandle(right)));
for (int j = 1; j < columns(); j++) System.out.print(" KEY:'"
System.out.print(", V[" + j + "]:'" + (new String(n.getValueCells()[j])).trim() + "'"); + (new String(n.getValueCells()[0])).trim() + "'");
System.out.println(); for (int j = 1; j < columns(); j++)
} System.out.print(", V[" + j + "]:'"
System.out.println(); + (new String(n.getValueCells()[j])).trim() + "'");
} catch (IOException e) { System.out.println();
System.out.println("File error: " + e.getMessage()); }
} System.out.println();
} catch (IOException e) {
System.out.println("File error: " + e.getMessage());
}
} }
private static void cmd(String[] args) { private static void cmd(String[] args) {
@ -372,8 +373,8 @@ public final class kelondroStack extends kelondroRecords {
ret = null; ret = null;
} else if (args[0].equals("-g")) { } else if (args[0].equals("-g")) {
fm = new kelondroStack(new File(args[1]), 0x100000); fm = new kelondroStack(new File(args[1]), 0x100000);
byte[][] ret2 = fm.pop(); kelondroRow.Entry ret2 = fm.pop();
ret = ((ret2 == null) ? null : ret2[1]); ret = ((ret2 == null) ? null : ret2.getColBytes(1));
fm.close(); fm.close();
} }
fm.close(); fm.close();
@ -404,8 +405,8 @@ public final class kelondroStack extends kelondroRecords {
} }
} else if (args[0].equals("-g")) { } else if (args[0].equals("-g")) {
kelondroStack fm = new kelondroStack(new File(args[2]), 0x100000); kelondroStack fm = new kelondroStack(new File(args[2]), 0x100000);
byte[][] ret2 = fm.pop(Integer.parseInt(args[1])); kelondroRow.Entry ret2 = fm.pop(Integer.parseInt(args[1]));
ret = ((ret2 == null) ? null : ret2[1]); ret = ((ret2 == null) ? null : ret2.getColBytes(1));
fm.close(); fm.close();
} }
} else if (args.length == 4) { } else if (args.length == 4) {
@ -420,7 +421,7 @@ public final class kelondroStack extends kelondroRecords {
fm.close(); fm.close();
} else if (args[0].equals("-p")) { } else if (args[0].equals("-p")) {
kelondroStack fm = new kelondroStack(new File(args[3]), 0x100000); kelondroStack fm = new kelondroStack(new File(args[3]), 0x100000);
fm.push(new byte[][] {args[1].getBytes(), args[2].getBytes()}); fm.push(fm.row().newEntry(new byte[][] {args[1].getBytes(), args[2].getBytes()}));
fm.close(); fm.close();
} }
} }

8
source/de/anomic/plasma/plasmaCrawlBalancer.java
Unescape View File

@ -108,7 +108,7 @@ public class plasmaCrawlBalancer {
while (i.hasNext()) { while (i.hasNext()) {
entry = (Map.Entry) i.next(); entry = (Map.Entry) i.next();
list = (ArrayList) entry.getValue(); list = (ArrayList) entry.getValue();
stack.push(new byte[][]{(byte[]) list.remove(0)}); stack.push(stack.row().newEntry(new byte[][]{(byte[]) list.remove(0)}));
if (list.size() == 0) i.remove(); if (list.size() == 0) i.remove();
} }
} }
@ -141,10 +141,10 @@ public class plasmaCrawlBalancer {
// returns a pair of domain/hash from the stack // returns a pair of domain/hash from the stack
// if the domain is unknown, a null/hash is returned // if the domain is unknown, a null/hash is returned
if (stack.size() > 0) { if (stack.size() > 0) {
return new Object[]{null, stack.pop()[0]}; return new Object[]{null, stack.pop().getColBytes(0)};
} else if (domainStacks.size() > 0) { } else if (domainStacks.size() > 0) {
flushOnce(); flushOnce();
return new Object[]{null, stack.pop()[0]}; return new Object[]{null, stack.pop().getColBytes(0)};
} else { } else {
return null; return null;
} }
@ -152,7 +152,7 @@ public class plasmaCrawlBalancer {
public synchronized byte[] top(int dist) throws IOException { public synchronized byte[] top(int dist) throws IOException {
flushAll(); flushAll();
return stack.top(dist)[0]; return stack.top(dist).getColBytes(0);
} }
public void clear() throws IOException { public void clear() throws IOException {

10
source/de/anomic/plasma/plasmaCrawlNURL.java
Unescape View File

@ -314,9 +314,9 @@ public class plasmaCrawlNURL extends indexURL {
case STACK_TYPE_LIMIT: limitStack.add(domain, hash.getBytes()); break; case STACK_TYPE_LIMIT: limitStack.add(domain, hash.getBytes()); break;
case STACK_TYPE_OVERHANG: overhangStack.add(domain, hash.getBytes()); break; case STACK_TYPE_OVERHANG: overhangStack.add(domain, hash.getBytes()); break;
case STACK_TYPE_REMOTE: remoteStack.add(domain, hash.getBytes()); break; case STACK_TYPE_REMOTE: remoteStack.add(domain, hash.getBytes()); break;
case STACK_TYPE_IMAGE: imageStack.push(new byte[][] {hash.getBytes()}); break; case STACK_TYPE_IMAGE: imageStack.push(imageStack.row().newEntry(new byte[][] {hash.getBytes()})); break;
case STACK_TYPE_MOVIE: movieStack.push(new byte[][] {hash.getBytes()}); break; case STACK_TYPE_MOVIE: movieStack.push(movieStack.row().newEntry(new byte[][] {hash.getBytes()})); break;
case STACK_TYPE_MUSIC: musicStack.push(new byte[][] {hash.getBytes()}); break; case STACK_TYPE_MUSIC: musicStack.push(musicStack.row().newEntry(new byte[][] {hash.getBytes()})); break;
default: break; default: break;
} }
stackIndex.add(hash); stackIndex.add(hash);
@ -389,7 +389,7 @@ public class plasmaCrawlNURL extends indexURL {
private Entry pop(kelondroStack stack) throws IOException { private Entry pop(kelondroStack stack) throws IOException {
// this is a filo - pop // this is a filo - pop
if (stack.size() > 0) { if (stack.size() > 0) {
Entry e = new Entry(new String(stack.pop()[0])); Entry e = new Entry(new String(stack.pop().getColBytes(0)));
stackIndex.remove(e.hash); stackIndex.remove(e.hash);
return e; return e;
} else { } else {
@ -414,7 +414,7 @@ public class plasmaCrawlNURL extends indexURL {
ArrayList list = new ArrayList(count); ArrayList list = new ArrayList(count);
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
try { try {
byte[] hash = stack.top(i)[0]; byte[] hash = stack.top(i).getColBytes(0);
list.add(new Entry(new String(hash))); list.add(new Entry(new String(hash)));
} catch (IOException e) { } catch (IOException e) {
continue; continue;

28
source/de/anomic/plasma/plasmaSwitchboardQueue.java
Unescape View File

@ -50,6 +50,7 @@ import de.anomic.index.indexURL;
import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroStack; import de.anomic.kelondro.kelondroStack;
import de.anomic.kelondro.kelondroRow;
import de.anomic.server.logging.serverLog; import de.anomic.server.logging.serverLog;
import de.anomic.server.serverDate; import de.anomic.server.serverDate;
import de.anomic.yacy.yacySeedDB; import de.anomic.yacy.yacySeedDB;
@ -118,7 +119,7 @@ public class plasmaSwitchboardQueue {
} }
public void push(Entry entry) throws IOException { public void push(Entry entry) throws IOException {
sbQueueStack.push(new byte[][]{ sbQueueStack.push(sbQueueStack.row().newEntry(new byte[][]{
entry.url.toString().getBytes(), entry.url.toString().getBytes(),
(entry.referrerHash == null) ? indexURL.dummyHash.getBytes() : entry.referrerHash.getBytes(), (entry.referrerHash == null) ? indexURL.dummyHash.getBytes() : entry.referrerHash.getBytes(),
kelondroBase64Order.enhancedCoder.encodeLong((entry.ifModifiedSince == null) ? 0 : entry.ifModifiedSince.getTime(), 11).getBytes(), kelondroBase64Order.enhancedCoder.encodeLong((entry.ifModifiedSince == null) ? 0 : entry.ifModifiedSince.getTime(), 11).getBytes(),
@ -127,12 +128,12 @@ public class plasmaSwitchboardQueue {
kelondroBase64Order.enhancedCoder.encodeLong((long) entry.depth, indexURL.urlCrawlDepthLength).getBytes(), kelondroBase64Order.enhancedCoder.encodeLong((long) entry.depth, indexURL.urlCrawlDepthLength).getBytes(),
(entry.profileHandle == null) ? indexURL.dummyHash.getBytes() : entry.profileHandle.getBytes(), (entry.profileHandle == null) ? indexURL.dummyHash.getBytes() : entry.profileHandle.getBytes(),
(entry.anchorName == null) ? "-".getBytes() : entry.anchorName.getBytes() (entry.anchorName == null) ? "-".getBytes() : entry.anchorName.getBytes()
}); }));
} }
public Entry pop() throws IOException { public Entry pop() throws IOException {
if (sbQueueStack.size() == 0) return null; if (sbQueueStack.size() == 0) return null;
byte[][] b = sbQueueStack.pot(); kelondroRow.Entry b = sbQueueStack.pot();
if (b == null) return null; if (b == null) return null;
return new Entry(b); return new Entry(b);
} }
@ -223,6 +224,27 @@ public class plasmaSwitchboardQueue {
this.referrerURL = null; this.referrerURL = null;
} }
public Entry(kelondroRow.Entry row) throws IOException {
long ims = row.getColLongB64E(2);
byte flags = row.getColByte(3);
try {
this.url = new URL(row.getColString(0, "UTF-8"));
} catch (MalformedURLException e) {
this.url = null;
}
this.referrerHash = row.getColString(1, "UTF-8");
this.ifModifiedSince = (ims == 0) ? null : new Date(ims);
this.flags = ((flags & 1) == 1) ? (byte) 1 : (byte) 0;
this.initiator = row.getColString(4, "UTF-8");
this.depth = (int) row.getColLongB64E(5);
this.profileHandle = row.getColString(6, "UTF-8");
this.anchorName = row.getColString(7, "UTF-8");
this.profileEntry = null;
this.responseHeader = null;
this.referrerURL = null;
}
public Entry(byte[][] row) throws IOException { public Entry(byte[][] row) throws IOException {
long ims = (row[2] == null) ? 0 : kelondroBase64Order.enhancedCoder.decodeLong(new String(row[2], "UTF-8")); long ims = (row[2] == null) ? 0 : kelondroBase64Order.enhancedCoder.decodeLong(new String(row[2], "UTF-8"));
byte flags = (row[3] == null) ? 0 : row[3][0]; byte flags = (row[3] == null) ? 0 : row[3][0];

25
source/de/anomic/yacy/yacyNewsQueue.java
Unescape View File

@ -48,6 +48,7 @@ import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.Date; import java.util.Date;
import de.anomic.kelondro.kelondroRow;
import de.anomic.kelondro.kelondroStack; import de.anomic.kelondro.kelondroStack;
import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroException;
@ -140,26 +141,14 @@ public class yacyNewsQueue {
return null; return null;
} }
private yacyNewsRecord b2r(kelondroRow.Entry b) throws IOException {
/*
public synchronized void incDistributedCounter(yacyNewsRecord entry) throws IOException {
// this works only if the entry element lies ontop of the stack
yacyNewsRecord topEntry = top();
if (!(topEntry.id().equals(entry.id()))) throw new IllegalArgumentException("entry is not ontop of the stack");
pop();
entry.incDistribution();
push(entry);
}
*/
private yacyNewsRecord b2r(byte[][] b) throws IOException {
if (b == null) return null; if (b == null) return null;
String id = new String(b[0]); String id = b.getColString(0, null);
//Date touched = yacyCore.parseUniversalDate(new String(b[1])); //Date touched = yacyCore.parseUniversalDate(new String(b[1]));
return newsDB.get(id); return newsDB.get(id);
} }
private byte[][] r2b(yacyNewsRecord r, boolean updateDB) throws IOException { private kelondroRow.Entry r2b(yacyNewsRecord r, boolean updateDB) throws IOException {
if (r == null) return null; if (r == null) return null;
if (updateDB) { if (updateDB) {
newsDB.put(r); newsDB.put(r);
@ -167,9 +156,9 @@ public class yacyNewsQueue {
yacyNewsRecord r1 = newsDB.get(r.id()); yacyNewsRecord r1 = newsDB.get(r.id());
if (r1 == null) newsDB.put(r); if (r1 == null) newsDB.put(r);
} }
byte[][] b = new byte[2][]; kelondroRow.Entry b = queueStack.row().newEntry(new byte[][]{
b[0] = r.id().getBytes(); r.id().getBytes(),
b[1] = yacyCore.universalDateShortString(new Date()).getBytes(); yacyCore.universalDateShortString(new Date()).getBytes()});
return b; return b;
} }

Write Preview
Loading…
Cancel
Save