boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

modified kelondroDyn to work better with new object caches

Browse Source 
(removed own single object cache)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2077 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent 26e3216bcc
commit 55c5b41bd0
3 changed files with 228 additions and 232 deletions
Show Stats Download Patch File Download Diff File

3
source/de/anomic/http/httpdProxyHandler.java
Unescape View File

@ -211,9 +211,6 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
isTransparentProxy = Boolean.valueOf(switchboard.getConfig("isTransparentProxy","false")).booleanValue(); isTransparentProxy = Boolean.valueOf(switchboard.getConfig("isTransparentProxy","false")).booleanValue();
// doing httpc init
httpc.useYacyReferer = sb.getConfig("useYacyReferer", "true").equals("true");
// // load remote proxy data // // load remote proxy data
// remoteProxyHost = switchboard.getConfig("remoteProxyHost",""); // remoteProxyHost = switchboard.getConfig("remoteProxyHost","");
// try { // try {

455
source/de/anomic/kelondro/kelondroDyn.java
Unescape View File

@ -57,49 +57,41 @@ import java.io.FileInputStream;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.util.Iterator; import java.util.Iterator;
import de.anomic.server.serverByteBuffer;
public class kelondroDyn extends kelondroTree { public class kelondroDyn extends kelondroTree {
private static final int counterlen = 8; private static final int counterlen = 8;
private byte[] segmentCacheKey, segmentCacheContent;
private int keylen; private int keylen;
private int reclen; private int reclen;
private int segmentCount; private int segmentCount;
private char fillChar; private char fillChar;
public kelondroDyn(File file, long buffersize /*bytes*/, int key, int nodesize, char fillChar, boolean exitOnFail) { public kelondroDyn(File file, long buffersize /*bytes*/, int key, int nodesize, char fillChar, boolean exitOnFail) {
this(file, buffersize, key, nodesize, fillChar, new kelondroNaturalOrder(true), exitOnFail); this(file, buffersize, key, nodesize, fillChar, new kelondroNaturalOrder(true), exitOnFail);
} }
public kelondroDyn(File file, long buffersize /*bytes*/, int key, int nodesize, char fillChar, kelondroOrder objectOrder, boolean exitOnFail) { public kelondroDyn(File file, long buffersize /* bytes */, int key,
// creates a new dynamic tree int nodesize, char fillChar, kelondroOrder objectOrder,
super(file, buffersize, kelondroTree.defaultObjectCachePercent, new int[] {key + counterlen, nodesize}, objectOrder, 1, 8, exitOnFail); boolean exitOnFail) {
this.keylen = columnSize(0) - counterlen; // creates a new dynamic tree
this.reclen = columnSize(1); super(file, buffersize, kelondroTree.defaultObjectCachePercent, new int[] { key + counterlen, nodesize }, objectOrder, 1, 8, exitOnFail);
this.fillChar = fillChar; this.keylen = columnSize(0) - counterlen;
this.segmentCacheKey = null; this.reclen = columnSize(1);
this.segmentCacheContent = null; this.fillChar = fillChar;
// init counter: write into text field
this.segmentCount = 0; this.segmentCount = 0;
writeSegmentCount(); writeSegmentCount();
} }
public kelondroDyn(File file, long buffersize, char fillChar) throws IOException { public kelondroDyn(File file, long buffersize, char fillChar) throws IOException {
// this opens a file with an existing dynamic tree // this opens a file with an existing dynamic tree
super(file, buffersize, kelondroTree.defaultObjectCachePercent); super(file, buffersize, kelondroTree.defaultObjectCachePercent);
this.keylen = columnSize(0) - counterlen; this.keylen = columnSize(0) - counterlen;
this.reclen = columnSize(1); this.reclen = columnSize(1);
this.fillChar = fillChar; this.fillChar = fillChar;
this.segmentCacheKey = null;
this.segmentCacheContent = null;
this.segmentCount = 0; this.segmentCount = 0;
//Iterator i = keys(true); while (i.hasNext()) segmentCount++;
//writeSegmentCount();
//readSegmentCount();
} }
private void writeSegmentCount() { private void writeSegmentCount() {
try { try {
setText(0, kelondroBase64Order.enhancedCoder.encodeLong(segmentCount, 8).getBytes()); setText(0, kelondroBase64Order.enhancedCoder.encodeLong(segmentCount, 8).getBytes());
@ -108,17 +100,6 @@ public class kelondroDyn extends kelondroTree {
} }
} }
/*
private void readSegmentCount() {
try {
segmentCount = (int) serverCodings.enhancedCoder.decodeBase64Long(new String(getText(0)));
} catch (Exception e) {
segmentCount = 0;
writeSegmentCount();
}
}
*/
public synchronized int sizeDyn() { public synchronized int sizeDyn() {
//this.segmentCount = 0; //this.segmentCount = 0;
//Iterator i = keys(true); while (i.hasNext()) segmentCount++; //Iterator i = keys(true); while (i.hasNext()) segmentCount++;
@ -127,44 +108,49 @@ public class kelondroDyn extends kelondroTree {
} }
private static String counter(int c) { private static String counter(int c) {
String s = Integer.toHexString(c); String s = Integer.toHexString(c);
while (s.length() < counterlen) s = "0" + s; while (s.length() < counterlen) s = "0" + s;
return s; return s;
} }
private byte[] dynKey(String key, int record) { private byte[] dynKey(String key, int record) {
if (key.length() > keylen) throw new RuntimeException("key len (" + key.length() + ") out of limit (" + keylen + "): '" + key + "'"); if (key.length() > keylen) throw new RuntimeException("key len (" + key.length() + ") out of limit (" + keylen + "): '" + key + "'");
while (key.length() < keylen) key = key + fillChar; while (key.length() < keylen) key = key + fillChar;
key = key + counter(record); key = key + counter(record);
return key.getBytes(); return key.getBytes();
} }
private String origKey(byte[] rawKey) { private String origKey(byte[] rawKey) {
int n = keylen - 1; int n = keylen - 1;
if (n >= rawKey.length) n = rawKey.length - 1; if (n >= rawKey.length) n = rawKey.length - 1;
while ((n > 0) && (rawKey[n] == (byte) fillChar)) n--; while ((n > 0) && (rawKey[n] == (byte) fillChar)) n--;
return new String(rawKey, 0, n + 1); return new String(rawKey, 0, n + 1);
} }
public class dynKeyIterator implements Iterator { public class dynKeyIterator implements Iterator {
// the iterator iterates all keys, which are byte[] objects // the iterator iterates all keys, which are byte[] objects
Iterator ri; Iterator ri;
String nextKey; String nextKey;
public dynKeyIterator(Iterator iter) {
ri = iter; public dynKeyIterator(Iterator iter) {
nextKey = n(); ri = iter;
} nextKey = n();
public boolean hasNext() { }
return nextKey != null;
} public boolean hasNext() {
public Object next() { return nextKey != null;
String result = nextKey; }
nextKey = n();
return origKey(result.getBytes()); public Object next() {
} String result = nextKey;
public void remove() { nextKey = n();
throw new UnsupportedOperationException("no remove in RawKeyIterator"); return origKey(result.getBytes());
} }
public void remove() {
throw new UnsupportedOperationException("no remove in RawKeyIterator");
}
private String n() { private String n() {
byte[] g; byte[] g;
String k; String k;
@ -201,102 +187,92 @@ public class kelondroDyn extends kelondroTree {
private byte[] getValueCached(byte[] key) throws IOException { private byte[] getValueCached(byte[] key) throws IOException {
if ((segmentCacheKey != null) && (serverByteBuffer.equals(key, segmentCacheKey))) {
// use cache
//System.out.println("cache hit: " + super.filename + "/" + new String(key));
return segmentCacheContent;
}
// read from db // read from db
final byte[][] r = get(key); byte[][] result = get(key);
if (r == null) return null; if (result == null) return null;
// update cache
segmentCacheKey = key;
segmentCacheContent = r[1];
// return result // return result
return r[1]; return result[1];
} }
private synchronized void setValueCached(byte[] key, byte[] value) throws IOException { private synchronized void setValueCached(byte[] key, byte[] value) throws IOException {
// update storage
// update cache put(key, value);
segmentCacheKey = key;
segmentCacheContent = value;
// update storage
put(key, value);
} }
public synchronized int getDyn(String key, int pos) throws IOException { public synchronized int getDyn(String key, int pos) throws IOException {
int reccnt = pos / reclen; int reccnt = pos / reclen;
// read within a single record // read within a single record
byte[] buf = getValueCached(dynKey(key, reccnt)); byte[] buf = getValueCached(dynKey(key, reccnt));
if (buf == null) return -1; if (buf == null) return -1;
int recpos = pos % reclen; int recpos = pos % reclen;
if (buf.length <= recpos) return -1; if (buf.length <= recpos) return -1;
return buf[recpos] & 0xFF; return buf[recpos] & 0xFF;
} }
public synchronized byte[] getDyn(String key, int pos, int len) throws IOException { public synchronized byte[] getDyn(String key, int pos, int len) throws IOException {
int recpos = pos % reclen; int recpos = pos % reclen;
int reccnt = pos / reclen; int reccnt = pos / reclen;
byte[] segment1; byte[] segment1;
// read first within a single record // read first within a single record
if ((recpos == 0) && (reclen == len)) { if ((recpos == 0) && (reclen == len)) {
segment1 = getValueCached(dynKey(key, reccnt)); segment1 = getValueCached(dynKey(key, reccnt));
if (segment1 == null) return null; if (segment1 == null) return null;
} else { } else {
byte[] buf = getValueCached(dynKey(key, reccnt)); byte[] buf = getValueCached(dynKey(key, reccnt));
if (buf == null) return null; if (buf == null) return null;
if (buf.length < reclen) { if (buf.length < reclen) {
byte[] buff = new byte[reclen]; byte[] buff = new byte[reclen];
System.arraycopy(buf, 0, buff, 0, buf.length); System.arraycopy(buf, 0, buff, 0, buf.length);
buf = buff; buf = buff;
buff = null; buff = null;
} }
//System.out.println("read: buf.length="+buf.length+",recpos="+recpos+",len="+len); // System.out.println("read:
if (recpos + len <= reclen) { // buf.length="+buf.length+",recpos="+recpos+",len="+len);
segment1 = new byte[len]; if (recpos + len <= reclen) {
System.arraycopy(buf, recpos, segment1, 0, len); segment1 = new byte[len];
} else { System.arraycopy(buf, recpos, segment1, 0, len);
segment1 = new byte[reclen - recpos]; } else {
System.arraycopy(buf, recpos, segment1, 0, reclen - recpos); segment1 = new byte[reclen - recpos];
} System.arraycopy(buf, recpos, segment1, 0, reclen - recpos);
} }
// if this is all, return }
if (recpos + len <= reclen) return segment1; // if this is all, return
// read from several records if (recpos + len <= reclen)
// we combine recursively all participating records return segment1;
// we have two segments: the one in the starting record, and the remaining // read from several records
// segment 1 in record <reccnt> : start = recpos, length = reclen - recpos // we combine recursively all participating records
// segment 2 in record <reccnt>+1: start = 0, length = len - reclen + recpos // we have two segments: the one in the starting record, and the
// recursively step further // remaining
byte[] segment2 = getDyn(key, pos + segment1.length, len - segment1.length); // segment 1 in record <reccnt> : start = recpos, length = reclen -
if (segment2 == null) return null; // recpos
// now combine the two segments into the result // segment 2 in record <reccnt>+1: start = 0, length = len - reclen +
byte[] result = new byte[len]; // recpos
System.arraycopy(segment1, 0, result, 0, segment1.length); // recursively step further
System.arraycopy(segment2, 0, result, segment1.length, segment2.length); byte[] segment2 = getDyn(key, pos + segment1.length, len - segment1.length);
return result; if (segment2 == null) return null;
// now combine the two segments into the result
byte[] result = new byte[len];
System.arraycopy(segment1, 0, result, 0, segment1.length);
System.arraycopy(segment2, 0, result, segment1.length, segment2.length);
return result;
} }
public synchronized void putDyn(String key, int pos, byte[] b, int off, int len) throws IOException { public synchronized void putDyn(String key, int pos, byte[] b, int off, int len) throws IOException {
int recpos = pos % reclen; int recpos = pos % reclen;
int reccnt = pos / reclen; int reccnt = pos / reclen;
byte[] buf; byte[] buf;
// first write current record // first write current record
if ((recpos == 0) && (reclen == len)) { if ((recpos == 0) && (reclen == len)) {
if (off == 0) { if (off == 0) {
setValueCached(dynKey(key, reccnt), b); setValueCached(dynKey(key, reccnt), b);
} else { } else {
buf = new byte[len]; buf = new byte[len];
System.arraycopy(b, off, buf, 0, len); System.arraycopy(b, off, buf, 0, len);
setValueCached(dynKey(key, reccnt), b); setValueCached(dynKey(key, reccnt), b);
} }
} else { } else {
buf = getValueCached(dynKey(key, reccnt)); buf = getValueCached(dynKey(key, reccnt));
if (buf == null) { if (buf == null) {
buf = new byte[reclen]; buf = new byte[reclen];
} else if (buf.length < reclen) { } else if (buf.length < reclen) {
@ -305,87 +281,86 @@ public class kelondroDyn extends kelondroTree {
buf = buff; buf = buff;
buff = null; buff = null;
} }
//System.out.println("write: b.length="+b.length+",off="+off+",len="+(reclen-recpos)); // System.out.println("write:
if (len < (reclen - recpos)) // b.length="+b.length+",off="+off+",len="+(reclen-recpos));
System.arraycopy(b, off, buf, recpos, len); if (len < (reclen - recpos))
else System.arraycopy(b, off, buf, recpos, len);
System.arraycopy(b, off, buf, recpos, reclen - recpos); else
setValueCached(dynKey(key, reccnt), buf); System.arraycopy(b, off, buf, recpos, reclen - recpos);
} setValueCached(dynKey(key, reccnt), buf);
// if more records are necessary, write to them also recursively }
if (recpos + len > reclen) { // if more records are necessary, write to them also recursively
putDyn(key, pos + reclen - recpos, b, off + reclen - recpos, len - reclen + recpos); if (recpos + len > reclen) {
} putDyn(key, pos + reclen - recpos, b, off + reclen - recpos, len - reclen + recpos);
}
} }
public synchronized void remove(String key) throws IOException { public synchronized void remove(String key) throws IOException {
// remove value in cache and tree // remove value in cache and tree
if (key == null) return; if (key == null) return;
int recpos = 0; int recpos = 0;
byte[] k; byte[] k;
while (super.get(k = dynKey(key, recpos)) != null) { while (super.get(k = dynKey(key, recpos)) != null) {
segmentCacheKey = null;
segmentCacheContent = null;
super.remove(k); super.remove(k);
recpos++; recpos++;
} }
//segmentCount--; writeSegmentCount(); //segmentCount--; writeSegmentCount();
} }
public synchronized boolean existsDyn(String key) throws IOException { public synchronized boolean existsDyn(String key) throws IOException {
return (key != null) && (getValueCached(dynKey(key, 0)) != null); return (key != null) && (getValueCached(dynKey(key, 0)) != null);
} }
public synchronized kelondroRA getRA(String filekey) { public synchronized kelondroRA getRA(String filekey) {
// this returns always a RARecord, even if no existed bevore // this returns always a RARecord, even if no existed bevore
//return new kelondroBufferedRA(new RARecord(filekey), 512, 0); //return new kelondroBufferedRA(new RARecord(filekey), 512, 0);
return new RARecord(filekey); return new RARecord(filekey);
} }
public class RARecord extends kelondroAbstractRA implements kelondroRA { public class RARecord extends kelondroAbstractRA implements kelondroRA {
int seekpos = 0; int seekpos = 0;
String filekey;
String filekey;
public RARecord(String filekey) {
this.filekey = filekey; public RARecord(String filekey) {
} this.filekey = filekey;
}
public int read() throws IOException {
return getDyn(filekey, seekpos++); public int read() throws IOException {
//byte[] b = getDyn(filekey, seekpos++, 1); return getDyn(filekey, seekpos++);
//return (b == null) ? -1 : b[0] & 0xFF; }
}
public void write(int i) throws IOException {
public void write(int i) throws IOException { byte[] b = new byte[1];
byte[] b = new byte[1]; b[0] = (byte) i;
b[0] = (byte) i; putDyn(filekey, seekpos++, b, 0, 1);
putDyn(filekey, seekpos++, b, 0, 1); }
}
public int read(byte[] b, int off, int len) throws IOException {
public int read(byte[] b, int off, int len) throws IOException { byte[] buf = getDyn(filekey, seekpos, len);
byte[] buf = getDyn(filekey, seekpos, len); if (buf == null)
if (buf == null) return 0; return 0;
System.arraycopy(buf, 0, b, off, len); System.arraycopy(buf, 0, b, off, len);
seekpos += len; seekpos += len;
return len; return len;
} }
public void write(byte[] b, int off, int len) throws IOException { public void write(byte[] b, int off, int len) throws IOException {
putDyn(filekey, seekpos, b, off, len); putDyn(filekey, seekpos, b, off, len);
seekpos += len; seekpos += len;
} }
public void seek(long pos) throws IOException { public void seek(long pos) throws IOException {
seekpos = (int) pos; seekpos = (int) pos;
} }
public void close() throws IOException { public void close() throws IOException {
// no need to do anything here // no need to do anything here
} }
} }
public synchronized void writeFile(String key, File f) throws IOException { public synchronized void writeFile(String key, File f) throws IOException {
// reads a file from the FS and writes it into the database // reads a file from the FS and writes it into the database
kelondroRA kra = null; kelondroRA kra = null;
@ -404,11 +379,19 @@ public class kelondroDyn extends kelondroTree {
fis.close(); fis.close();
kra.writeArray(result); kra.writeArray(result);
} finally { } finally {
if (fis != null) try{fis.close();}catch(Exception e){} if (fis != null)
if (kra != null) try{kra.close();}catch(Exception e){} try {
fis.close();
} catch (Exception e) {
}
if (kra != null)
try {
kra.close();
} catch (Exception e) {
}
} }
} }
public synchronized void readFile(String key, File f) throws IOException { public synchronized void readFile(String key, File f) throws IOException {
// reads a file from the DB and writes it to the FS // reads a file from the DB and writes it to the FS
kelondroRA kra = null; kelondroRA kra = null;
@ -419,44 +402,60 @@ public class kelondroDyn extends kelondroTree {
fos = new FileOutputStream(f); fos = new FileOutputStream(f);
fos.write(result); fos.write(result);
} finally { } finally {
if (fos != null) try{fos.close();}catch(Exception e){} if (fos != null)
if (kra != null) try{kra.close();}catch(Exception e){} try {
fos.close();
} catch (Exception e) {
}
if (kra != null)
try {
kra.close();
} catch (Exception e) {
}
} }
} }
public static void main(String[] args) { public static void main(String[] args) {
// test app for DB functions // test app for DB functions
// reads/writes files to a database table // reads/writes files to a database table
// arguments: // arguments:
// {-f2db/-db2f} <db-name> <key> <filename> // {-f2db/-db2f} <db-name> <key> <filename>
if (args.length == 0) { if (args.length == 0) {
randomtest(20); randomtest(20);
} else if (args.length == 1) { } else if (args.length == 1) {
// open a db and list keys // open a db and list keys
try { try {
kelondroDyn kd = new kelondroDyn(new File(args[0]), 0x100000, '_'); kelondroDyn kd = new kelondroDyn(new File(args[0]), 0x100000,
System.out.println(kd.size() + " elements in DB"); '_');
Iterator i = kd.dynKeys(true, false); System.out.println(kd.size() + " elements in DB");
while (i.hasNext()) System.out.println((String) i.next()); Iterator i = kd.dynKeys(true, false);
kd.close(); while (i.hasNext())
} catch (IOException e) { System.out.println((String) i.next());
e.printStackTrace(); kd.close();
} } catch (IOException e) {
} e.printStackTrace();
if (args.length == 4) { }
boolean writeFile = (args[0].equals("-db2f")); }
File db = new File(args[1]); if (args.length == 4) {
String key = args[2]; boolean writeFile = (args[0].equals("-db2f"));
File f = new File(args[3]); File db = new File(args[1]);
kelondroDyn kd; String key = args[2];
try { File f = new File(args[3]);
if (db.exists()) kd = new kelondroDyn(db, 0x100000, '_'); else kd = new kelondroDyn(db, 0x100000, 80, 200, '_', true); kelondroDyn kd;
if (writeFile) kd.readFile(key, f); else kd.writeFile(key, f); try {
} catch (IOException e) { if (db.exists())
System.out.println("ERROR: " + e.toString()); kd = new kelondroDyn(db, 0x100000, '_');
} else
} kd = new kelondroDyn(db, 0x100000, 80, 200, '_', true);
if (writeFile)
kd.readFile(key, f);
else
kd.writeFile(key, f);
} catch (IOException e) {
System.out.println("ERROR: " + e.toString());
}
}
} }
public static void randomtest(int elements) { public static void randomtest(int elements) {

2
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -348,7 +348,7 @@ public final class yacyClient {
if ((result == null) || (result.size() == 0)) return -1; if ((result == null) || (result.size() == 0)) return -1;
final String resp = (String) result.get("response"); final String resp = (String) result.get("response");
if (resp == null) { return -1; } else { return Integer.parseInt(resp); } if (resp == null) { return -1; } else { return Integer.parseInt(resp); }
} catch (Exception e) { } catch (IOException e) {
yacyCore.log.logSevere("yacyClient.queryUrlCount error asking peer '" + target.getName() + "':" + e.toString()); yacyCore.log.logSevere("yacyClient.queryUrlCount error asking peer '" + target.getName() + "':" + e.toString());
return -1; return -1;
} }

Write Preview
Loading…
Cancel
Save