boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

modified kelondroDyn to work better with new object caches

Browse Source 
(removed own single object cache)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2077 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent 26e3216bcc
commit 55c5b41bd0
3 changed files with 228 additions and 232 deletions
Show Stats Download Patch File Download Diff File

3
source/de/anomic/http/httpdProxyHandler.java
Unescape View File

@ -211,9 +211,6 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
isTransparentProxy = Boolean.valueOf(switchboard.getConfig("isTransparentProxy","false")).booleanValue();
// doing httpc init
httpc.useYacyReferer = sb.getConfig("useYacyReferer", "true").equals("true");
// // load remote proxy data
// remoteProxyHost = switchboard.getConfig("remoteProxyHost","");
// try {

119
source/de/anomic/kelondro/kelondroDyn.java
Unescape View File

@ -57,13 +57,11 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Iterator;
import de.anomic.server.serverByteBuffer;
public class kelondroDyn extends kelondroTree {
private static final int counterlen = 8;
private byte[] segmentCacheKey, segmentCacheContent;
private int keylen;
private int reclen;
private int segmentCount;
@ -73,15 +71,14 @@ public class kelondroDyn extends kelondroTree {
this(file, buffersize, key, nodesize, fillChar, new kelondroNaturalOrder(true), exitOnFail);
}
public kelondroDyn(File file, long buffersize /*bytes*/, int key, int nodesize, char fillChar, kelondroOrder objectOrder, boolean exitOnFail) {
public kelondroDyn(File file, long buffersize /* bytes */, int key,
int nodesize, char fillChar, kelondroOrder objectOrder,
boolean exitOnFail) {
// creates a new dynamic tree
super(file, buffersize, kelondroTree.defaultObjectCachePercent, new int[] {key + counterlen, nodesize}, objectOrder, 1, 8, exitOnFail);
super(file, buffersize, kelondroTree.defaultObjectCachePercent, new int[] { key + counterlen, nodesize }, objectOrder, 1, 8, exitOnFail);
this.keylen = columnSize(0) - counterlen;
this.reclen = columnSize(1);
this.fillChar = fillChar;
this.segmentCacheKey = null;
this.segmentCacheContent = null;
// init counter: write into text field
this.segmentCount = 0;
writeSegmentCount();
}
@ -92,12 +89,7 @@ public class kelondroDyn extends kelondroTree {
this.keylen = columnSize(0) - counterlen;
this.reclen = columnSize(1);
this.fillChar = fillChar;
this.segmentCacheKey = null;
this.segmentCacheContent = null;
this.segmentCount = 0;
//Iterator i = keys(true); while (i.hasNext()) segmentCount++;
//writeSegmentCount();
//readSegmentCount();
}
private void writeSegmentCount() {
@ -108,17 +100,6 @@ public class kelondroDyn extends kelondroTree {
}
}
/*
private void readSegmentCount() {
try {
segmentCount = (int) serverCodings.enhancedCoder.decodeBase64Long(new String(getText(0)));
} catch (Exception e) {
segmentCount = 0;
writeSegmentCount();
}
}
*/
public synchronized int sizeDyn() {
//this.segmentCount = 0;
//Iterator i = keys(true); while (i.hasNext()) segmentCount++;
@ -150,21 +131,26 @@ public class kelondroDyn extends kelondroTree {
// the iterator iterates all keys, which are byte[] objects
Iterator ri;
String nextKey;
public dynKeyIterator(Iterator iter) {
ri = iter;
nextKey = n();
}
public boolean hasNext() {
return nextKey != null;
}
public Object next() {
String result = nextKey;
nextKey = n();
return origKey(result.getBytes());
}
public void remove() {
throw new UnsupportedOperationException("no remove in RawKeyIterator");
}
private String n() {
byte[] g;
String k;
@ -201,30 +187,15 @@ public class kelondroDyn extends kelondroTree {
private byte[] getValueCached(byte[] key) throws IOException {
if ((segmentCacheKey != null) && (serverByteBuffer.equals(key, segmentCacheKey))) {
// use cache
//System.out.println("cache hit: " + super.filename + "/" + new String(key));
return segmentCacheContent;
}
// read from db
final byte[][] r = get(key);
if (r == null) return null;
// update cache
segmentCacheKey = key;
segmentCacheContent = r[1];
byte[][] result = get(key);
if (result == null) return null;
// return result
return r[1];
return result[1];
}
private synchronized void setValueCached(byte[] key, byte[] value) throws IOException {
// update cache
segmentCacheKey = key;
segmentCacheContent = value;
// update storage
put(key, value);
}
@ -256,7 +227,8 @@ public class kelondroDyn extends kelondroTree {
buf = buff;
buff = null;
}
//System.out.println("read: buf.length="+buf.length+",recpos="+recpos+",len="+len);
// System.out.println("read:
// buf.length="+buf.length+",recpos="+recpos+",len="+len);
if (recpos + len <= reclen) {
segment1 = new byte[len];
System.arraycopy(buf, recpos, segment1, 0, len);
@ -266,12 +238,16 @@ public class kelondroDyn extends kelondroTree {
}
}
// if this is all, return
if (recpos + len <= reclen) return segment1;
if (recpos + len <= reclen)
return segment1;
// read from several records
// we combine recursively all participating records
// we have two segments: the one in the starting record, and the remaining
// segment 1 in record <reccnt> : start = recpos, length = reclen - recpos
// segment 2 in record <reccnt>+1: start = 0, length = len - reclen + recpos
// we have two segments: the one in the starting record, and the
// remaining
// segment 1 in record <reccnt> : start = recpos, length = reclen -
// recpos
// segment 2 in record <reccnt>+1: start = 0, length = len - reclen +
// recpos
// recursively step further
byte[] segment2 = getDyn(key, pos + segment1.length, len - segment1.length);
if (segment2 == null) return null;
@ -305,7 +281,8 @@ public class kelondroDyn extends kelondroTree {
buf = buff;
buff = null;
}
//System.out.println("write: b.length="+b.length+",off="+off+",len="+(reclen-recpos));
// System.out.println("write:
// b.length="+b.length+",off="+off+",len="+(reclen-recpos));
if (len < (reclen - recpos))
System.arraycopy(b, off, buf, recpos, len);
else
@ -324,8 +301,6 @@ public class kelondroDyn extends kelondroTree {
int recpos = 0;
byte[] k;
while (super.get(k = dynKey(key, recpos)) != null) {
segmentCacheKey = null;
segmentCacheContent = null;
super.remove(k);
recpos++;
}
@ -345,6 +320,7 @@ public class kelondroDyn extends kelondroTree {
public class RARecord extends kelondroAbstractRA implements kelondroRA {
int seekpos = 0;
String filekey;
public RARecord(String filekey) {
@ -353,8 +329,6 @@ public class kelondroDyn extends kelondroTree {
public int read() throws IOException {
return getDyn(filekey, seekpos++);
//byte[] b = getDyn(filekey, seekpos++, 1);
//return (b == null) ? -1 : b[0] & 0xFF;
}
public void write(int i) throws IOException {
@ -365,7 +339,8 @@ public class kelondroDyn extends kelondroTree {
public int read(byte[] b, int off, int len) throws IOException {
byte[] buf = getDyn(filekey, seekpos, len);
if (buf == null) return 0;
if (buf == null)
return 0;
System.arraycopy(buf, 0, b, off, len);
seekpos += len;
return len;
@ -404,8 +379,16 @@ public class kelondroDyn extends kelondroTree {
fis.close();
kra.writeArray(result);
} finally {
if (fis != null) try{fis.close();}catch(Exception e){}
if (kra != null) try{kra.close();}catch(Exception e){}
if (fis != null)
try {
fis.close();
} catch (Exception e) {
}
if (kra != null)
try {
kra.close();
} catch (Exception e) {
}
}
}
@ -419,8 +402,16 @@ public class kelondroDyn extends kelondroTree {
fos = new FileOutputStream(f);
fos.write(result);
} finally {
if (fos != null) try{fos.close();}catch(Exception e){}
if (kra != null) try{kra.close();}catch(Exception e){}
if (fos != null)
try {
fos.close();
} catch (Exception e) {
}
if (kra != null)
try {
kra.close();
} catch (Exception e) {
}
}
}
@ -435,10 +426,12 @@ public class kelondroDyn extends kelondroTree {
} else if (args.length == 1) {
// open a db and list keys
try {
kelondroDyn kd = new kelondroDyn(new File(args[0]), 0x100000, '_');
kelondroDyn kd = new kelondroDyn(new File(args[0]), 0x100000,
'_');
System.out.println(kd.size() + " elements in DB");
Iterator i = kd.dynKeys(true, false);
while (i.hasNext()) System.out.println((String) i.next());
while (i.hasNext())
System.out.println((String) i.next());
kd.close();
} catch (IOException e) {
e.printStackTrace();
@ -451,8 +444,14 @@ public class kelondroDyn extends kelondroTree {
File f = new File(args[3]);
kelondroDyn kd;
try {
if (db.exists()) kd = new kelondroDyn(db, 0x100000, '_'); else kd = new kelondroDyn(db, 0x100000, 80, 200, '_', true);
if (writeFile) kd.readFile(key, f); else kd.writeFile(key, f);
if (db.exists())
kd = new kelondroDyn(db, 0x100000, '_');
else
kd = new kelondroDyn(db, 0x100000, 80, 200, '_', true);
if (writeFile)
kd.readFile(key, f);
else
kd.writeFile(key, f);
} catch (IOException e) {
System.out.println("ERROR: " + e.toString());
}

2
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -348,7 +348,7 @@ public final class yacyClient {
if ((result == null) || (result.size() == 0)) return -1;
final String resp = (String) result.get("response");
if (resp == null) { return -1; } else { return Integer.parseInt(resp); }
} catch (Exception e) {
} catch (IOException e) {
yacyCore.log.logSevere("yacyClient.queryUrlCount error asking peer '" + target.getName() + "':" + e.toString());
return -1;
}

Write Preview
Loading…
Cancel
Save