fixed caching+synchronization+brute-force-denial

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@67 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 20 years ago
parent 56409402f0
commit 2de90020ed

@ -45,7 +45,7 @@
# Contributions and changes to the program code must be marked as such. # Contributions and changes to the program code must be marked as such.
# define variables # define variables
version='0.367' version='0.368'
datestr=`date +%Y%m%d` datestr=`date +%Y%m%d`
#release='yacy_v'$version'_'$datestr #release='yacy_v'$version'_'$datestr
release='yacy_dev_v'$version'_'$datestr release='yacy_dev_v'$version'_'$datestr
@ -109,8 +109,6 @@ mv -f $source/$mainclass $source/$mainclass.orig
sed `echo 's/@REPL_DATE@/'$datestr'/'` $source/$mainclass.orig > $source/$mainclass.sed1 sed `echo 's/@REPL_DATE@/'$datestr'/'` $source/$mainclass.orig > $source/$mainclass.sed1
sed `echo 's/@REPL_VERSION@/'$version'/'` $source/$mainclass.sed1 > $source/$mainclass sed `echo 's/@REPL_VERSION@/'$version'/'` $source/$mainclass.sed1 > $source/$mainclass
rm $source/$mainclass.sed1 rm $source/$mainclass.sed1
#javac -classpath $classpath -sourcepath $source -d $classes -g:none $source/httpd.java
#javac -classpath $classpath -sourcepath $source -d $classes -g:none $source/$mainclass
javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/tools/*.java javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/tools/*.java
javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/net/*.java javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/net/*.java
javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/htmlFilter/*.java javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/htmlFilter/*.java
@ -125,7 +123,6 @@ javac -classpath $classpath -sourcepath $source -d $classes -g $source/$mainclas
mv -f $source/$mainclass.orig $source/$mainclass mv -f $source/$mainclass.orig $source/$mainclass
# compile server pages # compile server pages
#javac -classpath $classes -sourcepath htroot -d $classes -g htroot/*.java
javac -classpath $classes -sourcepath htroot -d htroot -g htroot/*.java javac -classpath $classes -sourcepath htroot -d htroot -g htroot/*.java
javac -classpath $classes -sourcepath htroot/yacy -d htroot/yacy -g htroot/yacy/*.java javac -classpath $classes -sourcepath htroot/yacy -d htroot/yacy -g htroot/yacy/*.java
javac -classpath $classes -sourcepath htroot/htdocsdefault -d htroot/htdocsdefault -g htroot/htdocsdefault/*.java javac -classpath $classes -sourcepath htroot/htdocsdefault -d htroot/htdocsdefault -g htroot/htdocsdefault/*.java

@ -225,10 +225,14 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
out.write(("\r\n").getBytes()); out.write(("\r\n").getBytes());
out.flush(); out.flush();
return; return;
} else if (!(adminAccountBase64MD5.equals(serverCodings.standardCoder.encodeMD5Hex(auth.trim().substring(6))))) { } else if (adminAccountBase64MD5.equals(serverCodings.standardCoder.encodeMD5Hex(auth.trim().substring(6)))) {
// remove brute-force flag
serverCore.bfHost.remove(conProp.getProperty("CLIENTIP"));
} else {
// a wrong authentication was given. Ask again // a wrong authentication was given. Ask again
serverLog.logInfo("HTTPD", "Wrong log-in for account 'admin' in http file handler for path '" + path + "' from host '" + conProp.getProperty("CLIENTIP", "unknown-IP") + "'"); serverLog.logInfo("HTTPD", "Wrong log-in for account 'admin' in http file handler for path '" + path + "' from host '" + conProp.getProperty("CLIENTIP", "unknown-IP") + "'");
try {Thread.currentThread().sleep(3000);} catch (InterruptedException e) {} // add a delay to make brute-force harder //try {Thread.currentThread().sleep(3000);} catch (InterruptedException e) {} // add a delay to make brute-force harder
serverCore.bfHost.put(conProp.getProperty("CLIENTIP"), "sleep");
out.write(("HTTP/1.1 401 log-in required\r\n").getBytes()); out.write(("HTTP/1.1 401 log-in required\r\n").getBytes());
out.write(("WWW-Authenticate: Basic realm=\"admin log-in\"\r\n").getBytes()); out.write(("WWW-Authenticate: Basic realm=\"admin log-in\"\r\n").getBytes());
out.write(("\r\n").getBytes()); out.write(("\r\n").getBytes());

@ -323,11 +323,10 @@ public class kelondroRecords {
Node n = (Node) cache.get(handle); Node n = (Node) cache.get(handle);
if (n == null) { if (n == null) {
n = new Node(handle, parentNode, referenceInParent); n = new Node(handle, parentNode, referenceInParent);
cache.put(handle, n); checkCacheSpace();
cacheScore.setScore(handle, (int) ((System.currentTimeMillis() - startup) / 1000));
checkCacheSpace();
return n; return n;
} else { } else {
//System.out.println("read from cache " + n.toString());
cacheScore.setScore(handle, (int) ((System.currentTimeMillis() - startup) / 1000)); cacheScore.setScore(handle, (int) ((System.currentTimeMillis() - startup) / 1000));
return n; return n;
} }
@ -336,7 +335,7 @@ public class kelondroRecords {
protected void deleteNode(Handle handle) throws IOException { protected void deleteNode(Handle handle) throws IOException {
if (cachesize != 0) { if (cachesize != 0) {
Node n = (Node) cache.get(handle); Node n = (Node) cache.get(handle);
if (n != null) { if (n != null) synchronized (cache) {
cacheScore.deleteScore(handle); cacheScore.deleteScore(handle);
cache.remove(handle); cache.remove(handle);
} }
@ -351,14 +350,18 @@ public class kelondroRecords {
// delete one entry // delete one entry
try { try {
Handle delkey = (Handle) cacheScore.getMinObject(); // error (see below) here Handle delkey = (Handle) cacheScore.getMinObject(); // error (see below) here
cacheScore.deleteScore(delkey); synchronized (cache) {
cache.remove(delkey); cacheScore.deleteScore(delkey);
cache.remove(delkey);
}
} catch (NoSuchElementException e) { } catch (NoSuchElementException e) {
System.out.println("strange kelondroRecords error: " + e.getMessage() + "; cachesize=" + cachesize + ", cache.size()=" + cache.size() + ", cacheScore.size()=" + cacheScore.size()); System.out.println("strange kelondroRecords error: " + e.getMessage() + "; cachesize=" + cachesize + ", cache.size()=" + cache.size() + ", cacheScore.size()=" + cacheScore.size());
// this is a strange error and could be caused by internal java problems // this is a strange error and could be caused by internal java problems
// we simply clear the cache // we simply clear the cache
this.cacheScore = new kelondroMScoreCluster(); synchronized (cache) {
this.cache = new HashMap(); this.cacheScore = new kelondroMScoreCluster();
this.cache = new HashMap();
}
} }
} }
} }
@ -441,11 +444,11 @@ public class kelondroRecords {
if (b.length != OHBYTEC) throw new IllegalArgumentException("setOHByte: wrong array size"); if (b.length != OHBYTEC) throw new IllegalArgumentException("setOHByte: wrong array size");
if (this.handle.index == NUL) throw new kelondroException(filename, "setOHByte: no handle assigned"); if (this.handle.index == NUL) throw new kelondroException(filename, "setOHByte: no handle assigned");
if (this.ohBytes == null) this.ohBytes = new byte[OHBYTEC]; if (this.ohBytes == null) this.ohBytes = new byte[OHBYTEC];
entryFile.seek(seekpos(this.handle)); entryFile.seek(seekpos(this.handle));
for (int j = 0; j < ohBytes.length; j++) { for (int j = 0; j < ohBytes.length; j++) {
ohBytes[j] = b[j]; ohBytes[j] = b[j];
entryFile.writeByte(b[j]); entryFile.writeByte(b[j]);
} }
updateNode(); updateNode();
} }
protected synchronized void setOHHandle(Handle[] i) throws IOException { protected synchronized void setOHHandle(Handle[] i) throws IOException {
@ -453,9 +456,9 @@ public class kelondroRecords {
if (i.length != OHHANDLEC) throw new IllegalArgumentException("setOHHandle: wrong array size"); if (i.length != OHHANDLEC) throw new IllegalArgumentException("setOHHandle: wrong array size");
if (this.handle.index == NUL) throw new kelondroException(filename, "setOHHandle: no handle assigned"); if (this.handle.index == NUL) throw new kelondroException(filename, "setOHHandle: no handle assigned");
if (this.ohHandle == null) this.ohHandle = new Handle[OHHANDLEC]; if (this.ohHandle == null) this.ohHandle = new Handle[OHHANDLEC];
entryFile.seek(seekpos(this.handle) + OHBYTEC); entryFile.seek(seekpos(this.handle) + OHBYTEC);
for (int j = 0; j < ohHandle.length; j++) { for (int j = 0; j < ohHandle.length; j++) {
ohHandle[j] = i[j]; ohHandle[j] = i[j];
if (i[j] == null) if (i[j] == null)
entryFile.writeInt(NUL); entryFile.writeInt(NUL);
else else
@ -469,8 +472,8 @@ public class kelondroRecords {
ohBytes = new byte[OHBYTEC]; ohBytes = new byte[OHBYTEC];
entryFile.seek(seekpos(this.handle)); entryFile.seek(seekpos(this.handle));
for (int j = 0; j < ohBytes.length; j++) { for (int j = 0; j < ohBytes.length; j++) {
ohBytes[j] = entryFile.readByte(); ohBytes[j] = entryFile.readByte();
} }
updateNode(); updateNode();
} }
return ohBytes; return ohBytes;
@ -555,20 +558,20 @@ public class kelondroRecords {
} }
} else if ((values.length > 1) && (values[1] == null)) { } else if ((values.length > 1) && (values[1] == null)) {
// only the key has been read; load the remaining // only the key has been read; load the remaining
long seek = seekpos(this.handle) + overhead + COLWIDTHS[0]; long seek = seekpos(this.handle) + overhead + COLWIDTHS[0];
for (int i = 1; i < COLWIDTHS.length; i++) { for (int i = 1; i < COLWIDTHS.length; i++) {
entryFile.seek(seek); entryFile.seek(seek);
values[i] = new byte[COLWIDTHS[i]]; values[i] = new byte[COLWIDTHS[i]];
entryFile.read(values[i], 0, values[i].length); entryFile.read(values[i], 0, values[i].length);
seek = seek + COLWIDTHS[i]; seek = seek + COLWIDTHS[i];
} }
updateNode(); updateNode();
return values; return values;
} else { } else {
return values; return values;
} }
} }
protected void save() throws IOException { protected synchronized void save() throws IOException {
// this is called when an entry was defined with values only and not by retrieving with an index // this is called when an entry was defined with values only and not by retrieving with an index
// if this happens, nothing of the internal array values have been written to the file // if this happens, nothing of the internal array values have been written to the file
// then writing to the file is done here // then writing to the file is done here
@ -585,9 +588,9 @@ public class kelondroRecords {
throw new kelondroException(filename, "no values to save"); throw new kelondroException(filename, "no values to save");
} }
entryFile.seek(seekpos(this.handle)); entryFile.seek(seekpos(this.handle));
if (ohBytes == null) {for (int i = 0; i < OHBYTEC; i++) entryFile.writeByte(0);} if (ohBytes == null) {for (int i = 0; i < OHBYTEC; i++) entryFile.writeByte(0);}
else {for (int i = 0; i < OHBYTEC; i++) entryFile.writeByte(ohBytes[i]);} else {for (int i = 0; i < OHBYTEC; i++) entryFile.writeByte(ohBytes[i]);}
if (ohHandle == null) {for (int i = 0; i < OHHANDLEC; i++) entryFile.writeInt(0);} if (ohHandle == null) {for (int i = 0; i < OHHANDLEC; i++) entryFile.writeInt(0);}
else {for (int i = 0; i < OHHANDLEC; i++) entryFile.writeInt(ohHandle[i].index);} else {for (int i = 0; i < OHHANDLEC; i++) entryFile.writeInt(ohHandle[i].index);}
long seek = seekpos(this.handle) + overhead; long seek = seekpos(this.handle) + overhead;
for (int i = 0; i < values.length; i++) { for (int i = 0; i < values.length; i++) {
@ -623,8 +626,12 @@ public class kelondroRecords {
private void updateNode() { private void updateNode() {
if (cachesize != 0) { if (cachesize != 0) {
if (!(cache.containsKey(handle))) checkCacheSpace(); if (!(cache.containsKey(handle))) checkCacheSpace();
cache.put(handle, this); synchronized (cache) {
cacheScore.setScore(handle, (int) ((System.currentTimeMillis() - startup) / 1000)); //System.out.println("updateNode " + this.toString());
cache.put(handle, this);
cacheScore.setScore(handle, (int) ((System.currentTimeMillis() - startup) / 1000));
//System.out.println("cache now: " + cache.toString());
}
} }
} }
} }
@ -844,7 +851,9 @@ public class kelondroRecords {
if (index > ((Handle) h).index) return 1; if (index > ((Handle) h).index) return 1;
return 0; return 0;
} }
public int hashCode() {
return this.index;
}
} }

@ -114,7 +114,8 @@ public class kelondroTree extends kelondroRecords implements Comparator {
} }
// Returns the value to which this map maps the specified key. // Returns the value to which this map maps the specified key.
public synchronized byte[][] get(byte[] key) throws IOException { public byte[][] get(byte[] key) throws IOException {
//System.out.println("kelondroTree.get " + new String(key) + " in " + filename);
Search search = new Search(key); Search search = new Search(key);
if (search.found()) { if (search.found()) {
return search.getMatcher().getValues(); return search.getMatcher().getValues();
@ -123,7 +124,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
} }
} }
public synchronized long[] getLong(byte[] key) throws IOException { public long[] getLong(byte[] key) throws IOException {
byte[][] row = get(key); byte[][] row = get(key);
long[] longs = new long[columns() - 1]; long[] longs = new long[columns() - 1];
if (row == null) { if (row == null) {
@ -181,6 +182,9 @@ public class kelondroTree extends kelondroRecords implements Comparator {
found = false; found = false;
int c; int c;
Handle[] handles; Handle[] handles;
HashMap visitedNodeKeys = new HashMap(); // to detect loops
String otherkey;
//System.out.println("Starting Compare Loop in Database " + filename); // debug
while (thisHandle != null) { while (thisHandle != null) {
try { try {
parentnode = thenode; parentnode = thenode;
@ -190,7 +194,23 @@ public class kelondroTree extends kelondroRecords implements Comparator {
found = false; found = false;
return; return;
} }
//System.out.print("Comparing key = '" + new String(key) + "' with '" + new String(thenode.node().getKey()) + "':"); // debug otherkey = new String(thenode.getKey());
if (visitedNodeKeys.containsKey(otherkey)) {
// we have loops in the database.
// to fix this, all affected nodes must be patched
thenode.setOHByte(new byte[] {1, 0});
thenode.setOHHandle(new Handle[] {null, null, null});
Iterator fix = visitedNodeKeys.entrySet().iterator();
Map.Entry entry;
while (fix.hasNext()) {
entry = (Map.Entry) fix.next();
thenode = (Node) entry.getValue();
thenode.setOHByte(new byte[] {1, 0});
thenode.setOHHandle(new Handle[] {null, null, null});
}
throw new kelondroException(filename, "database contains loops; the loop-nodes have been auto-fixed");
}
//System.out.print("Comparing key = '" + new String(key) + "' with '" + otherkey + "':"); // debug
c = compare(key, thenode.getKey()); c = compare(key, thenode.getKey());
//System.out.println(c); // debug //System.out.println(c); // debug
if (c == 0) { if (c == 0) {
@ -203,6 +223,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
child = 1; child = 1;
thisHandle = thenode.getOHHandle()[rightchild]; thisHandle = thenode.getOHHandle()[rightchild];
} }
visitedNodeKeys.put(otherkey, thenode);
} }
} }
// we reached a node where we must insert the new value // we reached a node where we must insert the new value
@ -238,7 +259,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
} }
} }
public synchronized boolean isChild(Node childn, Node parentn, int child) throws IOException { public boolean isChild(Node childn, Node parentn, int child) throws IOException {
if (childn == null) throw new IllegalArgumentException("isLeftChild: Node parameter is NULL"); if (childn == null) throw new IllegalArgumentException("isLeftChild: Node parameter is NULL");
Handle lc = parentn.getOHHandle()[child]; Handle lc = parentn.getOHHandle()[child];
if (lc == null) return false; if (lc == null) return false;
@ -356,7 +377,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
} }
} }
public synchronized long[] putLong(byte[] key, long[] newlongs) throws IOException { public long[] putLong(byte[] key, long[] newlongs) throws IOException {
byte[][] newrow = new byte[newlongs.length + 1][]; byte[][] newrow = new byte[newlongs.length + 1][];
newrow[0] = key; newrow[0] = key;
for (int i = 0; i < newlongs.length; i++) { for (int i = 0; i < newlongs.length; i++) {
@ -377,7 +398,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
} }
// Associates the specified value with the specified key in this map // Associates the specified value with the specified key in this map
public synchronized byte[][] put(byte[][] newrow) throws IOException { public byte[][] put(byte[][] newrow) throws IOException {
if (newrow.length != columns()) throw new IllegalArgumentException("put: wrong row length " + newrow.length + "; must be " + columns()); if (newrow.length != columns()) throw new IllegalArgumentException("put: wrong row length " + newrow.length + "; must be " + columns());
// first try to find the key element in the database // first try to find the key element in the database
Search searchResult = new Search(newrow[0]); Search searchResult = new Search(newrow[0]);
@ -625,7 +646,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
} }
// Associates the specified value with the specified key in this map // Associates the specified value with the specified key in this map
public synchronized byte[] put(byte[] key, byte[] value) throws IOException { public byte[] put(byte[] key, byte[] value) throws IOException {
byte[][] row = new byte[2][]; byte[][] row = new byte[2][];
row[0] = key; row[0] = key;
row[1] = value; row[1] = value;
@ -634,7 +655,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
} }
// Removes the mapping for this key from this map if present (optional operation). // Removes the mapping for this key from this map if present (optional operation).
public synchronized byte[][] remove(byte[] key) throws IOException { public byte[][] remove(byte[] key) throws IOException {
Search search = new Search(key); Search search = new Search(key);
if (search.found()) { if (search.found()) {
Node result = search.getMatcher(); Node result = search.getMatcher();
@ -646,11 +667,11 @@ public class kelondroTree extends kelondroRecords implements Comparator {
} }
} }
public synchronized void removeAll() throws IOException { public void removeAll() throws IOException {
while (size() > 0) remove(lastNode(), null); while (size() > 0) remove(lastNode(), null);
} }
public synchronized void remove(Node node, Node parentOfNode) throws IOException { public void remove(Node node, Node parentOfNode) throws IOException {
// there are three cases when removing a node // there are three cases when removing a node
// - the node is a leaf - it can be removed easily // - the node is a leaf - it can be removed easily
// - the node has one child - the child replaces the node // - the node has one child - the child replaces the node
@ -814,50 +835,6 @@ public class kelondroTree extends kelondroRecords implements Comparator {
throw new RuntimeException("error creating an iteration: " + e.getMessage()); throw new RuntimeException("error creating an iteration: " + e.getMessage());
} }
} }
/*
public synchronized keyIterator keys(boolean up, boolean rotating) throws IOException {
// iterates only the keys of the Nodes
// enumerated objects are of type byte[]
// iterates the elements in a sorted way.
return new keyIterator(new nodeIterator(up, rotating));
}
public synchronized keyIterator keys(boolean up, boolean rotating, byte[] firstKey) throws IOException {
Search s = new Search(firstKey);
if (s.found()) {
return new keyIterator(new nodeIterator(up, rotating, s.getMatcher()));
} else {
Node nn = s.getParent();
if (nn == null) {
return (keyIterator) (new HashSet()).iterator();
} else {
return new keyIterator(new nodeIterator(up, rotating, nn));
}
}
}
public class keyIterator implements Iterator {
// the iterator iterates all keys, which are byte[] objects
Iterator nodeIterator;
public keyIterator(nodeIterator nodeIterator) {
this.nodeIterator = nodeIterator;
}
public boolean hasNext() {
return nodeIterator.hasNext();
}
public Object next() {
try {
return ((Node) nodeIterator.next()).getKey();
} catch (IOException e) {
return null;
}
}
public void remove() {
nodeIterator.remove();
}
}
*/
public synchronized rowIterator rows(boolean up, boolean rotating) throws IOException { public synchronized rowIterator rows(boolean up, boolean rotating) throws IOException {
// iterates only the keys of the Nodes // iterates only the keys of the Nodes
@ -907,7 +884,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
} }
public synchronized int imp(File file, String separator) throws IOException { public int imp(File file, String separator) throws IOException {
// imports a value-separated file, returns number of records that have been read // imports a value-separated file, returns number of records that have been read
RandomAccessFile f = new RandomAccessFile(file,"r"); RandomAccessFile f = new RandomAccessFile(file,"r");
@ -1162,7 +1139,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
} }
public synchronized int compare(Object a, Object b) { public int compare(Object a, Object b) {
try { try {
if ((a instanceof byte[]) && (b instanceof byte[])) { if ((a instanceof byte[]) && (b instanceof byte[])) {
return compare((byte[]) a, (byte[]) b); return compare((byte[]) a, (byte[]) b);
@ -1208,8 +1185,8 @@ public class kelondroTree extends kelondroRecords implements Comparator {
public static void main(String[] args) { public static void main(String[] args) {
//cmd(args); //cmd(args);
bigtest(Integer.parseInt(args[0])); //bigtest(Integer.parseInt(args[0]));
//randomtest(Integer.parseInt(args[0])); randomtest(Integer.parseInt(args[0]));
//smalltest(); //smalltest();
} }
@ -1254,7 +1231,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
String s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".substring(0, elements); String s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".substring(0, elements);
String t, d; String t, d;
char c; char c;
kelondroTree tt; kelondroTree tt = null;
File testFile = new File("test.db"); File testFile = new File("test.db");
byte[] b; byte[] b;
try { try {
@ -1284,6 +1261,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
t = t + c; t = t + c;
System.out.println("removed " + new String(b)); System.out.println("removed " + new String(b));
} }
//tt.print();
if (countElements(tt) != tt.size()) { if (countElements(tt) != tt.size()) {
System.out.println("wrong size for "); System.out.println("wrong size for ");
tt.print(); tt.print();
@ -1313,6 +1291,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
try {tt.print();} catch (IOException ee) {}
System.out.println("TERMINATED"); System.out.println("TERMINATED");
} }
} }
@ -1321,7 +1300,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
File f = new File("test.db"); File f = new File("test.db");
if (f.exists()) f.delete(); if (f.exists()) f.delete();
try { try {
kelondroTree tt = new kelondroTree(f, 0, 4, 4); kelondroTree tt = new kelondroTree(f, 1000, 4, 4);
byte[] b; byte[] b;
b = testWord('b'); tt.put(b, b); b = testWord('b'); tt.put(b, b);
b = testWord('c'); tt.put(b, b); b = testWord('c'); tt.put(b, b);

@ -296,7 +296,9 @@ public class plasmaCrawlNURL extends plasmaURL {
urlHashCache.put(entry); urlHashCache.put(entry);
} catch (IOException e) { } catch (IOException e) {
System.out.println("INTERNAL ERROR AT plasmaNURL:url2hash:" + e.toString()); System.out.println("INTERNAL ERROR AT plasmaNURL:url2hash:" + e.toString());
} } catch (kelondroException e) {
serverLog.logError("PLASMA", "plasmaCrawlNURL.store failed: " + e.getMessage());
}
} }
public String hash() { public String hash() {

@ -199,15 +199,15 @@ public final class plasmaHTCache {
return header; return header;
} }
boolean idle() { public boolean idle() {
return (System.currentTimeMillis() > (idleDelay + lastAcc)); return (System.currentTimeMillis() > (idleDelay + lastAcc));
} }
boolean full() { public boolean full() {
return (cacheStack.size() > stackLimit); return (cacheStack.size() > stackLimit);
} }
boolean empty() { public boolean empty() {
return (cacheStack.size() == 0); return (cacheStack.size() == 0);
} }
@ -311,7 +311,7 @@ public final class plasmaHTCache {
try { try {
File f; File f;
int workoff; int workoff;
workoff = cacheStack.size() / 10; workoff = 1 + cacheStack.size() / 10;
// we want to work off always 10 % to prevent that we collaps // we want to work off always 10 % to prevent that we collaps
while ((workoff-- > 0) && (!(empty()))) { while ((workoff-- > 0) && (!(empty()))) {
process((Entry) cacheStack.removeFirst()); process((Entry) cacheStack.removeFirst());
@ -427,8 +427,8 @@ public final class plasmaHTCache {
} }
public static boolean isCGI(String urlString) { public static boolean isCGI(String urlString) {
return ((urlString.toLowerCase().indexOf("cgi") >= 0) || return ((urlString.toLowerCase().indexOf(".cgi") >= 0) ||
(urlString.toLowerCase().indexOf("exe") >= 0)); (urlString.toLowerCase().indexOf(".exe") >= 0));
} }
public Entry newEntry(Date initDate, int depth, URL url, public Entry newEntry(Date initDate, int depth, URL url,

@ -148,15 +148,13 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi
public HashMap outgoingCookies, incomingCookies; public HashMap outgoingCookies, incomingCookies;
public kelondroTables facilityDB; public kelondroTables facilityDB;
public plasmaParser parser; public plasmaParser parser;
public int serverJobs;
private serverSemaphore shutdownSync = new serverSemaphore(0); private serverSemaphore shutdownSync = new serverSemaphore(0);
private boolean terminate = false; private boolean terminate = false;
public plasmaSwitchboard(String rootPath, String initPath, String configPath) throws IOException { public plasmaSwitchboard(String rootPath, String initPath, String configPath) throws IOException {
super(rootPath, initPath, configPath); super(rootPath, initPath, configPath);
serverJobs = 0;
// set loglevel // set loglevel
int loglevel = Integer.parseInt(getConfig("plasmaLoglevel", "2")); int loglevel = Integer.parseInt(getConfig("plasmaLoglevel", "2"));
log = new serverLog("PLASMA", loglevel); log = new serverLog("PLASMA", loglevel);
@ -300,10 +298,6 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi
return (bytes / 1024) + "TByte"; return (bytes / 1024) + "TByte";
} }
public void handleBusyState(int jobs) {
this.serverJobs = jobs;
}
private void initProfiles() throws IOException { private void initProfiles() throws IOException {
if ((profiles.size() == 0) || if ((profiles.size() == 0) ||
(getConfig("defaultProxyProfile", "").length() == 0) || (getConfig("defaultProxyProfile", "").length() == 0) ||
@ -414,10 +408,10 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi
if (processStack.size() == 0) return false; // nothing to do if (processStack.size() == 0) return false; // nothing to do
// in case that the server is very busy we do not work off the queue too fast // in case that the server is very busy we do not work off the queue too fast
if (serverJobs > 10) try {Thread.currentThread().sleep(10 * serverJobs);} catch (InterruptedException e) {} if (!(cacheManager.idle())) try {Thread.currentThread().sleep(1000);} catch (InterruptedException e) {}
// do one processing step // do one processing step
log.logDebug("DEQUEUE: serverJobs=" + serverJobs + log.logDebug("DEQUEUE: cacheManager=" + ((cacheManager.idle()) ? "idle" : "busy") +
", processStack=" + processStack.size() + ", processStack=" + processStack.size() +
", localStackSize=" + noticeURL.localStackSize() + ", localStackSize=" + noticeURL.localStackSize() +
", remoteStackSize=" + noticeURL.remoteStackSize()); ", remoteStackSize=" + noticeURL.remoteStackSize());
@ -473,7 +467,7 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi
} }
// if the server is busy, we do crawling more slowly // if the server is busy, we do crawling more slowly
if (serverJobs > 3) try {Thread.currentThread().sleep(100 * serverJobs);} catch (InterruptedException e) {} if (!(cacheManager.idle())) try {Thread.currentThread().sleep(2000);} catch (InterruptedException e) {}
// do a local crawl (may start a global crawl) // do a local crawl (may start a global crawl)
plasmaCrawlNURL.entry nex = noticeURL.localPop(); plasmaCrawlNURL.entry nex = noticeURL.localPop();
@ -502,9 +496,8 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi
return false; return false;
} }
// if the server is busy, we do this more slowly // if the server is busy, we do this more slowly
if (serverJobs > 3) try {Thread.currentThread().sleep(100 * serverJobs);} catch (InterruptedException e) {} if (!(cacheManager.idle())) try {Thread.currentThread().sleep(2000);} catch (InterruptedException e) {}
// we don't want to crawl a global URL globally, since WE are the global part. (from this point of view) // we don't want to crawl a global URL globally, since WE are the global part. (from this point of view)
plasmaCrawlNURL.entry nex = noticeURL.remotePop(); plasmaCrawlNURL.entry nex = noticeURL.remotePop();
@ -514,10 +507,7 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi
private synchronized void processResourceStack(plasmaHTCache.Entry entry) { private synchronized void processResourceStack(plasmaHTCache.Entry entry) {
// work off one stack entry with a fresh resource (scraped web page) // work off one stack entry with a fresh resource (scraped web page)
String stats = "DEQUEUE: dequeueing one step (processStack=" + processStack.size() + ", localStackSize=" + noticeURL.localStackSize() + ", remoteStackSize=" + noticeURL.remoteStackSize() + ")"; try {
try {
// we must distinguish the following cases: resource-load was initiated by // we must distinguish the following cases: resource-load was initiated by
// 1) global crawling: the index is extern, not here (not possible here) // 1) global crawling: the index is extern, not here (not possible here)
// 2) result of search queries, some indexes are here (not possible here) // 2) result of search queries, some indexes are here (not possible here)
@ -540,7 +530,7 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi
processCase = 6; processCase = 6;
} }
log.logDebug(stats + " processCase=" + processCase + ", depth=" + entry.depth + ", maxDepth=" + entry.profile.generalDepth() + ", filter=" + entry.profile.generalFilter() + ", initiatorHash=" + initiatorHash + ", status=" + entry.status + ", source=" + ((entry.cacheArray == null) ? "scraper" : "byte[]") + ", url=" + entry.nomalizedURLString); // DEBUG log.logDebug("processResourceStack processCase=" + processCase + ", depth=" + entry.depth + ", maxDepth=" + entry.profile.generalDepth() + ", filter=" + entry.profile.generalFilter() + ", initiatorHash=" + initiatorHash + ", status=" + entry.status + ", source=" + ((entry.cacheArray == null) ? "scraper" : "byte[]") + ", url=" + entry.nomalizedURLString); // DEBUG
// parse content // parse content
plasmaParserDocument document; plasmaParserDocument document;

@ -121,9 +121,10 @@ public class plasmaWordIndexFileCache {
try { try {
row = indexCache.get(wordHash.getBytes()); row = indexCache.get(wordHash.getBytes());
} catch (Exception e) { } catch (Exception e) {
// we had some negativeSeekOffsetExceptions here; in that case the indexCache is corrupt // we had some negativeSeekOffsetExceptions here, and also loops may cause this
// in that case the indexCache is corrupt
System.out.println("Error in plasmaWordINdexFileCache.getCache: index for hash " + wordHash + " is corrupt:" + e.toString()); System.out.println("Error in plasmaWordINdexFileCache.getCache: index for hash " + wordHash + " is corrupt:" + e.toString());
e.printStackTrace(); //e.printStackTrace();
row = null; row = null;
} }
if (row == null) { if (row == null) {
@ -159,7 +160,7 @@ public class plasmaWordIndexFileCache {
} }
protected void addEntriesToIndex(String wordHash, Vector /* of plasmaIndexEntry */ newEntries) throws IOException { protected void addEntriesToIndex(String wordHash, Vector /* of plasmaIndexEntry */ newEntries) throws IOException {
//System.out.println("* adding cached word index: " + wordHash + "=" + word + ":" + entry.toEncodedForm()); // debug //System.out.println("* adding " + newEntries.size() + " cached word index entries for word " + wordHash); // debug
// fetch the index cache // fetch the index cache
if (newEntries.size() == 0) return; if (newEntries.size() == 0) return;
byte[][] row = getCache(wordHash); byte[][] row = getCache(wordHash);

@ -97,7 +97,7 @@ public class plasmaWordIndexRAMCache extends Thread {
while (!(terminate)) { while (!(terminate)) {
if (hashScore.size() < 100) try {Thread.currentThread().sleep(10000);} catch (InterruptedException e) {} if (hashScore.size() < 100) try {Thread.currentThread().sleep(10000);} catch (InterruptedException e) {}
while ((!(terminate)) && (cache != null) && (hashScore.size() > 0)) try { while ((!(terminate)) && (cache != null) && (hashScore.size() > 0)) try {
//check = hashScore.size(); check = hashScore.size();
flushSpecific(false); flushSpecific(false);
//serverLog.logDebug("PLASMA INDEXING", "single flush. bevore=" + check + "; after=" + hashScore.size()); //serverLog.logDebug("PLASMA INDEXING", "single flush. bevore=" + check + "; after=" + hashScore.size());
try {Thread.currentThread().sleep(10 + ((maxWords / 10) / (1 + hashScore.size())));} catch (InterruptedException e) {} try {Thread.currentThread().sleep(10 + ((maxWords / 10) / (1 + hashScore.size())));} catch (InterruptedException e) {}
@ -171,14 +171,12 @@ public class plasmaWordIndexRAMCache extends Thread {
return flushKey(key, "flushSpecific"); return flushKey(key, "flushSpecific");
} }
private int flushKey(String key, String caller) throws IOException { private synchronized int flushKey(String key, String caller) throws IOException {
Vector v = null; Vector v = null;
synchronized (cache) { v = (Vector) cache.get(key);
v = (Vector) cache.get(key); if (v == null) return 0; // flushing of nonexisting key
if (v == null) return 0; // flushing of nonexisting key cache.remove(key);
cache.remove(key); hashScore.deleteScore(key);
hashScore.deleteScore(key);
}
pic.addEntriesToIndex(key, v); pic.addEntriesToIndex(key, v);
return v.size(); return v.size();
} }

@ -53,6 +53,7 @@ public abstract class serverAbstractSwitch implements serverSwitch {
private final Hashtable authorization; private final Hashtable authorization;
private String rootPath; private String rootPath;
private final TreeMap workerThreads; private final TreeMap workerThreads;
protected int serverJobs;
public serverAbstractSwitch(String rootPath, String initPath, String configPath) throws IOException { public serverAbstractSwitch(String rootPath, String initPath, String configPath) throws IOException {
// we initialize the switchboard with a property file, // we initialize the switchboard with a property file,
@ -99,6 +100,9 @@ public abstract class serverAbstractSwitch implements serverSwitch {
// init thread control // init thread control
workerThreads = new TreeMap(); workerThreads = new TreeMap();
// init busy state control
serverJobs = 0;
} }
public static Hashtable loadHashtable(File f) { public static Hashtable loadHashtable(File f) {
@ -287,6 +291,6 @@ public abstract class serverAbstractSwitch implements serverSwitch {
} }
public void handleBusyState(int jobs) { public void handleBusyState(int jobs) {
// do nothing here; should be overridden serverJobs = jobs;
} }
} }

@ -76,6 +76,7 @@ public final class serverCore extends serverAbstractThread implements serverThre
// static variables // static variables
public static final Boolean TERMINATE_CONNECTION = Boolean.FALSE; public static final Boolean TERMINATE_CONNECTION = Boolean.FALSE;
public static final Boolean RESUME_CONNECTION = Boolean.TRUE; public static final Boolean RESUME_CONNECTION = Boolean.TRUE;
public static Hashtable bfHost = new Hashtable(); // for brute-force prevention
// class variables // class variables
private int port; // the listening port private int port; // the listening port
@ -298,19 +299,19 @@ public final class serverCore extends serverAbstractThread implements serverThre
announceThreadBlockApply(); announceThreadBlockApply();
Socket controlSocket = this.socket.accept(); Socket controlSocket = this.socket.accept();
announceThreadBlockRelease(); announceThreadBlockRelease();
if ((this.denyHost == null) || (this.denyHost.get((""+controlSocket.getInetAddress().getHostAddress())) == null)) { String clientIP = ""+controlSocket.getInetAddress().getHostAddress();
//log.logDebug("* catched request from " + controlSocket.getInetAddress().getHostAddress()); if (bfHost.get(clientIP) != null) {
log.logInfo("SLOWING DOWN ACCESS FOR BRUTE-FORCE PREVENTION FROM " + clientIP);
// add a delay to make brute-force harder
try {Thread.currentThread().sleep(1000);} catch (InterruptedException e) {}
}
if ((this.denyHost == null) || (this.denyHost.get(clientIP) == null)) {
controlSocket.setSoTimeout(this.timeout); controlSocket.setSoTimeout(this.timeout);
Session connection = (Session) this.theSessionPool.borrowObject(); Session connection = (Session) this.theSessionPool.borrowObject();
connection.execute(controlSocket); connection.execute(controlSocket);
//log.logDebug("* NEW SESSION: " + connection.request + " from " + clientIP);
//try {Thread.currentThread().sleep(1000);} catch (InterruptedException e) {} // wait for debug
// activeThreads.put(connection, new Long(System.currentTimeMillis()));
//log.logDebug("* NEW SESSION: " + connection.request);
} else { } else {
System.out.println("ACCESS FROM " + controlSocket.getInetAddress().getHostAddress() + " DENIED"); System.out.println("ACCESS FROM " + clientIP + " DENIED");
} }
// idle until number of maximal threads is (again) reached // idle until number of maximal threads is (again) reached
//synchronized(this) { //synchronized(this) {
@ -760,9 +761,9 @@ public final class serverCore extends serverAbstractThread implements serverThre
} catch (NoSuchMethodException nsme) { } catch (NoSuchMethodException nsme) {
System.out.println("ERROR B " + userAddress.getHostAddress()); System.out.println("ERROR B " + userAddress.getHostAddress());
if (isNotLocal(userAddress.getHostAddress().toString())) { if (isNotLocal(userAddress.getHostAddress().toString())) {
if (denyHost != null) if (denyHost != null)
denyHost.put((""+userAddress.getHostAddress()), "deny"); // block client: hacker attempt denyHost.put((""+userAddress.getHostAddress()), "deny"); // block client: hacker attempt
} }
break; break;
// the client requested a command that does not exist // the client requested a command that does not exist
//Object[] errorParameter = { nsme }; //Object[] errorParameter = { nsme };

@ -78,8 +78,8 @@ import de.anomic.yacy.*;
public final class yacy { public final class yacy {
// static objects // static objects
private static final String vString = "0.367"; private static final String vString = "@REPL_VERSION@";
private static final String vDATE = "20050426"; private static final String vDATE = "@REPL_DATE@";
private static final String copyright = "[ YACY Proxy v" + vString + ", build " + vDATE + " by Michael Christen / www.yacy.net ]"; private static final String copyright = "[ YACY Proxy v" + vString + ", build " + vDATE + " by Michael Christen / www.yacy.net ]";
private static final String hline = "-------------------------------------------------------------------------------"; private static final String hline = "-------------------------------------------------------------------------------";

Loading…
Cancel
Save