fixed caching+synchronization+brute-force-denial

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@67 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 20 years ago
parent 56409402f0
commit 2de90020ed

@ -45,7 +45,7 @@
# Contributions and changes to the program code must be marked as such.
# define variables
version='0.367'
version='0.368'
datestr=`date +%Y%m%d`
#release='yacy_v'$version'_'$datestr
release='yacy_dev_v'$version'_'$datestr
@ -109,8 +109,6 @@ mv -f $source/$mainclass $source/$mainclass.orig
sed `echo 's/@REPL_DATE@/'$datestr'/'` $source/$mainclass.orig > $source/$mainclass.sed1
sed `echo 's/@REPL_VERSION@/'$version'/'` $source/$mainclass.sed1 > $source/$mainclass
rm $source/$mainclass.sed1
#javac -classpath $classpath -sourcepath $source -d $classes -g:none $source/httpd.java
#javac -classpath $classpath -sourcepath $source -d $classes -g:none $source/$mainclass
javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/tools/*.java
javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/net/*.java
javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/htmlFilter/*.java
@ -125,7 +123,6 @@ javac -classpath $classpath -sourcepath $source -d $classes -g $source/$mainclas
mv -f $source/$mainclass.orig $source/$mainclass
# compile server pages
#javac -classpath $classes -sourcepath htroot -d $classes -g htroot/*.java
javac -classpath $classes -sourcepath htroot -d htroot -g htroot/*.java
javac -classpath $classes -sourcepath htroot/yacy -d htroot/yacy -g htroot/yacy/*.java
javac -classpath $classes -sourcepath htroot/htdocsdefault -d htroot/htdocsdefault -g htroot/htdocsdefault/*.java

@ -225,10 +225,14 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
out.write(("\r\n").getBytes());
out.flush();
return;
} else if (!(adminAccountBase64MD5.equals(serverCodings.standardCoder.encodeMD5Hex(auth.trim().substring(6))))) {
} else if (adminAccountBase64MD5.equals(serverCodings.standardCoder.encodeMD5Hex(auth.trim().substring(6)))) {
// remove brute-force flag
serverCore.bfHost.remove(conProp.getProperty("CLIENTIP"));
} else {
// a wrong authentication was given. Ask again
serverLog.logInfo("HTTPD", "Wrong log-in for account 'admin' in http file handler for path '" + path + "' from host '" + conProp.getProperty("CLIENTIP", "unknown-IP") + "'");
try {Thread.currentThread().sleep(3000);} catch (InterruptedException e) {} // add a delay to make brute-force harder
//try {Thread.currentThread().sleep(3000);} catch (InterruptedException e) {} // add a delay to make brute-force harder
serverCore.bfHost.put(conProp.getProperty("CLIENTIP"), "sleep");
out.write(("HTTP/1.1 401 log-in required\r\n").getBytes());
out.write(("WWW-Authenticate: Basic realm=\"admin log-in\"\r\n").getBytes());
out.write(("\r\n").getBytes());

@ -323,11 +323,10 @@ public class kelondroRecords {
Node n = (Node) cache.get(handle);
if (n == null) {
n = new Node(handle, parentNode, referenceInParent);
cache.put(handle, n);
cacheScore.setScore(handle, (int) ((System.currentTimeMillis() - startup) / 1000));
checkCacheSpace();
checkCacheSpace();
return n;
} else {
//System.out.println("read from cache " + n.toString());
cacheScore.setScore(handle, (int) ((System.currentTimeMillis() - startup) / 1000));
return n;
}
@ -336,7 +335,7 @@ public class kelondroRecords {
protected void deleteNode(Handle handle) throws IOException {
if (cachesize != 0) {
Node n = (Node) cache.get(handle);
if (n != null) {
if (n != null) synchronized (cache) {
cacheScore.deleteScore(handle);
cache.remove(handle);
}
@ -351,14 +350,18 @@ public class kelondroRecords {
// delete one entry
try {
Handle delkey = (Handle) cacheScore.getMinObject(); // error (see below) here
cacheScore.deleteScore(delkey);
cache.remove(delkey);
synchronized (cache) {
cacheScore.deleteScore(delkey);
cache.remove(delkey);
}
} catch (NoSuchElementException e) {
System.out.println("strange kelondroRecords error: " + e.getMessage() + "; cachesize=" + cachesize + ", cache.size()=" + cache.size() + ", cacheScore.size()=" + cacheScore.size());
// this is a strange error and could be caused by internal java problems
// we simply clear the cache
this.cacheScore = new kelondroMScoreCluster();
this.cache = new HashMap();
synchronized (cache) {
this.cacheScore = new kelondroMScoreCluster();
this.cache = new HashMap();
}
}
}
}
@ -441,11 +444,11 @@ public class kelondroRecords {
if (b.length != OHBYTEC) throw new IllegalArgumentException("setOHByte: wrong array size");
if (this.handle.index == NUL) throw new kelondroException(filename, "setOHByte: no handle assigned");
if (this.ohBytes == null) this.ohBytes = new byte[OHBYTEC];
entryFile.seek(seekpos(this.handle));
entryFile.seek(seekpos(this.handle));
for (int j = 0; j < ohBytes.length; j++) {
ohBytes[j] = b[j];
entryFile.writeByte(b[j]);
}
ohBytes[j] = b[j];
entryFile.writeByte(b[j]);
}
updateNode();
}
protected synchronized void setOHHandle(Handle[] i) throws IOException {
@ -453,9 +456,9 @@ public class kelondroRecords {
if (i.length != OHHANDLEC) throw new IllegalArgumentException("setOHHandle: wrong array size");
if (this.handle.index == NUL) throw new kelondroException(filename, "setOHHandle: no handle assigned");
if (this.ohHandle == null) this.ohHandle = new Handle[OHHANDLEC];
entryFile.seek(seekpos(this.handle) + OHBYTEC);
entryFile.seek(seekpos(this.handle) + OHBYTEC);
for (int j = 0; j < ohHandle.length; j++) {
ohHandle[j] = i[j];
ohHandle[j] = i[j];
if (i[j] == null)
entryFile.writeInt(NUL);
else
@ -469,8 +472,8 @@ public class kelondroRecords {
ohBytes = new byte[OHBYTEC];
entryFile.seek(seekpos(this.handle));
for (int j = 0; j < ohBytes.length; j++) {
ohBytes[j] = entryFile.readByte();
}
ohBytes[j] = entryFile.readByte();
}
updateNode();
}
return ohBytes;
@ -555,20 +558,20 @@ public class kelondroRecords {
}
} else if ((values.length > 1) && (values[1] == null)) {
// only the key has been read; load the remaining
long seek = seekpos(this.handle) + overhead + COLWIDTHS[0];
for (int i = 1; i < COLWIDTHS.length; i++) {
entryFile.seek(seek);
values[i] = new byte[COLWIDTHS[i]];
entryFile.read(values[i], 0, values[i].length);
seek = seek + COLWIDTHS[i];
}
updateNode();
return values;
long seek = seekpos(this.handle) + overhead + COLWIDTHS[0];
for (int i = 1; i < COLWIDTHS.length; i++) {
entryFile.seek(seek);
values[i] = new byte[COLWIDTHS[i]];
entryFile.read(values[i], 0, values[i].length);
seek = seek + COLWIDTHS[i];
}
updateNode();
return values;
} else {
return values;
}
}
protected void save() throws IOException {
protected synchronized void save() throws IOException {
// this is called when an entry was defined with values only and not by retrieving with an index
// if this happens, nothing of the internal array values have been written to the file
// then writing to the file is done here
@ -585,9 +588,9 @@ public class kelondroRecords {
throw new kelondroException(filename, "no values to save");
}
entryFile.seek(seekpos(this.handle));
if (ohBytes == null) {for (int i = 0; i < OHBYTEC; i++) entryFile.writeByte(0);}
if (ohBytes == null) {for (int i = 0; i < OHBYTEC; i++) entryFile.writeByte(0);}
else {for (int i = 0; i < OHBYTEC; i++) entryFile.writeByte(ohBytes[i]);}
if (ohHandle == null) {for (int i = 0; i < OHHANDLEC; i++) entryFile.writeInt(0);}
if (ohHandle == null) {for (int i = 0; i < OHHANDLEC; i++) entryFile.writeInt(0);}
else {for (int i = 0; i < OHHANDLEC; i++) entryFile.writeInt(ohHandle[i].index);}
long seek = seekpos(this.handle) + overhead;
for (int i = 0; i < values.length; i++) {
@ -623,8 +626,12 @@ public class kelondroRecords {
private void updateNode() {
if (cachesize != 0) {
if (!(cache.containsKey(handle))) checkCacheSpace();
cache.put(handle, this);
cacheScore.setScore(handle, (int) ((System.currentTimeMillis() - startup) / 1000));
synchronized (cache) {
//System.out.println("updateNode " + this.toString());
cache.put(handle, this);
cacheScore.setScore(handle, (int) ((System.currentTimeMillis() - startup) / 1000));
//System.out.println("cache now: " + cache.toString());
}
}
}
}
@ -844,7 +851,9 @@ public class kelondroRecords {
if (index > ((Handle) h).index) return 1;
return 0;
}
public int hashCode() {
return this.index;
}
}

@ -114,7 +114,8 @@ public class kelondroTree extends kelondroRecords implements Comparator {
}
// Returns the value to which this map maps the specified key.
public synchronized byte[][] get(byte[] key) throws IOException {
public byte[][] get(byte[] key) throws IOException {
//System.out.println("kelondroTree.get " + new String(key) + " in " + filename);
Search search = new Search(key);
if (search.found()) {
return search.getMatcher().getValues();
@ -123,7 +124,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
}
}
public synchronized long[] getLong(byte[] key) throws IOException {
public long[] getLong(byte[] key) throws IOException {
byte[][] row = get(key);
long[] longs = new long[columns() - 1];
if (row == null) {
@ -181,6 +182,9 @@ public class kelondroTree extends kelondroRecords implements Comparator {
found = false;
int c;
Handle[] handles;
HashMap visitedNodeKeys = new HashMap(); // to detect loops
String otherkey;
//System.out.println("Starting Compare Loop in Database " + filename); // debug
while (thisHandle != null) {
try {
parentnode = thenode;
@ -190,7 +194,23 @@ public class kelondroTree extends kelondroRecords implements Comparator {
found = false;
return;
}
//System.out.print("Comparing key = '" + new String(key) + "' with '" + new String(thenode.node().getKey()) + "':"); // debug
otherkey = new String(thenode.getKey());
if (visitedNodeKeys.containsKey(otherkey)) {
// we have loops in the database.
// to fix this, all affected nodes must be patched
thenode.setOHByte(new byte[] {1, 0});
thenode.setOHHandle(new Handle[] {null, null, null});
Iterator fix = visitedNodeKeys.entrySet().iterator();
Map.Entry entry;
while (fix.hasNext()) {
entry = (Map.Entry) fix.next();
thenode = (Node) entry.getValue();
thenode.setOHByte(new byte[] {1, 0});
thenode.setOHHandle(new Handle[] {null, null, null});
}
throw new kelondroException(filename, "database contains loops; the loop-nodes have been auto-fixed");
}
//System.out.print("Comparing key = '" + new String(key) + "' with '" + otherkey + "':"); // debug
c = compare(key, thenode.getKey());
//System.out.println(c); // debug
if (c == 0) {
@ -203,6 +223,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
child = 1;
thisHandle = thenode.getOHHandle()[rightchild];
}
visitedNodeKeys.put(otherkey, thenode);
}
}
// we reached a node where we must insert the new value
@ -238,7 +259,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
}
}
public synchronized boolean isChild(Node childn, Node parentn, int child) throws IOException {
public boolean isChild(Node childn, Node parentn, int child) throws IOException {
if (childn == null) throw new IllegalArgumentException("isLeftChild: Node parameter is NULL");
Handle lc = parentn.getOHHandle()[child];
if (lc == null) return false;
@ -356,7 +377,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
}
}
public synchronized long[] putLong(byte[] key, long[] newlongs) throws IOException {
public long[] putLong(byte[] key, long[] newlongs) throws IOException {
byte[][] newrow = new byte[newlongs.length + 1][];
newrow[0] = key;
for (int i = 0; i < newlongs.length; i++) {
@ -377,7 +398,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
}
// Associates the specified value with the specified key in this map
public synchronized byte[][] put(byte[][] newrow) throws IOException {
public byte[][] put(byte[][] newrow) throws IOException {
if (newrow.length != columns()) throw new IllegalArgumentException("put: wrong row length " + newrow.length + "; must be " + columns());
// first try to find the key element in the database
Search searchResult = new Search(newrow[0]);
@ -625,7 +646,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
}
// Associates the specified value with the specified key in this map
public synchronized byte[] put(byte[] key, byte[] value) throws IOException {
public byte[] put(byte[] key, byte[] value) throws IOException {
byte[][] row = new byte[2][];
row[0] = key;
row[1] = value;
@ -634,7 +655,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
}
// Removes the mapping for this key from this map if present (optional operation).
public synchronized byte[][] remove(byte[] key) throws IOException {
public byte[][] remove(byte[] key) throws IOException {
Search search = new Search(key);
if (search.found()) {
Node result = search.getMatcher();
@ -646,11 +667,11 @@ public class kelondroTree extends kelondroRecords implements Comparator {
}
}
public synchronized void removeAll() throws IOException {
public void removeAll() throws IOException {
while (size() > 0) remove(lastNode(), null);
}
public synchronized void remove(Node node, Node parentOfNode) throws IOException {
public void remove(Node node, Node parentOfNode) throws IOException {
// there are three cases when removing a node
// - the node is a leaf - it can be removed easily
// - the node has one child - the child replaces the node
@ -814,50 +835,6 @@ public class kelondroTree extends kelondroRecords implements Comparator {
throw new RuntimeException("error creating an iteration: " + e.getMessage());
}
}
/*
public synchronized keyIterator keys(boolean up, boolean rotating) throws IOException {
// iterates only the keys of the Nodes
// enumerated objects are of type byte[]
// iterates the elements in a sorted way.
return new keyIterator(new nodeIterator(up, rotating));
}
public synchronized keyIterator keys(boolean up, boolean rotating, byte[] firstKey) throws IOException {
Search s = new Search(firstKey);
if (s.found()) {
return new keyIterator(new nodeIterator(up, rotating, s.getMatcher()));
} else {
Node nn = s.getParent();
if (nn == null) {
return (keyIterator) (new HashSet()).iterator();
} else {
return new keyIterator(new nodeIterator(up, rotating, nn));
}
}
}
public class keyIterator implements Iterator {
// the iterator iterates all keys, which are byte[] objects
Iterator nodeIterator;
public keyIterator(nodeIterator nodeIterator) {
this.nodeIterator = nodeIterator;
}
public boolean hasNext() {
return nodeIterator.hasNext();
}
public Object next() {
try {
return ((Node) nodeIterator.next()).getKey();
} catch (IOException e) {
return null;
}
}
public void remove() {
nodeIterator.remove();
}
}
*/
public synchronized rowIterator rows(boolean up, boolean rotating) throws IOException {
// iterates only the keys of the Nodes
@ -907,7 +884,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
}
public synchronized int imp(File file, String separator) throws IOException {
public int imp(File file, String separator) throws IOException {
// imports a value-separated file, returns number of records that have been read
RandomAccessFile f = new RandomAccessFile(file,"r");
@ -1162,7 +1139,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
}
public synchronized int compare(Object a, Object b) {
public int compare(Object a, Object b) {
try {
if ((a instanceof byte[]) && (b instanceof byte[])) {
return compare((byte[]) a, (byte[]) b);
@ -1208,8 +1185,8 @@ public class kelondroTree extends kelondroRecords implements Comparator {
public static void main(String[] args) {
//cmd(args);
bigtest(Integer.parseInt(args[0]));
//randomtest(Integer.parseInt(args[0]));
//bigtest(Integer.parseInt(args[0]));
randomtest(Integer.parseInt(args[0]));
//smalltest();
}
@ -1254,7 +1231,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
String s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".substring(0, elements);
String t, d;
char c;
kelondroTree tt;
kelondroTree tt = null;
File testFile = new File("test.db");
byte[] b;
try {
@ -1284,6 +1261,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
t = t + c;
System.out.println("removed " + new String(b));
}
//tt.print();
if (countElements(tt) != tt.size()) {
System.out.println("wrong size for ");
tt.print();
@ -1313,6 +1291,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
} catch (Exception e) {
e.printStackTrace();
try {tt.print();} catch (IOException ee) {}
System.out.println("TERMINATED");
}
}
@ -1321,7 +1300,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
File f = new File("test.db");
if (f.exists()) f.delete();
try {
kelondroTree tt = new kelondroTree(f, 0, 4, 4);
kelondroTree tt = new kelondroTree(f, 1000, 4, 4);
byte[] b;
b = testWord('b'); tt.put(b, b);
b = testWord('c'); tt.put(b, b);

@ -296,7 +296,9 @@ public class plasmaCrawlNURL extends plasmaURL {
urlHashCache.put(entry);
} catch (IOException e) {
System.out.println("INTERNAL ERROR AT plasmaNURL:url2hash:" + e.toString());
}
} catch (kelondroException e) {
serverLog.logError("PLASMA", "plasmaCrawlNURL.store failed: " + e.getMessage());
}
}
public String hash() {

@ -199,15 +199,15 @@ public final class plasmaHTCache {
return header;
}
boolean idle() {
public boolean idle() {
return (System.currentTimeMillis() > (idleDelay + lastAcc));
}
boolean full() {
public boolean full() {
return (cacheStack.size() > stackLimit);
}
boolean empty() {
public boolean empty() {
return (cacheStack.size() == 0);
}
@ -311,7 +311,7 @@ public final class plasmaHTCache {
try {
File f;
int workoff;
workoff = cacheStack.size() / 10;
workoff = 1 + cacheStack.size() / 10;
// we want to work off always 10 % to prevent that we collaps
while ((workoff-- > 0) && (!(empty()))) {
process((Entry) cacheStack.removeFirst());
@ -427,8 +427,8 @@ public final class plasmaHTCache {
}
public static boolean isCGI(String urlString) {
return ((urlString.toLowerCase().indexOf("cgi") >= 0) ||
(urlString.toLowerCase().indexOf("exe") >= 0));
return ((urlString.toLowerCase().indexOf(".cgi") >= 0) ||
(urlString.toLowerCase().indexOf(".exe") >= 0));
}
public Entry newEntry(Date initDate, int depth, URL url,

@ -148,15 +148,13 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi
public HashMap outgoingCookies, incomingCookies;
public kelondroTables facilityDB;
public plasmaParser parser;
public int serverJobs;
private serverSemaphore shutdownSync = new serverSemaphore(0);
private boolean terminate = false;
public plasmaSwitchboard(String rootPath, String initPath, String configPath) throws IOException {
super(rootPath, initPath, configPath);
serverJobs = 0;
// set loglevel
int loglevel = Integer.parseInt(getConfig("plasmaLoglevel", "2"));
log = new serverLog("PLASMA", loglevel);
@ -300,10 +298,6 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi
return (bytes / 1024) + "TByte";
}
public void handleBusyState(int jobs) {
this.serverJobs = jobs;
}
private void initProfiles() throws IOException {
if ((profiles.size() == 0) ||
(getConfig("defaultProxyProfile", "").length() == 0) ||
@ -414,10 +408,10 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi
if (processStack.size() == 0) return false; // nothing to do
// in case that the server is very busy we do not work off the queue too fast
if (serverJobs > 10) try {Thread.currentThread().sleep(10 * serverJobs);} catch (InterruptedException e) {}
if (!(cacheManager.idle())) try {Thread.currentThread().sleep(1000);} catch (InterruptedException e) {}
// do one processing step
log.logDebug("DEQUEUE: serverJobs=" + serverJobs +
log.logDebug("DEQUEUE: cacheManager=" + ((cacheManager.idle()) ? "idle" : "busy") +
", processStack=" + processStack.size() +
", localStackSize=" + noticeURL.localStackSize() +
", remoteStackSize=" + noticeURL.remoteStackSize());
@ -473,7 +467,7 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi
}
// if the server is busy, we do crawling more slowly
if (serverJobs > 3) try {Thread.currentThread().sleep(100 * serverJobs);} catch (InterruptedException e) {}
if (!(cacheManager.idle())) try {Thread.currentThread().sleep(2000);} catch (InterruptedException e) {}
// do a local crawl (may start a global crawl)
plasmaCrawlNURL.entry nex = noticeURL.localPop();
@ -502,9 +496,8 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi
return false;
}
// if the server is busy, we do this more slowly
if (serverJobs > 3) try {Thread.currentThread().sleep(100 * serverJobs);} catch (InterruptedException e) {}
if (!(cacheManager.idle())) try {Thread.currentThread().sleep(2000);} catch (InterruptedException e) {}
// we don't want to crawl a global URL globally, since WE are the global part. (from this point of view)
plasmaCrawlNURL.entry nex = noticeURL.remotePop();
@ -514,10 +507,7 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi
private synchronized void processResourceStack(plasmaHTCache.Entry entry) {
// work off one stack entry with a fresh resource (scraped web page)
String stats = "DEQUEUE: dequeueing one step (processStack=" + processStack.size() + ", localStackSize=" + noticeURL.localStackSize() + ", remoteStackSize=" + noticeURL.remoteStackSize() + ")";
try {
try {
// we must distinguish the following cases: resource-load was initiated by
// 1) global crawling: the index is extern, not here (not possible here)
// 2) result of search queries, some indexes are here (not possible here)
@ -540,7 +530,7 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi
processCase = 6;
}
log.logDebug(stats + " processCase=" + processCase + ", depth=" + entry.depth + ", maxDepth=" + entry.profile.generalDepth() + ", filter=" + entry.profile.generalFilter() + ", initiatorHash=" + initiatorHash + ", status=" + entry.status + ", source=" + ((entry.cacheArray == null) ? "scraper" : "byte[]") + ", url=" + entry.nomalizedURLString); // DEBUG
log.logDebug("processResourceStack processCase=" + processCase + ", depth=" + entry.depth + ", maxDepth=" + entry.profile.generalDepth() + ", filter=" + entry.profile.generalFilter() + ", initiatorHash=" + initiatorHash + ", status=" + entry.status + ", source=" + ((entry.cacheArray == null) ? "scraper" : "byte[]") + ", url=" + entry.nomalizedURLString); // DEBUG
// parse content
plasmaParserDocument document;

@ -121,9 +121,10 @@ public class plasmaWordIndexFileCache {
try {
row = indexCache.get(wordHash.getBytes());
} catch (Exception e) {
// we had some negativeSeekOffsetExceptions here; in that case the indexCache is corrupt
// we had some negativeSeekOffsetExceptions here, and also loops may cause this
// in that case the indexCache is corrupt
System.out.println("Error in plasmaWordINdexFileCache.getCache: index for hash " + wordHash + " is corrupt:" + e.toString());
e.printStackTrace();
//e.printStackTrace();
row = null;
}
if (row == null) {
@ -159,7 +160,7 @@ public class plasmaWordIndexFileCache {
}
protected void addEntriesToIndex(String wordHash, Vector /* of plasmaIndexEntry */ newEntries) throws IOException {
//System.out.println("* adding cached word index: " + wordHash + "=" + word + ":" + entry.toEncodedForm()); // debug
//System.out.println("* adding " + newEntries.size() + " cached word index entries for word " + wordHash); // debug
// fetch the index cache
if (newEntries.size() == 0) return;
byte[][] row = getCache(wordHash);

@ -97,7 +97,7 @@ public class plasmaWordIndexRAMCache extends Thread {
while (!(terminate)) {
if (hashScore.size() < 100) try {Thread.currentThread().sleep(10000);} catch (InterruptedException e) {}
while ((!(terminate)) && (cache != null) && (hashScore.size() > 0)) try {
//check = hashScore.size();
check = hashScore.size();
flushSpecific(false);
//serverLog.logDebug("PLASMA INDEXING", "single flush. bevore=" + check + "; after=" + hashScore.size());
try {Thread.currentThread().sleep(10 + ((maxWords / 10) / (1 + hashScore.size())));} catch (InterruptedException e) {}
@ -171,14 +171,12 @@ public class plasmaWordIndexRAMCache extends Thread {
return flushKey(key, "flushSpecific");
}
private int flushKey(String key, String caller) throws IOException {
private synchronized int flushKey(String key, String caller) throws IOException {
Vector v = null;
synchronized (cache) {
v = (Vector) cache.get(key);
if (v == null) return 0; // flushing of nonexisting key
cache.remove(key);
hashScore.deleteScore(key);
}
v = (Vector) cache.get(key);
if (v == null) return 0; // flushing of nonexisting key
cache.remove(key);
hashScore.deleteScore(key);
pic.addEntriesToIndex(key, v);
return v.size();
}

@ -53,6 +53,7 @@ public abstract class serverAbstractSwitch implements serverSwitch {
private final Hashtable authorization;
private String rootPath;
private final TreeMap workerThreads;
protected int serverJobs;
public serverAbstractSwitch(String rootPath, String initPath, String configPath) throws IOException {
// we initialize the switchboard with a property file,
@ -99,6 +100,9 @@ public abstract class serverAbstractSwitch implements serverSwitch {
// init thread control
workerThreads = new TreeMap();
// init busy state control
serverJobs = 0;
}
public static Hashtable loadHashtable(File f) {
@ -287,6 +291,6 @@ public abstract class serverAbstractSwitch implements serverSwitch {
}
public void handleBusyState(int jobs) {
// do nothing here; should be overridden
serverJobs = jobs;
}
}

@ -76,6 +76,7 @@ public final class serverCore extends serverAbstractThread implements serverThre
// static variables
public static final Boolean TERMINATE_CONNECTION = Boolean.FALSE;
public static final Boolean RESUME_CONNECTION = Boolean.TRUE;
public static Hashtable bfHost = new Hashtable(); // for brute-force prevention
// class variables
private int port; // the listening port
@ -298,19 +299,19 @@ public final class serverCore extends serverAbstractThread implements serverThre
announceThreadBlockApply();
Socket controlSocket = this.socket.accept();
announceThreadBlockRelease();
if ((this.denyHost == null) || (this.denyHost.get((""+controlSocket.getInetAddress().getHostAddress())) == null)) {
//log.logDebug("* catched request from " + controlSocket.getInetAddress().getHostAddress());
String clientIP = ""+controlSocket.getInetAddress().getHostAddress();
if (bfHost.get(clientIP) != null) {
log.logInfo("SLOWING DOWN ACCESS FOR BRUTE-FORCE PREVENTION FROM " + clientIP);
// add a delay to make brute-force harder
try {Thread.currentThread().sleep(1000);} catch (InterruptedException e) {}
}
if ((this.denyHost == null) || (this.denyHost.get(clientIP) == null)) {
controlSocket.setSoTimeout(this.timeout);
Session connection = (Session) this.theSessionPool.borrowObject();
connection.execute(controlSocket);
//try {Thread.currentThread().sleep(1000);} catch (InterruptedException e) {} // wait for debug
// activeThreads.put(connection, new Long(System.currentTimeMillis()));
//log.logDebug("* NEW SESSION: " + connection.request);
//log.logDebug("* NEW SESSION: " + connection.request + " from " + clientIP);
} else {
System.out.println("ACCESS FROM " + controlSocket.getInetAddress().getHostAddress() + " DENIED");
System.out.println("ACCESS FROM " + clientIP + " DENIED");
}
// idle until number of maximal threads is (again) reached
//synchronized(this) {
@ -760,9 +761,9 @@ public final class serverCore extends serverAbstractThread implements serverThre
} catch (NoSuchMethodException nsme) {
System.out.println("ERROR B " + userAddress.getHostAddress());
if (isNotLocal(userAddress.getHostAddress().toString())) {
if (denyHost != null)
denyHost.put((""+userAddress.getHostAddress()), "deny"); // block client: hacker attempt
}
if (denyHost != null)
denyHost.put((""+userAddress.getHostAddress()), "deny"); // block client: hacker attempt
}
break;
// the client requested a command that does not exist
//Object[] errorParameter = { nsme };

@ -78,8 +78,8 @@ import de.anomic.yacy.*;
public final class yacy {
// static objects
private static final String vString = "0.367";
private static final String vDATE = "20050426";
private static final String vString = "@REPL_VERSION@";
private static final String vDATE = "@REPL_DATE@";
private static final String copyright = "[ YACY Proxy v" + vString + ", build " + vDATE + " by Michael Christen / www.yacy.net ]";
private static final String hline = "-------------------------------------------------------------------------------";

Loading…
Cancel
Save