From 2de90020eddd7997cb6ff7920d35fe8be32dadb8 Mon Sep 17 00:00:00 2001 From: orbiter Date: Wed, 27 Apr 2005 21:09:40 +0000 Subject: [PATCH] fixed caching+synchronization+brute-force-denial git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@67 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- makerelease.sh | 5 +- source/de/anomic/http/httpdFileHandler.java | 8 +- .../de/anomic/kelondro/kelondroRecords.java | 71 +++++++------ source/de/anomic/kelondro/kelondroTree.java | 99 ++++++++----------- source/de/anomic/plasma/plasmaCrawlNURL.java | 4 +- source/de/anomic/plasma/plasmaHTCache.java | 12 +-- .../de/anomic/plasma/plasmaSwitchboard.java | 24 ++--- .../plasma/plasmaWordIndexFileCache.java | 7 +- .../plasma/plasmaWordIndexRAMCache.java | 14 ++- .../anomic/server/serverAbstractSwitch.java | 6 +- source/de/anomic/server/serverCore.java | 25 ++--- source/yacy.java | 4 +- 12 files changed, 132 insertions(+), 147 deletions(-) diff --git a/makerelease.sh b/makerelease.sh index 4b3de8c85..1081afd6e 100755 --- a/makerelease.sh +++ b/makerelease.sh @@ -45,7 +45,7 @@ # Contributions and changes to the program code must be marked as such. # define variables -version='0.367' +version='0.368' datestr=`date +%Y%m%d` #release='yacy_v'$version'_'$datestr release='yacy_dev_v'$version'_'$datestr @@ -109,8 +109,6 @@ mv -f $source/$mainclass $source/$mainclass.orig sed `echo 's/@REPL_DATE@/'$datestr'/'` $source/$mainclass.orig > $source/$mainclass.sed1 sed `echo 's/@REPL_VERSION@/'$version'/'` $source/$mainclass.sed1 > $source/$mainclass rm $source/$mainclass.sed1 -#javac -classpath $classpath -sourcepath $source -d $classes -g:none $source/httpd.java -#javac -classpath $classpath -sourcepath $source -d $classes -g:none $source/$mainclass javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/tools/*.java javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/net/*.java javac -classpath $classpath -sourcepath $source -d $classes -g $source/de/anomic/htmlFilter/*.java @@ -125,7 +123,6 @@ javac -classpath $classpath -sourcepath $source -d $classes -g $source/$mainclas mv -f $source/$mainclass.orig $source/$mainclass # compile server pages -#javac -classpath $classes -sourcepath htroot -d $classes -g htroot/*.java javac -classpath $classes -sourcepath htroot -d htroot -g htroot/*.java javac -classpath $classes -sourcepath htroot/yacy -d htroot/yacy -g htroot/yacy/*.java javac -classpath $classes -sourcepath htroot/htdocsdefault -d htroot/htdocsdefault -g htroot/htdocsdefault/*.java diff --git a/source/de/anomic/http/httpdFileHandler.java b/source/de/anomic/http/httpdFileHandler.java index 7d38874a1..4badd3624 100644 --- a/source/de/anomic/http/httpdFileHandler.java +++ b/source/de/anomic/http/httpdFileHandler.java @@ -225,10 +225,14 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http out.write(("\r\n").getBytes()); out.flush(); return; - } else if (!(adminAccountBase64MD5.equals(serverCodings.standardCoder.encodeMD5Hex(auth.trim().substring(6))))) { + } else if (adminAccountBase64MD5.equals(serverCodings.standardCoder.encodeMD5Hex(auth.trim().substring(6)))) { + // remove brute-force flag + serverCore.bfHost.remove(conProp.getProperty("CLIENTIP")); + } else { // a wrong authentication was given. Ask again serverLog.logInfo("HTTPD", "Wrong log-in for account 'admin' in http file handler for path '" + path + "' from host '" + conProp.getProperty("CLIENTIP", "unknown-IP") + "'"); - try {Thread.currentThread().sleep(3000);} catch (InterruptedException e) {} // add a delay to make brute-force harder + //try {Thread.currentThread().sleep(3000);} catch (InterruptedException e) {} // add a delay to make brute-force harder + serverCore.bfHost.put(conProp.getProperty("CLIENTIP"), "sleep"); out.write(("HTTP/1.1 401 log-in required\r\n").getBytes()); out.write(("WWW-Authenticate: Basic realm=\"admin log-in\"\r\n").getBytes()); out.write(("\r\n").getBytes()); diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java index 9ee3bd23c..3999168a5 100644 --- a/source/de/anomic/kelondro/kelondroRecords.java +++ b/source/de/anomic/kelondro/kelondroRecords.java @@ -323,11 +323,10 @@ public class kelondroRecords { Node n = (Node) cache.get(handle); if (n == null) { n = new Node(handle, parentNode, referenceInParent); - cache.put(handle, n); - cacheScore.setScore(handle, (int) ((System.currentTimeMillis() - startup) / 1000)); - checkCacheSpace(); + checkCacheSpace(); return n; } else { + //System.out.println("read from cache " + n.toString()); cacheScore.setScore(handle, (int) ((System.currentTimeMillis() - startup) / 1000)); return n; } @@ -336,7 +335,7 @@ public class kelondroRecords { protected void deleteNode(Handle handle) throws IOException { if (cachesize != 0) { Node n = (Node) cache.get(handle); - if (n != null) { + if (n != null) synchronized (cache) { cacheScore.deleteScore(handle); cache.remove(handle); } @@ -351,14 +350,18 @@ public class kelondroRecords { // delete one entry try { Handle delkey = (Handle) cacheScore.getMinObject(); // error (see below) here - cacheScore.deleteScore(delkey); - cache.remove(delkey); + synchronized (cache) { + cacheScore.deleteScore(delkey); + cache.remove(delkey); + } } catch (NoSuchElementException e) { System.out.println("strange kelondroRecords error: " + e.getMessage() + "; cachesize=" + cachesize + ", cache.size()=" + cache.size() + ", cacheScore.size()=" + cacheScore.size()); // this is a strange error and could be caused by internal java problems // we simply clear the cache - this.cacheScore = new kelondroMScoreCluster(); - this.cache = new HashMap(); + synchronized (cache) { + this.cacheScore = new kelondroMScoreCluster(); + this.cache = new HashMap(); + } } } } @@ -441,11 +444,11 @@ public class kelondroRecords { if (b.length != OHBYTEC) throw new IllegalArgumentException("setOHByte: wrong array size"); if (this.handle.index == NUL) throw new kelondroException(filename, "setOHByte: no handle assigned"); if (this.ohBytes == null) this.ohBytes = new byte[OHBYTEC]; - entryFile.seek(seekpos(this.handle)); + entryFile.seek(seekpos(this.handle)); for (int j = 0; j < ohBytes.length; j++) { - ohBytes[j] = b[j]; - entryFile.writeByte(b[j]); - } + ohBytes[j] = b[j]; + entryFile.writeByte(b[j]); + } updateNode(); } protected synchronized void setOHHandle(Handle[] i) throws IOException { @@ -453,9 +456,9 @@ public class kelondroRecords { if (i.length != OHHANDLEC) throw new IllegalArgumentException("setOHHandle: wrong array size"); if (this.handle.index == NUL) throw new kelondroException(filename, "setOHHandle: no handle assigned"); if (this.ohHandle == null) this.ohHandle = new Handle[OHHANDLEC]; - entryFile.seek(seekpos(this.handle) + OHBYTEC); + entryFile.seek(seekpos(this.handle) + OHBYTEC); for (int j = 0; j < ohHandle.length; j++) { - ohHandle[j] = i[j]; + ohHandle[j] = i[j]; if (i[j] == null) entryFile.writeInt(NUL); else @@ -469,8 +472,8 @@ public class kelondroRecords { ohBytes = new byte[OHBYTEC]; entryFile.seek(seekpos(this.handle)); for (int j = 0; j < ohBytes.length; j++) { - ohBytes[j] = entryFile.readByte(); - } + ohBytes[j] = entryFile.readByte(); + } updateNode(); } return ohBytes; @@ -555,20 +558,20 @@ public class kelondroRecords { } } else if ((values.length > 1) && (values[1] == null)) { // only the key has been read; load the remaining - long seek = seekpos(this.handle) + overhead + COLWIDTHS[0]; - for (int i = 1; i < COLWIDTHS.length; i++) { - entryFile.seek(seek); - values[i] = new byte[COLWIDTHS[i]]; - entryFile.read(values[i], 0, values[i].length); - seek = seek + COLWIDTHS[i]; - } - updateNode(); - return values; + long seek = seekpos(this.handle) + overhead + COLWIDTHS[0]; + for (int i = 1; i < COLWIDTHS.length; i++) { + entryFile.seek(seek); + values[i] = new byte[COLWIDTHS[i]]; + entryFile.read(values[i], 0, values[i].length); + seek = seek + COLWIDTHS[i]; + } + updateNode(); + return values; } else { return values; } } - protected void save() throws IOException { + protected synchronized void save() throws IOException { // this is called when an entry was defined with values only and not by retrieving with an index // if this happens, nothing of the internal array values have been written to the file // then writing to the file is done here @@ -585,9 +588,9 @@ public class kelondroRecords { throw new kelondroException(filename, "no values to save"); } entryFile.seek(seekpos(this.handle)); - if (ohBytes == null) {for (int i = 0; i < OHBYTEC; i++) entryFile.writeByte(0);} + if (ohBytes == null) {for (int i = 0; i < OHBYTEC; i++) entryFile.writeByte(0);} else {for (int i = 0; i < OHBYTEC; i++) entryFile.writeByte(ohBytes[i]);} - if (ohHandle == null) {for (int i = 0; i < OHHANDLEC; i++) entryFile.writeInt(0);} + if (ohHandle == null) {for (int i = 0; i < OHHANDLEC; i++) entryFile.writeInt(0);} else {for (int i = 0; i < OHHANDLEC; i++) entryFile.writeInt(ohHandle[i].index);} long seek = seekpos(this.handle) + overhead; for (int i = 0; i < values.length; i++) { @@ -623,8 +626,12 @@ public class kelondroRecords { private void updateNode() { if (cachesize != 0) { if (!(cache.containsKey(handle))) checkCacheSpace(); - cache.put(handle, this); - cacheScore.setScore(handle, (int) ((System.currentTimeMillis() - startup) / 1000)); + synchronized (cache) { + //System.out.println("updateNode " + this.toString()); + cache.put(handle, this); + cacheScore.setScore(handle, (int) ((System.currentTimeMillis() - startup) / 1000)); + //System.out.println("cache now: " + cache.toString()); + } } } } @@ -844,7 +851,9 @@ public class kelondroRecords { if (index > ((Handle) h).index) return 1; return 0; } - + public int hashCode() { + return this.index; + } } diff --git a/source/de/anomic/kelondro/kelondroTree.java b/source/de/anomic/kelondro/kelondroTree.java index 1144e789a..fc2535cd4 100644 --- a/source/de/anomic/kelondro/kelondroTree.java +++ b/source/de/anomic/kelondro/kelondroTree.java @@ -114,7 +114,8 @@ public class kelondroTree extends kelondroRecords implements Comparator { } // Returns the value to which this map maps the specified key. - public synchronized byte[][] get(byte[] key) throws IOException { + public byte[][] get(byte[] key) throws IOException { + //System.out.println("kelondroTree.get " + new String(key) + " in " + filename); Search search = new Search(key); if (search.found()) { return search.getMatcher().getValues(); @@ -123,7 +124,7 @@ public class kelondroTree extends kelondroRecords implements Comparator { } } - public synchronized long[] getLong(byte[] key) throws IOException { + public long[] getLong(byte[] key) throws IOException { byte[][] row = get(key); long[] longs = new long[columns() - 1]; if (row == null) { @@ -181,6 +182,9 @@ public class kelondroTree extends kelondroRecords implements Comparator { found = false; int c; Handle[] handles; + HashMap visitedNodeKeys = new HashMap(); // to detect loops + String otherkey; + //System.out.println("Starting Compare Loop in Database " + filename); // debug while (thisHandle != null) { try { parentnode = thenode; @@ -190,7 +194,23 @@ public class kelondroTree extends kelondroRecords implements Comparator { found = false; return; } - //System.out.print("Comparing key = '" + new String(key) + "' with '" + new String(thenode.node().getKey()) + "':"); // debug + otherkey = new String(thenode.getKey()); + if (visitedNodeKeys.containsKey(otherkey)) { + // we have loops in the database. + // to fix this, all affected nodes must be patched + thenode.setOHByte(new byte[] {1, 0}); + thenode.setOHHandle(new Handle[] {null, null, null}); + Iterator fix = visitedNodeKeys.entrySet().iterator(); + Map.Entry entry; + while (fix.hasNext()) { + entry = (Map.Entry) fix.next(); + thenode = (Node) entry.getValue(); + thenode.setOHByte(new byte[] {1, 0}); + thenode.setOHHandle(new Handle[] {null, null, null}); + } + throw new kelondroException(filename, "database contains loops; the loop-nodes have been auto-fixed"); + } + //System.out.print("Comparing key = '" + new String(key) + "' with '" + otherkey + "':"); // debug c = compare(key, thenode.getKey()); //System.out.println(c); // debug if (c == 0) { @@ -203,6 +223,7 @@ public class kelondroTree extends kelondroRecords implements Comparator { child = 1; thisHandle = thenode.getOHHandle()[rightchild]; } + visitedNodeKeys.put(otherkey, thenode); } } // we reached a node where we must insert the new value @@ -238,7 +259,7 @@ public class kelondroTree extends kelondroRecords implements Comparator { } } - public synchronized boolean isChild(Node childn, Node parentn, int child) throws IOException { + public boolean isChild(Node childn, Node parentn, int child) throws IOException { if (childn == null) throw new IllegalArgumentException("isLeftChild: Node parameter is NULL"); Handle lc = parentn.getOHHandle()[child]; if (lc == null) return false; @@ -356,7 +377,7 @@ public class kelondroTree extends kelondroRecords implements Comparator { } } - public synchronized long[] putLong(byte[] key, long[] newlongs) throws IOException { + public long[] putLong(byte[] key, long[] newlongs) throws IOException { byte[][] newrow = new byte[newlongs.length + 1][]; newrow[0] = key; for (int i = 0; i < newlongs.length; i++) { @@ -377,7 +398,7 @@ public class kelondroTree extends kelondroRecords implements Comparator { } // Associates the specified value with the specified key in this map - public synchronized byte[][] put(byte[][] newrow) throws IOException { + public byte[][] put(byte[][] newrow) throws IOException { if (newrow.length != columns()) throw new IllegalArgumentException("put: wrong row length " + newrow.length + "; must be " + columns()); // first try to find the key element in the database Search searchResult = new Search(newrow[0]); @@ -625,7 +646,7 @@ public class kelondroTree extends kelondroRecords implements Comparator { } // Associates the specified value with the specified key in this map - public synchronized byte[] put(byte[] key, byte[] value) throws IOException { + public byte[] put(byte[] key, byte[] value) throws IOException { byte[][] row = new byte[2][]; row[0] = key; row[1] = value; @@ -634,7 +655,7 @@ public class kelondroTree extends kelondroRecords implements Comparator { } // Removes the mapping for this key from this map if present (optional operation). - public synchronized byte[][] remove(byte[] key) throws IOException { + public byte[][] remove(byte[] key) throws IOException { Search search = new Search(key); if (search.found()) { Node result = search.getMatcher(); @@ -646,11 +667,11 @@ public class kelondroTree extends kelondroRecords implements Comparator { } } - public synchronized void removeAll() throws IOException { + public void removeAll() throws IOException { while (size() > 0) remove(lastNode(), null); } - public synchronized void remove(Node node, Node parentOfNode) throws IOException { + public void remove(Node node, Node parentOfNode) throws IOException { // there are three cases when removing a node // - the node is a leaf - it can be removed easily // - the node has one child - the child replaces the node @@ -814,50 +835,6 @@ public class kelondroTree extends kelondroRecords implements Comparator { throw new RuntimeException("error creating an iteration: " + e.getMessage()); } } - - /* - public synchronized keyIterator keys(boolean up, boolean rotating) throws IOException { - // iterates only the keys of the Nodes - // enumerated objects are of type byte[] - // iterates the elements in a sorted way. - return new keyIterator(new nodeIterator(up, rotating)); - } - - public synchronized keyIterator keys(boolean up, boolean rotating, byte[] firstKey) throws IOException { - Search s = new Search(firstKey); - if (s.found()) { - return new keyIterator(new nodeIterator(up, rotating, s.getMatcher())); - } else { - Node nn = s.getParent(); - if (nn == null) { - return (keyIterator) (new HashSet()).iterator(); - } else { - return new keyIterator(new nodeIterator(up, rotating, nn)); - } - } - } - - public class keyIterator implements Iterator { - // the iterator iterates all keys, which are byte[] objects - Iterator nodeIterator; - public keyIterator(nodeIterator nodeIterator) { - this.nodeIterator = nodeIterator; - } - public boolean hasNext() { - return nodeIterator.hasNext(); - } - public Object next() { - try { - return ((Node) nodeIterator.next()).getKey(); - } catch (IOException e) { - return null; - } - } - public void remove() { - nodeIterator.remove(); - } - } - */ public synchronized rowIterator rows(boolean up, boolean rotating) throws IOException { // iterates only the keys of the Nodes @@ -907,7 +884,7 @@ public class kelondroTree extends kelondroRecords implements Comparator { } - public synchronized int imp(File file, String separator) throws IOException { + public int imp(File file, String separator) throws IOException { // imports a value-separated file, returns number of records that have been read RandomAccessFile f = new RandomAccessFile(file,"r"); @@ -1162,7 +1139,7 @@ public class kelondroTree extends kelondroRecords implements Comparator { } - public synchronized int compare(Object a, Object b) { + public int compare(Object a, Object b) { try { if ((a instanceof byte[]) && (b instanceof byte[])) { return compare((byte[]) a, (byte[]) b); @@ -1208,8 +1185,8 @@ public class kelondroTree extends kelondroRecords implements Comparator { public static void main(String[] args) { //cmd(args); - bigtest(Integer.parseInt(args[0])); - //randomtest(Integer.parseInt(args[0])); + //bigtest(Integer.parseInt(args[0])); + randomtest(Integer.parseInt(args[0])); //smalltest(); } @@ -1254,7 +1231,7 @@ public class kelondroTree extends kelondroRecords implements Comparator { String s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".substring(0, elements); String t, d; char c; - kelondroTree tt; + kelondroTree tt = null; File testFile = new File("test.db"); byte[] b; try { @@ -1284,6 +1261,7 @@ public class kelondroTree extends kelondroRecords implements Comparator { t = t + c; System.out.println("removed " + new String(b)); } + //tt.print(); if (countElements(tt) != tt.size()) { System.out.println("wrong size for "); tt.print(); @@ -1313,6 +1291,7 @@ public class kelondroTree extends kelondroRecords implements Comparator { } catch (Exception e) { e.printStackTrace(); + try {tt.print();} catch (IOException ee) {} System.out.println("TERMINATED"); } } @@ -1321,7 +1300,7 @@ public class kelondroTree extends kelondroRecords implements Comparator { File f = new File("test.db"); if (f.exists()) f.delete(); try { - kelondroTree tt = new kelondroTree(f, 0, 4, 4); + kelondroTree tt = new kelondroTree(f, 1000, 4, 4); byte[] b; b = testWord('b'); tt.put(b, b); b = testWord('c'); tt.put(b, b); diff --git a/source/de/anomic/plasma/plasmaCrawlNURL.java b/source/de/anomic/plasma/plasmaCrawlNURL.java index 61f5463de..ab6b51e96 100644 --- a/source/de/anomic/plasma/plasmaCrawlNURL.java +++ b/source/de/anomic/plasma/plasmaCrawlNURL.java @@ -296,7 +296,9 @@ public class plasmaCrawlNURL extends plasmaURL { urlHashCache.put(entry); } catch (IOException e) { System.out.println("INTERNAL ERROR AT plasmaNURL:url2hash:" + e.toString()); - } + } catch (kelondroException e) { + serverLog.logError("PLASMA", "plasmaCrawlNURL.store failed: " + e.getMessage()); + } } public String hash() { diff --git a/source/de/anomic/plasma/plasmaHTCache.java b/source/de/anomic/plasma/plasmaHTCache.java index 65dba7449..3b89e105b 100644 --- a/source/de/anomic/plasma/plasmaHTCache.java +++ b/source/de/anomic/plasma/plasmaHTCache.java @@ -199,15 +199,15 @@ public final class plasmaHTCache { return header; } - boolean idle() { + public boolean idle() { return (System.currentTimeMillis() > (idleDelay + lastAcc)); } - boolean full() { + public boolean full() { return (cacheStack.size() > stackLimit); } - boolean empty() { + public boolean empty() { return (cacheStack.size() == 0); } @@ -311,7 +311,7 @@ public final class plasmaHTCache { try { File f; int workoff; - workoff = cacheStack.size() / 10; + workoff = 1 + cacheStack.size() / 10; // we want to work off always 10 % to prevent that we collaps while ((workoff-- > 0) && (!(empty()))) { process((Entry) cacheStack.removeFirst()); @@ -427,8 +427,8 @@ public final class plasmaHTCache { } public static boolean isCGI(String urlString) { - return ((urlString.toLowerCase().indexOf("cgi") >= 0) || - (urlString.toLowerCase().indexOf("exe") >= 0)); + return ((urlString.toLowerCase().indexOf(".cgi") >= 0) || + (urlString.toLowerCase().indexOf(".exe") >= 0)); } public Entry newEntry(Date initDate, int depth, URL url, diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index f6ab7987d..702e376c5 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -148,15 +148,13 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi public HashMap outgoingCookies, incomingCookies; public kelondroTables facilityDB; public plasmaParser parser; - public int serverJobs; private serverSemaphore shutdownSync = new serverSemaphore(0); private boolean terminate = false; public plasmaSwitchboard(String rootPath, String initPath, String configPath) throws IOException { super(rootPath, initPath, configPath); - serverJobs = 0; - + // set loglevel int loglevel = Integer.parseInt(getConfig("plasmaLoglevel", "2")); log = new serverLog("PLASMA", loglevel); @@ -300,10 +298,6 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi return (bytes / 1024) + "TByte"; } - public void handleBusyState(int jobs) { - this.serverJobs = jobs; - } - private void initProfiles() throws IOException { if ((profiles.size() == 0) || (getConfig("defaultProxyProfile", "").length() == 0) || @@ -414,10 +408,10 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi if (processStack.size() == 0) return false; // nothing to do // in case that the server is very busy we do not work off the queue too fast - if (serverJobs > 10) try {Thread.currentThread().sleep(10 * serverJobs);} catch (InterruptedException e) {} + if (!(cacheManager.idle())) try {Thread.currentThread().sleep(1000);} catch (InterruptedException e) {} // do one processing step - log.logDebug("DEQUEUE: serverJobs=" + serverJobs + + log.logDebug("DEQUEUE: cacheManager=" + ((cacheManager.idle()) ? "idle" : "busy") + ", processStack=" + processStack.size() + ", localStackSize=" + noticeURL.localStackSize() + ", remoteStackSize=" + noticeURL.remoteStackSize()); @@ -473,7 +467,7 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi } // if the server is busy, we do crawling more slowly - if (serverJobs > 3) try {Thread.currentThread().sleep(100 * serverJobs);} catch (InterruptedException e) {} + if (!(cacheManager.idle())) try {Thread.currentThread().sleep(2000);} catch (InterruptedException e) {} // do a local crawl (may start a global crawl) plasmaCrawlNURL.entry nex = noticeURL.localPop(); @@ -502,9 +496,8 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi return false; } - // if the server is busy, we do this more slowly - if (serverJobs > 3) try {Thread.currentThread().sleep(100 * serverJobs);} catch (InterruptedException e) {} + if (!(cacheManager.idle())) try {Thread.currentThread().sleep(2000);} catch (InterruptedException e) {} // we don't want to crawl a global URL globally, since WE are the global part. (from this point of view) plasmaCrawlNURL.entry nex = noticeURL.remotePop(); @@ -514,10 +507,7 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi private synchronized void processResourceStack(plasmaHTCache.Entry entry) { // work off one stack entry with a fresh resource (scraped web page) - String stats = "DEQUEUE: dequeueing one step (processStack=" + processStack.size() + ", localStackSize=" + noticeURL.localStackSize() + ", remoteStackSize=" + noticeURL.remoteStackSize() + ")"; - - try { - + try { // we must distinguish the following cases: resource-load was initiated by // 1) global crawling: the index is extern, not here (not possible here) // 2) result of search queries, some indexes are here (not possible here) @@ -540,7 +530,7 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi processCase = 6; } - log.logDebug(stats + " processCase=" + processCase + ", depth=" + entry.depth + ", maxDepth=" + entry.profile.generalDepth() + ", filter=" + entry.profile.generalFilter() + ", initiatorHash=" + initiatorHash + ", status=" + entry.status + ", source=" + ((entry.cacheArray == null) ? "scraper" : "byte[]") + ", url=" + entry.nomalizedURLString); // DEBUG + log.logDebug("processResourceStack processCase=" + processCase + ", depth=" + entry.depth + ", maxDepth=" + entry.profile.generalDepth() + ", filter=" + entry.profile.generalFilter() + ", initiatorHash=" + initiatorHash + ", status=" + entry.status + ", source=" + ((entry.cacheArray == null) ? "scraper" : "byte[]") + ", url=" + entry.nomalizedURLString); // DEBUG // parse content plasmaParserDocument document; diff --git a/source/de/anomic/plasma/plasmaWordIndexFileCache.java b/source/de/anomic/plasma/plasmaWordIndexFileCache.java index c714ce6f7..eefdf8111 100644 --- a/source/de/anomic/plasma/plasmaWordIndexFileCache.java +++ b/source/de/anomic/plasma/plasmaWordIndexFileCache.java @@ -121,9 +121,10 @@ public class plasmaWordIndexFileCache { try { row = indexCache.get(wordHash.getBytes()); } catch (Exception e) { - // we had some negativeSeekOffsetExceptions here; in that case the indexCache is corrupt + // we had some negativeSeekOffsetExceptions here, and also loops may cause this + // in that case the indexCache is corrupt System.out.println("Error in plasmaWordINdexFileCache.getCache: index for hash " + wordHash + " is corrupt:" + e.toString()); - e.printStackTrace(); + //e.printStackTrace(); row = null; } if (row == null) { @@ -159,7 +160,7 @@ public class plasmaWordIndexFileCache { } protected void addEntriesToIndex(String wordHash, Vector /* of plasmaIndexEntry */ newEntries) throws IOException { - //System.out.println("* adding cached word index: " + wordHash + "=" + word + ":" + entry.toEncodedForm()); // debug + //System.out.println("* adding " + newEntries.size() + " cached word index entries for word " + wordHash); // debug // fetch the index cache if (newEntries.size() == 0) return; byte[][] row = getCache(wordHash); diff --git a/source/de/anomic/plasma/plasmaWordIndexRAMCache.java b/source/de/anomic/plasma/plasmaWordIndexRAMCache.java index 77245d015..a9b0efb50 100644 --- a/source/de/anomic/plasma/plasmaWordIndexRAMCache.java +++ b/source/de/anomic/plasma/plasmaWordIndexRAMCache.java @@ -97,7 +97,7 @@ public class plasmaWordIndexRAMCache extends Thread { while (!(terminate)) { if (hashScore.size() < 100) try {Thread.currentThread().sleep(10000);} catch (InterruptedException e) {} while ((!(terminate)) && (cache != null) && (hashScore.size() > 0)) try { - //check = hashScore.size(); + check = hashScore.size(); flushSpecific(false); //serverLog.logDebug("PLASMA INDEXING", "single flush. bevore=" + check + "; after=" + hashScore.size()); try {Thread.currentThread().sleep(10 + ((maxWords / 10) / (1 + hashScore.size())));} catch (InterruptedException e) {} @@ -171,14 +171,12 @@ public class plasmaWordIndexRAMCache extends Thread { return flushKey(key, "flushSpecific"); } - private int flushKey(String key, String caller) throws IOException { + private synchronized int flushKey(String key, String caller) throws IOException { Vector v = null; - synchronized (cache) { - v = (Vector) cache.get(key); - if (v == null) return 0; // flushing of nonexisting key - cache.remove(key); - hashScore.deleteScore(key); - } + v = (Vector) cache.get(key); + if (v == null) return 0; // flushing of nonexisting key + cache.remove(key); + hashScore.deleteScore(key); pic.addEntriesToIndex(key, v); return v.size(); } diff --git a/source/de/anomic/server/serverAbstractSwitch.java b/source/de/anomic/server/serverAbstractSwitch.java index 7530dae24..9a4ee9555 100644 --- a/source/de/anomic/server/serverAbstractSwitch.java +++ b/source/de/anomic/server/serverAbstractSwitch.java @@ -53,6 +53,7 @@ public abstract class serverAbstractSwitch implements serverSwitch { private final Hashtable authorization; private String rootPath; private final TreeMap workerThreads; + protected int serverJobs; public serverAbstractSwitch(String rootPath, String initPath, String configPath) throws IOException { // we initialize the switchboard with a property file, @@ -99,6 +100,9 @@ public abstract class serverAbstractSwitch implements serverSwitch { // init thread control workerThreads = new TreeMap(); + + // init busy state control + serverJobs = 0; } public static Hashtable loadHashtable(File f) { @@ -287,6 +291,6 @@ public abstract class serverAbstractSwitch implements serverSwitch { } public void handleBusyState(int jobs) { - // do nothing here; should be overridden + serverJobs = jobs; } } diff --git a/source/de/anomic/server/serverCore.java b/source/de/anomic/server/serverCore.java index 199672be2..8d8b5a462 100644 --- a/source/de/anomic/server/serverCore.java +++ b/source/de/anomic/server/serverCore.java @@ -76,6 +76,7 @@ public final class serverCore extends serverAbstractThread implements serverThre // static variables public static final Boolean TERMINATE_CONNECTION = Boolean.FALSE; public static final Boolean RESUME_CONNECTION = Boolean.TRUE; + public static Hashtable bfHost = new Hashtable(); // for brute-force prevention // class variables private int port; // the listening port @@ -298,19 +299,19 @@ public final class serverCore extends serverAbstractThread implements serverThre announceThreadBlockApply(); Socket controlSocket = this.socket.accept(); announceThreadBlockRelease(); - if ((this.denyHost == null) || (this.denyHost.get((""+controlSocket.getInetAddress().getHostAddress())) == null)) { - //log.logDebug("* catched request from " + controlSocket.getInetAddress().getHostAddress()); + String clientIP = ""+controlSocket.getInetAddress().getHostAddress(); + if (bfHost.get(clientIP) != null) { + log.logInfo("SLOWING DOWN ACCESS FOR BRUTE-FORCE PREVENTION FROM " + clientIP); + // add a delay to make brute-force harder + try {Thread.currentThread().sleep(1000);} catch (InterruptedException e) {} + } + if ((this.denyHost == null) || (this.denyHost.get(clientIP) == null)) { controlSocket.setSoTimeout(this.timeout); - Session connection = (Session) this.theSessionPool.borrowObject(); connection.execute(controlSocket); - - //try {Thread.currentThread().sleep(1000);} catch (InterruptedException e) {} // wait for debug - // activeThreads.put(connection, new Long(System.currentTimeMillis())); - //log.logDebug("* NEW SESSION: " + connection.request); - + //log.logDebug("* NEW SESSION: " + connection.request + " from " + clientIP); } else { - System.out.println("ACCESS FROM " + controlSocket.getInetAddress().getHostAddress() + " DENIED"); + System.out.println("ACCESS FROM " + clientIP + " DENIED"); } // idle until number of maximal threads is (again) reached //synchronized(this) { @@ -760,9 +761,9 @@ public final class serverCore extends serverAbstractThread implements serverThre } catch (NoSuchMethodException nsme) { System.out.println("ERROR B " + userAddress.getHostAddress()); if (isNotLocal(userAddress.getHostAddress().toString())) { - if (denyHost != null) - denyHost.put((""+userAddress.getHostAddress()), "deny"); // block client: hacker attempt - } + if (denyHost != null) + denyHost.put((""+userAddress.getHostAddress()), "deny"); // block client: hacker attempt + } break; // the client requested a command that does not exist //Object[] errorParameter = { nsme }; diff --git a/source/yacy.java b/source/yacy.java index d44981150..66fb40c6a 100644 --- a/source/yacy.java +++ b/source/yacy.java @@ -78,8 +78,8 @@ import de.anomic.yacy.*; public final class yacy { // static objects - private static final String vString = "0.367"; - private static final String vDATE = "20050426"; + private static final String vString = "@REPL_VERSION@"; + private static final String vDATE = "@REPL_DATE@"; private static final String copyright = "[ YACY Proxy v" + vString + ", build " + vDATE + " by Michael Christen / www.yacy.net ]"; private static final String hline = "-------------------------------------------------------------------------------";