From a5fec449c8c8d338251c581dbb5ebe07f31d2d9a Mon Sep 17 00:00:00 2001 From: theli Date: Mon, 9 May 2005 08:53:39 +0000 Subject: [PATCH] *) setting threadnames for kelondroMap:writequeue and publishSeed so that a thread dump is more verbose *) Moving code for transparent proxy support to a separate function git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@98 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/Settings_p.html | 7 +- source/de/anomic/http/httpd.java | 79 +++++-------------- source/de/anomic/kelondro/kelondroMap.java | 1 + source/de/anomic/yacy/yacyCore.java | 1 + source/de/anomic/yacy/yacySearch.java | 88 +++++++++++----------- source/yacy.java | 1 + 6 files changed, 72 insertions(+), 105 deletions(-) diff --git a/htroot/Settings_p.html b/htroot/Settings_p.html index d6d42ae3c..e195ba8d8 100644 --- a/htroot/Settings_p.html +++ b/htroot/Settings_p.html @@ -45,9 +45,12 @@ delete the file 'DATA/SETTINGS/httpProxy.conf' in the YaCy application root fold (hint: choose a name that appears on a web page that tells something about you, vistit the page, get the 'senior' status, and you can be found..) - Use as Transparent Proxy: + Transparent Proxy: - With this you can specify if YaCy can be used as transparent proxy. + With this you can specify if YaCy can be used as transparent proxy.
+ Hint: On linux you can configure your firewall to transparently redirect all http traffic through yacy using this iptables rule:
+ iptables -t nat -A PREROUTING -p tcp -s 192.168.0.0/16 --dport 80 -j DNAT --to 192.168.0.1:#[port]# + diff --git a/source/de/anomic/http/httpd.java b/source/de/anomic/http/httpd.java index 779f0b964..7cd4bc2ad 100644 --- a/source/de/anomic/http/httpd.java +++ b/source/de/anomic/http/httpd.java @@ -201,6 +201,22 @@ public final class httpd implements serverHandler { return header; } + private void transparentProxyHandling(httpHeader header) { + if (!(httpdProxyHandler.isTransparentProxy && header.containsKey("HOST"))) return; + + String dstHost, dstHostSocket = (String) header.get("HOST"); + + int idx = dstHostSocket.indexOf(":"); + dstHost = (idx != -1) ? dstHostSocket.substring(0,idx).trim() : dstHostSocket.trim(); + + try { + InetAddress dstHostAddress = InetAddress.getByName(dstHost); + if (!(dstHostAddress.isAnyLocalAddress() || dstHostAddress.isLoopbackAddress())) { + this.prop.setProperty("HOST",dstHostSocket); + } + } catch (Exception e) { } + } + public Boolean GET(String arg) throws IOException { parseQuery(prop, arg); prop.setProperty("METHOD", "GET"); @@ -213,25 +229,7 @@ public final class httpd implements serverHandler { header = new httpHeader(reverseMappingCache); } else { header = readHeader(); - if ((httpdProxyHandler.isTransparentProxy) && header.containsKey("HOST")){ - Integer dstPort; - String dstHost = (String) header.get("HOST"); - - int idx = dstHost.indexOf(":"); - if (idx != -1) { - dstPort = Integer.valueOf(dstHost.substring(idx+1)); - dstHost = dstHost.substring(0,idx); - } else { - dstPort = new Integer(80); - } - - if (dstPort.intValue() == 80) { - InetAddress dstHostAddress = InetAddress.getByName(dstHost); - if (!(dstHostAddress.isAnyLocalAddress() || dstHostAddress.isLoopbackAddress())) { - this.prop.setProperty("HOST",dstHost); - } - } - } + this.transparentProxyHandling(header); } // managing keep-alive: in HTTP/0.9 and HTTP/1.0 every connection is closed @@ -314,28 +312,9 @@ public final class httpd implements serverHandler { String httpVersion = prop.getProperty("HTTP", "HTTP/0.9"); if (httpVersion.equals("HTTP/0.9")) { header = new httpHeader(reverseMappingCache); - } - else { + } else { header = readHeader(); - if ((httpdProxyHandler.isTransparentProxy) && header.containsKey("HOST")){ - Integer dstPort; - String dstHost = (String) header.get("HOST"); - - int idx = dstHost.indexOf(":"); - if (idx != -1) { - dstPort = Integer.valueOf(dstHost.substring(idx+1)); - dstHost = dstHost.substring(0,idx); - } else { - dstPort = new Integer(80); - } - - if (dstPort.intValue() == 80) { - InetAddress dstHostAddress = InetAddress.getByName(dstHost); - if (!(dstHostAddress.isAnyLocalAddress() || dstHostAddress.isLoopbackAddress())) { - this.prop.setProperty("HOST",dstHost); - } - } - } + this.transparentProxyHandling(header); } // return multi-line message @@ -411,25 +390,7 @@ public final class httpd implements serverHandler { header = new httpHeader(reverseMappingCache); } else { header = readHeader(); - if ((httpdProxyHandler.isTransparentProxy) && header.containsKey("HOST")){ - Integer dstPort; - String dstHost = (String) header.get("HOST"); - - int idx = dstHost.indexOf(":"); - if (idx != -1) { - dstPort = Integer.valueOf(dstHost.substring(idx+1)); - dstHost = dstHost.substring(0,idx); - } else { - dstPort = new Integer(80); - } - - if (dstPort.intValue() == 80) { - InetAddress dstHostAddress = InetAddress.getByName(dstHost); - if (!(dstHostAddress.isAnyLocalAddress() || dstHostAddress.isLoopbackAddress())) { - this.prop.setProperty("HOST",dstHost); - } - } - } + this.transparentProxyHandling(header); } boolean persistent = (!((httpVersion.equals("HTTP/0.9")) || (httpVersion.equals("HTTP/1.0")))); diff --git a/source/de/anomic/kelondro/kelondroMap.java b/source/de/anomic/kelondro/kelondroMap.java index 9b3c1bfb0..fc8288a4a 100644 --- a/source/de/anomic/kelondro/kelondroMap.java +++ b/source/de/anomic/kelondro/kelondroMap.java @@ -136,6 +136,7 @@ public class kelondroMap { boolean run; public writeQueue() { + super("kelondroMap:WriteQueue"); run = true; } diff --git a/source/de/anomic/yacy/yacyCore.java b/source/de/anomic/yacy/yacyCore.java index afdecfd8b..dea6cab03 100644 --- a/source/de/anomic/yacy/yacyCore.java +++ b/source/de/anomic/yacy/yacyCore.java @@ -270,6 +270,7 @@ public class yacyCore { public Exception error; public publishThread(yacySeed seed) { + super("PublishSeed_" + seed.getName()); this.seed = seed; this.added = 0; this.error = null; diff --git a/source/de/anomic/yacy/yacySearch.java b/source/de/anomic/yacy/yacySearch.java index 0fad12a65..d4f005aa5 100644 --- a/source/de/anomic/yacy/yacySearch.java +++ b/source/de/anomic/yacy/yacySearch.java @@ -61,13 +61,13 @@ public class yacySearch extends Thread { public yacySearch(Set wordhashes, int count, boolean global, yacySeed targetPeer, plasmaCrawlLURL urlManager, plasmaSearch searchManager, long duetime) { - this.wordhashes = wordhashes; - this.count = count; - this.global = global; - this.urlManager = urlManager; - this.searchManager = searchManager; - this.targetPeer = targetPeer; - this.links = -1; + this.wordhashes = wordhashes; + this.count = count; + this.global = global; + this.urlManager = urlManager; + this.searchManager = searchManager; + this.targetPeer = targetPeer; + this.links = -1; this.duetime = duetime; } @@ -75,16 +75,16 @@ public class yacySearch extends Thread { String wh = ""; Iterator i = wordhashes.iterator(); while (i.hasNext()) wh = wh + (String) i.next(); - this.links = yacyClient.search(wh, count, global, targetPeer, urlManager, searchManager, duetime); + this.links = yacyClient.search(wh, count, global, targetPeer, urlManager, searchManager, duetime); if (links != 0) { //yacyCore.log.logInfo("REMOTE SEARCH - remote peer '" + targetPeer.get("Name", "anonymous") + "' contributed " + links + " links for word hash " + wordhashes); yacyCore.seedDB.mySeed.incRI(links); yacyCore.seedDB.mySeed.incRU(links); } } - + public int links() { - return this.links; + return this.links; } private static yacySeed[] selectPeers(Set wordhashes, int seedcount) { @@ -121,57 +121,57 @@ public class yacySearch extends Thread { public static int search(Set querywords, plasmaCrawlLURL urlManager, plasmaSearch searchManager, int count, int targets, long waitingtime) { - // check own peer status - if ((yacyCore.seedDB.mySeed == null) || (yacyCore.seedDB.mySeed.getAddress() == null)) return 0; - - // start delay control - long start = System.currentTimeMillis(); - + // check own peer status + if ((yacyCore.seedDB.mySeed == null) || (yacyCore.seedDB.mySeed.getAddress() == null)) return 0; + + // start delay control + long start = System.currentTimeMillis(); + // set a duetime for clients long duetime = waitingtime - 4000; // subtract network traffic overhead, guessed 4 seconds if (duetime < 1000) duetime = 1000; - // prepare seed targets and threads + // prepare seed targets and threads Set wordhashes = plasmaSearch.words2hashes(querywords); - yacySeed[] targetPeers = selectPeers(wordhashes, targets); - if (targetPeers == null) return 0; - targets = targetPeers.length; - if (targets == 0) return 0; - yacySearch[] searchThreads = new yacySearch[targets]; - for (int i = 0; i < targets; i++) { - searchThreads[i]= new yacySearch(wordhashes, count, true, targetPeers[i], - urlManager, searchManager, duetime); - searchThreads[i].start(); - try {Thread.currentThread().sleep(20);} catch (InterruptedException e) {} + yacySeed[] targetPeers = selectPeers(wordhashes, targets); + if (targetPeers == null) return 0; + targets = targetPeers.length; + if (targets == 0) return 0; + yacySearch[] searchThreads = new yacySearch[targets]; + for (int i = 0; i < targets; i++) { + searchThreads[i]= new yacySearch(wordhashes, count, true, targetPeers[i], + urlManager, searchManager, duetime); + searchThreads[i].start(); + try {Thread.currentThread().sleep(20);} catch (InterruptedException e) {} if ((System.currentTimeMillis() - start) > waitingtime) { targets = i + 1; break; } - } + } int c; - // wait until wanted delay passed or wanted result appeared - boolean anyIdle = true; - while ((anyIdle) && ((System.currentTimeMillis() - start) < waitingtime)) { - // wait.. - try {Thread.currentThread().sleep(200);} catch (InterruptedException e) {} - // check if all threads have been finished or results so far are enough - c = 0; - anyIdle = false; - for (int i = 0; i < targets; i++) { - if (searchThreads[i].links() < 0) anyIdle = true; else c = c + searchThreads[i].links(); - } + // wait until wanted delay passed or wanted result appeared + boolean anyIdle = true; + while ((anyIdle) && ((System.currentTimeMillis() - start) < waitingtime)) { + // wait.. + try {Thread.currentThread().sleep(200);} catch (InterruptedException e) {} + // check if all threads have been finished or results so far are enough + c = 0; + anyIdle = false; + for (int i = 0; i < targets; i++) { + if (searchThreads[i].links() < 0) anyIdle = true; else c = c + searchThreads[i].links(); + } if ((c >= count * 3) && ((System.currentTimeMillis() - start) > (waitingtime * 2 / 3))) { System.out.println("DEBUG yacySearch: c=" + c + ", count=" + count + ", waitingtime=" + waitingtime); break; // we have enough } if (c >= count * 5) break; - } + } - // collect results - c = 0; - for (int i = 0; i < targets; i++) c = c + ((searchThreads[i].links() > 0) ? searchThreads[i].links() : 0); - return c; + // collect results + c = 0; + for (int i = 0; i < targets; i++) c = c + ((searchThreads[i].links() > 0) ? searchThreads[i].links() : 0); + return c; } } diff --git a/source/yacy.java b/source/yacy.java index 6eb6f59f2..d3265b287 100644 --- a/source/yacy.java +++ b/source/yacy.java @@ -247,6 +247,7 @@ public final class yacy { sb, 30000 /*command max length incl. GET args*/, httpdLoglevel /*loglevel*/); + server.setName("httpd:"+port); if (server == null) { serverLog.logFailure("STARTUP", "Failed to start server. Probably port " + port + " already in use."); } else {