boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

*) setting threadnames for kelondroMap:writequeue and publishSeed

Browse Source 
so that a thread dump is more verbose

*) Moving code for transparent proxy support to a separate function

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@98 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
theli 20 years ago
parent dae6f46812
commit a5fec449c8
6 changed files with 72 additions and 105 deletions
Show Stats Download Patch File Download Diff File

7
htroot/Settings_p.html
Unescape View File

@ -45,9 +45,12 @@ delete the file 'DATA/SETTINGS/httpProxy.conf' in the YaCy application root fold
<i>(hint: choose a name that appears on a web page that tells something about you, vistit the page, get the 'senior' status, and you can be found..)</i></td>
</tr>
<tr valign="top">
<td>Use as Transparent Proxy:</td>
<td>Transparent Proxy:</td>
<td><input type="checkbox" name="isTransparentProxy" align="top" #(isTransparentProxy)#::checked#(/isTransparentProxy)#></td>
<td>With this you can specify if YaCy can be used as transparent proxy.</td>
<td>With this you can specify if YaCy can be used as transparent proxy.<br>
<i>Hint: On linux you can configure your firewall to transparently redirect all http traffic through yacy using this iptables rule:</i><br>
<small><code>iptables -t nat -A PREROUTING -p tcp -s 192.168.0.0/16 --dport 80 -j DNAT --to 192.168.0.1:#[port]#</code></small>
</td>
</tr>
<tr>
<td><input type="submit" name="generalsettings" value="submit"></td>

79
source/de/anomic/http/httpd.java
Unescape View File

@ -201,6 +201,22 @@ public final class httpd implements serverHandler {
return header;
}
private void transparentProxyHandling(httpHeader header) {
if (!(httpdProxyHandler.isTransparentProxy && header.containsKey("HOST"))) return;
String dstHost, dstHostSocket = (String) header.get("HOST");
int idx = dstHostSocket.indexOf(":");
dstHost = (idx != -1) ? dstHostSocket.substring(0,idx).trim() : dstHostSocket.trim();
try {
InetAddress dstHostAddress = InetAddress.getByName(dstHost);
if (!(dstHostAddress.isAnyLocalAddress() || dstHostAddress.isLoopbackAddress())) {
this.prop.setProperty("HOST",dstHostSocket);
}
} catch (Exception e) { }
}
public Boolean GET(String arg) throws IOException {
parseQuery(prop, arg);
prop.setProperty("METHOD", "GET");
@ -213,25 +229,7 @@ public final class httpd implements serverHandler {
header = new httpHeader(reverseMappingCache);
} else {
header = readHeader();
if ((httpdProxyHandler.isTransparentProxy) && header.containsKey("HOST")){
Integer dstPort;
String dstHost = (String) header.get("HOST");
int idx = dstHost.indexOf(":");
if (idx != -1) {
dstPort = Integer.valueOf(dstHost.substring(idx+1));
dstHost = dstHost.substring(0,idx);
} else {
dstPort = new Integer(80);
}
if (dstPort.intValue() == 80) {
InetAddress dstHostAddress = InetAddress.getByName(dstHost);
if (!(dstHostAddress.isAnyLocalAddress() || dstHostAddress.isLoopbackAddress())) {
this.prop.setProperty("HOST",dstHost);
}
}
}
this.transparentProxyHandling(header);
}
// managing keep-alive: in HTTP/0.9 and HTTP/1.0 every connection is closed
@ -314,28 +312,9 @@ public final class httpd implements serverHandler {
String httpVersion = prop.getProperty("HTTP", "HTTP/0.9");
if (httpVersion.equals("HTTP/0.9")) {
header = new httpHeader(reverseMappingCache);
}
else {
} else {
header = readHeader();
if ((httpdProxyHandler.isTransparentProxy) && header.containsKey("HOST")){
Integer dstPort;
String dstHost = (String) header.get("HOST");
int idx = dstHost.indexOf(":");
if (idx != -1) {
dstPort = Integer.valueOf(dstHost.substring(idx+1));
dstHost = dstHost.substring(0,idx);
} else {
dstPort = new Integer(80);
}
if (dstPort.intValue() == 80) {
InetAddress dstHostAddress = InetAddress.getByName(dstHost);
if (!(dstHostAddress.isAnyLocalAddress() || dstHostAddress.isLoopbackAddress())) {
this.prop.setProperty("HOST",dstHost);
}
}
}
this.transparentProxyHandling(header);
}
// return multi-line message
@ -411,25 +390,7 @@ public final class httpd implements serverHandler {
header = new httpHeader(reverseMappingCache);
} else {
header = readHeader();
if ((httpdProxyHandler.isTransparentProxy) && header.containsKey("HOST")){
Integer dstPort;
String dstHost = (String) header.get("HOST");
int idx = dstHost.indexOf(":");
if (idx != -1) {
dstPort = Integer.valueOf(dstHost.substring(idx+1));
dstHost = dstHost.substring(0,idx);
} else {
dstPort = new Integer(80);
}
if (dstPort.intValue() == 80) {
InetAddress dstHostAddress = InetAddress.getByName(dstHost);
if (!(dstHostAddress.isAnyLocalAddress() || dstHostAddress.isLoopbackAddress())) {
this.prop.setProperty("HOST",dstHost);
}
}
}
this.transparentProxyHandling(header);
}
boolean persistent = (!((httpVersion.equals("HTTP/0.9")) || (httpVersion.equals("HTTP/1.0"))));

1
source/de/anomic/kelondro/kelondroMap.java
Unescape View File

@ -136,6 +136,7 @@ public class kelondroMap {
boolean run;
public writeQueue() {
super("kelondroMap:WriteQueue");
run = true;
}

1
source/de/anomic/yacy/yacyCore.java
Unescape View File

@ -270,6 +270,7 @@ public class yacyCore {
public Exception error;
public publishThread(yacySeed seed) {
super("PublishSeed_" + seed.getName());
this.seed = seed;
this.added = 0;
this.error = null;

88
source/de/anomic/yacy/yacySearch.java
Unescape View File

@ -61,13 +61,13 @@ public class yacySearch extends Thread {
public yacySearch(Set wordhashes, int count, boolean global, yacySeed targetPeer,
plasmaCrawlLURL urlManager, plasmaSearch searchManager, long duetime) {
this.wordhashes = wordhashes;
this.count = count;
this.global = global;
this.urlManager = urlManager;
this.searchManager = searchManager;
this.targetPeer = targetPeer;
this.links = -1;
this.wordhashes = wordhashes;
this.count = count;
this.global = global;
this.urlManager = urlManager;
this.searchManager = searchManager;
this.targetPeer = targetPeer;
this.links = -1;
this.duetime = duetime;
}
@ -75,16 +75,16 @@ public class yacySearch extends Thread {
String wh = "";
Iterator i = wordhashes.iterator();
while (i.hasNext()) wh = wh + (String) i.next();
this.links = yacyClient.search(wh, count, global, targetPeer, urlManager, searchManager, duetime);
this.links = yacyClient.search(wh, count, global, targetPeer, urlManager, searchManager, duetime);
if (links != 0) {
//yacyCore.log.logInfo("REMOTE SEARCH - remote peer '" + targetPeer.get("Name", "anonymous") + "' contributed " + links + " links for word hash " + wordhashes);
yacyCore.seedDB.mySeed.incRI(links);
yacyCore.seedDB.mySeed.incRU(links);
}
}
public int links() {
return this.links;
return this.links;
}
private static yacySeed[] selectPeers(Set wordhashes, int seedcount) {
@ -121,57 +121,57 @@ public class yacySearch extends Thread {
public static int search(Set querywords, plasmaCrawlLURL urlManager, plasmaSearch searchManager,
int count, int targets, long waitingtime) {
// check own peer status
if ((yacyCore.seedDB.mySeed == null) || (yacyCore.seedDB.mySeed.getAddress() == null)) return 0;
// start delay control
long start = System.currentTimeMillis();
// check own peer status
if ((yacyCore.seedDB.mySeed == null) || (yacyCore.seedDB.mySeed.getAddress() == null)) return 0;
// start delay control
long start = System.currentTimeMillis();
// set a duetime for clients
long duetime = waitingtime - 4000; // subtract network traffic overhead, guessed 4 seconds
if (duetime < 1000) duetime = 1000;
// prepare seed targets and threads
// prepare seed targets and threads
Set wordhashes = plasmaSearch.words2hashes(querywords);
yacySeed[] targetPeers = selectPeers(wordhashes, targets);
if (targetPeers == null) return 0;
targets = targetPeers.length;
if (targets == 0) return 0;
yacySearch[] searchThreads = new yacySearch[targets];
for (int i = 0; i < targets; i++) {
searchThreads[i]= new yacySearch(wordhashes, count, true, targetPeers[i],
urlManager, searchManager, duetime);
searchThreads[i].start();
try {Thread.currentThread().sleep(20);} catch (InterruptedException e) {}
yacySeed[] targetPeers = selectPeers(wordhashes, targets);
if (targetPeers == null) return 0;
targets = targetPeers.length;
if (targets == 0) return 0;
yacySearch[] searchThreads = new yacySearch[targets];
for (int i = 0; i < targets; i++) {
searchThreads[i]= new yacySearch(wordhashes, count, true, targetPeers[i],
urlManager, searchManager, duetime);
searchThreads[i].start();
try {Thread.currentThread().sleep(20);} catch (InterruptedException e) {}
if ((System.currentTimeMillis() - start) > waitingtime) {
targets = i + 1;
break;
}
}
}
int c;
// wait until wanted delay passed or wanted result appeared
boolean anyIdle = true;
while ((anyIdle) && ((System.currentTimeMillis() - start) < waitingtime)) {
// wait..
try {Thread.currentThread().sleep(200);} catch (InterruptedException e) {}
// check if all threads have been finished or results so far are enough
c = 0;
anyIdle = false;
for (int i = 0; i < targets; i++) {
if (searchThreads[i].links() < 0) anyIdle = true; else c = c + searchThreads[i].links();
}
// wait until wanted delay passed or wanted result appeared
boolean anyIdle = true;
while ((anyIdle) && ((System.currentTimeMillis() - start) < waitingtime)) {
// wait..
try {Thread.currentThread().sleep(200);} catch (InterruptedException e) {}
// check if all threads have been finished or results so far are enough
c = 0;
anyIdle = false;
for (int i = 0; i < targets; i++) {
if (searchThreads[i].links() < 0) anyIdle = true; else c = c + searchThreads[i].links();
}
if ((c >= count * 3) && ((System.currentTimeMillis() - start) > (waitingtime * 2 / 3))) {
System.out.println("DEBUG yacySearch: c=" + c + ", count=" + count + ", waitingtime=" + waitingtime);
break; // we have enough
}
if (c >= count * 5) break;
}
}
// collect results
c = 0;
for (int i = 0; i < targets; i++) c = c + ((searchThreads[i].links() > 0) ? searchThreads[i].links() : 0);
return c;
// collect results
c = 0;
for (int i = 0; i < targets; i++) c = c + ((searchThreads[i].links() > 0) ? searchThreads[i].links() : 0);
return c;
}
}

1
source/yacy.java
Unescape View File

@ -247,6 +247,7 @@ public final class yacy {
sb,
30000 /*command max length incl. GET args*/,
httpdLoglevel /*loglevel*/);
server.setName("httpd:"+port);
if (server == null) {
serverLog.logFailure("STARTUP", "Failed to start server. Probably port " + port + " already in use.");
} else {

Write Preview
Loading…
Cancel
Save