From fd7c17e62429e5672f205127803451eae6e980a0 Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 9 May 2006 13:11:00 +0000 Subject: [PATCH] added virtual host support: all yacy-to-yacy communication now send the .yacyh virtual domain inside the http 'Host' property field. This shall enable running a yacy peer on a virtual host. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2074 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/ConfigLanguage_p.java | 3 ++- htroot/ConfigSkins_p.java | 3 ++- htroot/sharedBlacklist_p.java | 6 +++-- source/de/anomic/http/httpc.java | 27 ++++++++++++++------- source/de/anomic/net/natLib.java | 8 +++---- source/de/anomic/tools/loaderThreads.java | 2 +- source/de/anomic/yacy/yacyClient.java | 28 +++++++++++++++++----- source/de/anomic/yacy/yacyPeerActions.java | 5 ++-- source/de/anomic/yacy/yacySeedDB.java | 3 ++- source/yacy.java | 3 ++- 10 files changed, 60 insertions(+), 28 deletions(-) diff --git a/htroot/ConfigLanguage_p.java b/htroot/ConfigLanguage_p.java index eff1be5b9..7ec9d1785 100644 --- a/htroot/ConfigLanguage_p.java +++ b/htroot/ConfigLanguage_p.java @@ -96,7 +96,8 @@ public class ConfigLanguage_p { String url = (String)post.get("url"); ArrayList langVector; try{ - langVector = httpc.wget(new URL(url), 6000, null, null, switchboard.remoteProxyConfig); + URL u = new URL(url); + langVector = httpc.wget(u, u.getHost(), 6000, null, null, switchboard.remoteProxyConfig); }catch(IOException e){ prop.put("status", 1);//unable to get url prop.put("status_url", url); diff --git a/htroot/ConfigSkins_p.java b/htroot/ConfigSkins_p.java index 98b5da4e8..7545cc63c 100644 --- a/htroot/ConfigSkins_p.java +++ b/htroot/ConfigSkins_p.java @@ -125,7 +125,8 @@ public class ConfigSkins_p { String url = (String)post.get("url"); ArrayList skinVector; try{ - skinVector = httpc.wget(new URL(url), 6000, null, null, switchboard.remoteProxyConfig); + URL u = new URL(url); + skinVector = httpc.wget(u, u.getHost(), 6000, null, null, switchboard.remoteProxyConfig); }catch(IOException e){ prop.put("status", 1);//unable to get URL prop.put("status_url", url); diff --git a/htroot/sharedBlacklist_p.java b/htroot/sharedBlacklist_p.java index dbf180954..0b1ade3b0 100644 --- a/htroot/sharedBlacklist_p.java +++ b/htroot/sharedBlacklist_p.java @@ -147,7 +147,8 @@ public class sharedBlacklist_p { reqHeader.put(httpHeader.CACHE_CONTROL,"no-cache"); // get List - otherBlacklist = httpc.wget(new URL(address), 12000, null, null, switchboard.remoteProxyConfig,reqHeader); + URL u = new URL(address); + otherBlacklist = httpc.wget(u, u.getHost(), 12000, null, null, switchboard.remoteProxyConfig,reqHeader); } catch (Exception e) {} //Make HTML-Optionlist with retrieved items @@ -178,7 +179,8 @@ public class sharedBlacklist_p { Name = address; try { - otherBlacklist = httpc.wget(new URL(address), 6000, null, null, switchboard.remoteProxyConfig); //get List + URL u = new URL(address); + otherBlacklist = httpc.wget(u, u.getHost(), 6000, null, null, switchboard.remoteProxyConfig); //get List } catch (Exception e) {} prop.put("status", 0); //TODO: check if the wget failed... diff --git a/source/de/anomic/http/httpc.java b/source/de/anomic/http/httpc.java index 8235fad33..a74663cec 100644 --- a/source/de/anomic/http/httpc.java +++ b/source/de/anomic/http/httpc.java @@ -1152,6 +1152,7 @@ do upload public static byte[] singlePOST( URL u, + String host, int timeout, String user, String password, @@ -1186,8 +1187,10 @@ do upload serverObjects props ) throws IOException { try { + URL u = new URL(url); return singlePOST( - new URL(url), + u, + u.getHost(), timeout, null, null, @@ -1201,20 +1204,23 @@ do upload } public static ArrayList wget( - URL url, + URL url, + String host, int timeout, String user, String password, httpRemoteProxyConfig theRemoteProxyConfig ) throws IOException { - return wget(url,timeout,user,password,theRemoteProxyConfig,null); + return wget(url, host,timeout,user,password,theRemoteProxyConfig,null); } + public static ArrayList wget(URL url) throws IOException{ - return wget(url, 6000, null, null, plasmaSwitchboard.getSwitchboard().remoteProxyConfig); + return wget(url, url.getHost(), 6000, null, null, plasmaSwitchboard.getSwitchboard().remoteProxyConfig, null); } public static ArrayList wget( - URL url, + URL url, + String host, int timeout, String user, String password, @@ -1231,7 +1237,7 @@ do upload // splitting of the byte array into lines byte[] a = singleGET( - url.getHost(), + host, port, path, timeout, @@ -1346,7 +1352,8 @@ do upload */ public static ArrayList wput( - URL url, + URL url, + String host, int timeout, String user, String password, @@ -1356,7 +1363,8 @@ do upload ) throws IOException { // splitting of the byte array into lines byte[] a = singlePOST( - url, + url, + host, timeout, user, password, @@ -1402,7 +1410,8 @@ do upload httpRemoteProxyConfig theRemoteProxyConfig = httpRemoteProxyConfig.init(proxyHost,proxyPort); try { - text = wget(new URL(url), timeout, null, null, theRemoteProxyConfig); + URL u = new URL(url); + text = wget(u, u.getHost(), timeout, null, null, theRemoteProxyConfig); } catch (MalformedURLException e) { System.out.println("The url '" + url + "' is wrong."); } catch (IOException e) { diff --git a/source/de/anomic/net/natLib.java b/source/de/anomic/net/natLib.java index 7a72dceff..470ff3eb7 100644 --- a/source/de/anomic/net/natLib.java +++ b/source/de/anomic/net/natLib.java @@ -60,7 +60,7 @@ public class natLib { rm status.htm */ try { - ArrayList x = httpc.wget(new URL("http://192.168.0.1:80/status.htm"), 5000, "admin", password, null); + ArrayList x = httpc.wget(new URL("http://192.168.0.1:80/status.htm"), "192.168.0.1", 5000, "admin", password, null); x = nxTools.grep(x, 1, "IP Address"); if ((x == null) || (x.size() == 0)) return null; String line = nxTools.tail1(x); @@ -72,7 +72,7 @@ public class natLib { private static String getWhatIsMyIP() { try { - ArrayList x = httpc.wget(new URL("http://www.whatismyip.com/"), 5000, null, null, null); + ArrayList x = httpc.wget(new URL("http://www.whatismyip.com/"), "whatsmyip.com", 5000, null, null, null); x = nxTools.grep(x, 0, "Your IP is"); String line = nxTools.tail1(x); return nxTools.awk(line, " ", 4); @@ -83,7 +83,7 @@ public class natLib { private static String getStanford() { try { - ArrayList x = httpc.wget(new URL("http://www.slac.stanford.edu/cgi-bin/nph-traceroute.pl"), 5000, null, null, null); + ArrayList x = httpc.wget(new URL("http://www.slac.stanford.edu/cgi-bin/nph-traceroute.pl"), "stanford.edu", 5000, null, null, null); x = nxTools.grep(x, 0, "firewall protecting your browser"); String line = nxTools.tail1(x); return nxTools.awk(line, " ", 7); @@ -94,7 +94,7 @@ public class natLib { private static String getIPID() { try { - ArrayList x = httpc.wget(new URL("http://ipid.shat.net/"), 5000, null, null, null); + ArrayList x = httpc.wget(new URL("http://ipid.shat.net/"), "shat.net", 5000, null, null, null); x = nxTools.grep(x, 2, "Your IP address"); String line = nxTools.tail1(x); return nxTools.awk(nxTools.awk(nxTools.awk(line, " ", 5), ">", 2), "<", 1); diff --git a/source/de/anomic/tools/loaderThreads.java b/source/de/anomic/tools/loaderThreads.java index 7fc9f4891..f55ebe90a 100644 --- a/source/de/anomic/tools/loaderThreads.java +++ b/source/de/anomic/tools/loaderThreads.java @@ -146,7 +146,7 @@ public class loaderThreads { public void run() { try { - page = httpc.wget(url, timeout, user, password, remoteProxyConfig); + page = httpc.wget(url, url.getHost(), timeout, user, password, remoteProxyConfig); loaded = true; process.feed(page); if (process.status() == loaderCore.STATUS_FAILED) { diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index 5ba9633ed..f683af95c 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -124,7 +124,8 @@ public final class yacyClient { // sending request result = nxTools.table( - httpc.wput(url, + httpc.wput(url, + yacyCore.seedDB.mySeed.getHexHash() + ".yacyh", 105000, null, null, @@ -257,10 +258,12 @@ public final class yacyClient { "&object=seed" + "&env=" + seedHash ), + yacyCore.seedDB.mySeed.getHexHash() + ".yacyh", 10000, null, null, - (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null + (useProxy)?yacyCore.seedDB.sb.remoteProxyConfig:null, + null ) ); @@ -292,6 +295,7 @@ public final class yacyClient { "&env=" + wordHash + "&ttl=0" ), + yacyCore.seedDB.mySeed.getHexHash() + ".yacyh", 10000, null, null, @@ -331,7 +335,8 @@ public final class yacyClient { try { final HashMap result = nxTools.table( httpc.wget( - new URL(querystr), + new URL(querystr), + yacyCore.seedDB.mySeed.getHexHash() + ".yacyh", 6000, null, null, @@ -422,6 +427,7 @@ public final class yacyClient { final HashMap result = nxTools.table( httpc.wput( new URL(url), + yacyCore.seedDB.mySeed.getHexHash() + ".yacyh", 300000, null, null, @@ -556,6 +562,7 @@ public final class yacyClient { return nxTools.table( httpc.wput( new URL("http://" + address + "/yacy/message.html"), + yacyCore.seedDB.mySeed.getHexHash() + ".yacyh", 8000, null, null, @@ -599,7 +606,8 @@ public final class yacyClient { // sending request try { final ArrayList v = httpc.wput( - new URL("http://" + address + "/yacy/message.html"), + new URL("http://" + address + "/yacy/message.html"), + yacyCore.seedDB.mySeed.getHexHash() + ".yacyh", 20000, null, null, @@ -653,6 +661,7 @@ public final class yacyClient { return nxTools.table( httpc.wput( new URL("http://" + targetAddress + "/yacy/transfer.html"), + yacyCore.seedDB.mySeed.getHexHash() + ".yacyh", 8000, null, null, @@ -692,7 +701,8 @@ public final class yacyClient { // sending request try { final ArrayList v = httpc.wput( - new URL("http://" + targetAddress + "/yacy/transfer.html"), + new URL("http://" + targetAddress + "/yacy/transfer.html"), + yacyCore.seedDB.mySeed.getHexHash() + ".yacyh", 20000, null, null, @@ -777,6 +787,7 @@ public final class yacyClient { return nxTools.table( httpc.wput( new URL("http://" + address + "/yacy/crawlOrder.html"), + yacyCore.seedDB.mySeed.getHexHash() + ".yacyh", timeout, null, null, @@ -851,6 +862,7 @@ public final class yacyClient { "&wordh=" + wordhashes + "&lurlEntry=" + ((entry == null) ? "" : crypt.simpleEncode(entry.toString(), key)) ), + yacyCore.seedDB.mySeed.getHexHash() + ".yacyh", 60000, null, null, @@ -965,6 +977,7 @@ public final class yacyClient { try { final ArrayList v = httpc.wput( new URL("http://" + address + "/yacy/transferRWI.html"), + yacyCore.seedDB.mySeed.getHexHash() + ".yacyh", timeout, null, null, @@ -1021,7 +1034,8 @@ public final class yacyClient { post.put("urlc", Integer.toString(urlc)); try { final ArrayList v = httpc.wput( - new URL("http://" + address + "/yacy/transferURL.html"), + new URL("http://" + address + "/yacy/transferURL.html"), + yacyCore.seedDB.mySeed.getHexHash() + ".yacyh", timeout, null, null, @@ -1056,6 +1070,7 @@ public final class yacyClient { try { final ArrayList v = httpc.wput( new URL("http://" + address + "/yacy/profile.html"), + yacyCore.seedDB.mySeed.getHexHash() + ".yacyh", 20000, null, null, @@ -1095,6 +1110,7 @@ public final class yacyClient { "&count=10" + "&resource=global" + "&query=" + wordhashe), + yacyCore.seedDB.mySeed.getHexHash() + ".yacyh", 5000, null, null, diff --git a/source/de/anomic/yacy/yacyPeerActions.java b/source/de/anomic/yacy/yacyPeerActions.java index 491ba7667..06e6f8af0 100644 --- a/source/de/anomic/yacy/yacyPeerActions.java +++ b/source/de/anomic/yacy/yacyPeerActions.java @@ -190,7 +190,7 @@ public class yacyPeerActions { yacyCore.log.logInfo("BOOTSTRAP: seed-list URL " + seedListFileURL + " too old (" + (header.age() / 86400000) + " days)"); } else { ssc++; - seedList = httpc.wget(url, this.bootstrapLoadTimeout, null, null, this.sb.remoteProxyConfig,reqHeader); + seedList = httpc.wget(url, url.getHost(), this.bootstrapLoadTimeout, null, null, this.sb.remoteProxyConfig,reqHeader); enu = seedList.iterator(); lc = 0; while (enu.hasNext()) { @@ -243,7 +243,8 @@ public class yacyPeerActions { // read in remote file from url try { - ArrayList remote = httpc.wget(new URL(url), 5000, null, null, this.sb.remoteProxyConfig); + URL u = new URL(url); + ArrayList remote = httpc.wget(u, u.getHost(), 5000, null, null, this.sb.remoteProxyConfig); if ((remote != null) && (remote.size() > 0)) { Iterator e = remote.iterator(); while (e.hasNext()) { diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index 4c79c5882..90731c04a 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -703,7 +703,8 @@ public final class yacySeedDB { reqHeader.put(httpHeader.PRAGMA, "no-cache"); reqHeader.put(httpHeader.CACHE_CONTROL, "no-cache"); // httpc uses HTTP/1.0 is this necessary? ArrayList check = httpc.wget( - seedURL, + seedURL, + seedURL.getHost(), 10000, null, null, diff --git a/source/yacy.java b/source/yacy.java index af1eb07a5..faafc91fa 100644 --- a/source/yacy.java +++ b/source/yacy.java @@ -459,7 +459,8 @@ public final class yacy { server.terminate(false); server.interrupt(); if (server.isAlive()) try { - httpc.wget(new URL("http://localhost:" + serverCore.getPortNr(port)), 1000, null, null, null); // kick server + URL u = new URL("http://localhost:" + serverCore.getPortNr(port)); + httpc.wget(u, u.getHost(), 1000, null, null, null); // kick server serverLog.logConfig("SHUTDOWN", "sent termination signal to server socket"); } catch (IOException ee) { serverLog.logConfig("SHUTDOWN", "termination signal to server socket missed (server shutdown, ok)");