diff --git a/defaults/yacy.init b/defaults/yacy.init index 31196f8ed..1d8c67449 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -127,6 +127,10 @@ network.unit.domain.nocheck = false # that means it is not usable in YaCy p2p-configurations, only in private portal configurations network.unit.tenant.agent = +# Prefer https for in-protocol operations when available on remote peers +# A distinct general setting is available to control whether https sould be used for remote search queries : remotesearch.https.preferred +network.unit.protocol.https.preferred = false + # Update process properties # The update server location is given in the network.unit.definition, # but the settings for update processing and cycles are individual. diff --git a/htroot/MessageSend_p.java b/htroot/MessageSend_p.java index 52870ae78..57271bc00 100644 --- a/htroot/MessageSend_p.java +++ b/htroot/MessageSend_p.java @@ -26,6 +26,7 @@ //if the shell's current path is HTROOT import java.io.IOException; +import java.net.MalformedURLException; import java.util.Date; import java.util.Map; @@ -34,6 +35,7 @@ import org.apache.http.entity.mime.content.ContentBody; import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.util.FileUtils; import net.yacy.peers.Network; @@ -42,6 +44,7 @@ import net.yacy.peers.Protocol.Post; import net.yacy.peers.Seed; import net.yacy.peers.SeedDB; import net.yacy.search.Switchboard; +import net.yacy.search.SwitchboardConstants; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; import net.yacy.utils.crypt; @@ -77,8 +80,35 @@ public class MessageSend_p { // first ask if the other peer is online, and also what kind of document it accepts Seed seed = sb.peers.getConnected(ASCII.getBytes(hash)); if (seed != null) { - for (String ip : seed.getIPs()) { - final Map result = Protocol.permissionMessage(seed.getPublicAddress(ip), hash); + for (final String ip : seed.getIPs()) { + Map result = null; + MultiProtocolURL targetBaseURL = null; + final String targetBaseURLStr = seed.getPublicURL(ip, + sb.getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED, + SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT)); + try { + targetBaseURL = new MultiProtocolURL(targetBaseURLStr); + result = Protocol.permissionMessage(targetBaseURL, seed, sb); + } catch(final MalformedURLException e) { + Network.log.warn("yacyClient.permissionMessage malformed target peer URL :" + targetBaseURLStr); + } catch(final Exception e) { + // most probably a network time-out exception + Network.log.warn("yacyClient.permissionMessage error:" + e.getMessage()); + if(targetBaseURL.isHTTPS()) { + try { + /* Request made over https : retry using http on the same IP as a fallback */ + targetBaseURL = seed.getPublicMultiprotocolURL(ip, false); + result = Protocol.permissionMessage(targetBaseURL, seed, sb); + if(result != null) { + /* Got a successfull result with http : mark now SSl as not available ont the target peer */ + seed.setFlagSSLAvailable(false); + sb.peers.updateConnected(seed); + } + } catch (final IOException e2) { + Network.log.warn("yacyClient.postMessage error:" + e2.getMessage()); + } + } + } //System.out.println("DEBUG: permission request result = " + result.toString()); String peerName; Seed targetPeer = null; @@ -102,6 +132,7 @@ public class MessageSend_p { sb.peers.peerActions.interfaceDeparture(targetPeer, ip); } } else { + prop.put("mode_permission", "1"); // write input form @@ -146,28 +177,56 @@ public class MessageSend_p { final String salt = crypt.randomSalt(); // send request - final Map parts = Protocol.basicRequestParts(Switchboard.getSwitchboard(), hash, salt); + final Map parts = Protocol.basicRequestParts(sb, hash, salt); parts.put("process", UTF8.StringBody("post")); parts.put("myseed", UTF8.StringBody(seedDB.mySeed().genSeedStr(salt))); parts.put("subject", UTF8.StringBody(subject)); parts.put("message", UTF8.StringBody(mb)); - Seed seed = seedDB.getConnected(ASCII.getBytes(hash)); + final Seed seed = seedDB.getConnected(ASCII.getBytes(hash)); + boolean preferHttps = sb.getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED, + SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT); Post post1 = null; - for (String ip: seed.getIPs()) { - try { - post1 = new Post(seed.getPublicAddress(ip), hash, "/yacy/message.html", parts, 20000); - } catch (IOException e) { - Network.log.warn("yacyClient.postMessage error:" + e.getMessage()); - post1 = null; - } - if (post1 != null) break; - seedDB.peerActions.interfaceDeparture(seed, ip); + for(final String ip : seed.getIPs()) { + MultiProtocolURL targetBaseURL = null; + try { + targetBaseURL = seed.getPublicMultiprotocolURL(ip, preferHttps); + post1 = new Post(targetBaseURL, seed.hash, "/yacy/message.html", parts, 20000); + } catch(final MalformedURLException e) { + Network.log.warn("yacyClient.postMessage malformed target peer URL when using ip " + ip); + } catch (final IOException e) { + Network.log.warn("yacyClient.postMessage error:" + e.getMessage()); + if(targetBaseURL.isHTTPS()) { + try { + /* Request made over https : retry using http on the same IP as a fallback */ + targetBaseURL = seed.getPublicMultiprotocolURL(ip, false); + post1 = new Post(targetBaseURL, seed.hash, "/yacy/message.html", parts, 20000); + if(post1 != null) { + /* Got a successfull result with http : mark now SSl as not available ont the target peer */ + seed.setFlagSSLAvailable(false); + } + } catch (final IOException e2) { + Network.log.warn("yacyClient.postMessage error:" + e2.getMessage()); + } + } + } + + if (post1 != null) { + break; + } + seedDB.peerActions.interfaceDeparture(seed, ip); } - final Map result1 = post1 == null ? null : FileUtils.table(post1.result); + final Map result1 = post1 == null ? null : FileUtils.table(post1.getResult()); final Map result = result1; - //message has been sent - prop.put("mode_status_response", result.get("response")); + if(result != null) { + // message has been sent + prop.put("mode_status_response", result.get("response")); + } else { + prop.put("mode_status", "1"); + + // "unresolved pattern", the remote peer is alive but had an exception + prop.putXML("mode_status_message", message); + } } catch (final NumberFormatException e) { prop.put("mode_status", "1"); diff --git a/htroot/Network.java b/htroot/Network.java index ee5539dd8..11416b7e9 100644 --- a/htroot/Network.java +++ b/htroot/Network.java @@ -27,6 +27,7 @@ // javac -classpath .:../classes Network.java // if the shell's current path is HTROOT +import java.net.MalformedURLException; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -39,6 +40,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; @@ -223,29 +225,40 @@ public class Network { map.put(Seed.IP, challengeIP); map.put(Seed.PORT, challengePort); Seed peer = post.get("peerHash") == null ? null : new Seed(post.get("peerHash"), map); - String challengeAddress = peer.getPublicAddress(challengeIP); sb.updateMySeed(); Seed mySeed = sb.peers.mySeed(); - final Map response = Protocol.hello(mySeed, sb.peers.peerActions, challengeAddress, peer.hash); - - if (response == null) { - Seed peerd = sb.peers.get(peer.hash); - if (peerd != null) peer = peerd; - sb.peers.peerActions.interfaceDeparture(peer, challengeIP); - prop.put("table_comment",1); - prop.put("table_comment_status", "publish: no response from peer '" + peer.getName() + "/" + post.get("peerHash") + "' from " + challengeAddress + ""); - } else { - String yourtype = response.get("yourtype"); - String yourip = response.get("yourip"); - peer = sb.peers.getConnected(peer.hash); - if (peer == null) { + final String challengeURLStr = peer.getPublicURL(challengeIP, false); + try { + final MultiProtocolURL challengeURL = new MultiProtocolURL(challengeURLStr); + final Map response = Protocol.hello(mySeed, sb.peers.peerActions, challengeURL, peer.hash); + + if (response == null) { + Seed peerd = sb.peers.get(peer.hash); + if (peerd != null) peer = peerd; + sb.peers.peerActions.interfaceDeparture(peer, challengeIP); prop.put("table_comment",1); - prop.put("table_comment_status","publish: disconnected peer 'UNKNOWN/" + post.get("peerHash") + "' from " + challengeAddress + ", yourtype = " + yourtype + ", yourip = " + yourip); + prop.put("table_comment_status", "publish: no response from peer '" + peer.getName() + "/" + post.get("peerHash") + "' from " + challengeURL + ""); } else { - prop.put("table_comment",2); - prop.put("table_comment_status","publish: handshaked " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + peer.getName() + "' at " + challengeAddress + ", yourtype = " + yourtype + ", yourip = " + yourip); - prop.putHTML("table_comment_details",peer.toString()); + String yourtype = response.get("yourtype"); + String yourip = response.get("yourip"); + peer = sb.peers.getConnected(peer.hash); + if (peer == null) { + prop.put("table_comment",1); + prop.put("table_comment_status","publish: disconnected peer 'UNKNOWN/" + post.get("peerHash") + "' from " + challengeURL + ", yourtype = " + yourtype + ", yourip = " + yourip); + } else { + prop.put("table_comment",2); + prop.put("table_comment_status","publish: handshaked " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + peer.getName() + "' at " + challengeURL + ", yourtype = " + yourtype + ", yourip = " + yourip); + prop.putHTML("table_comment_details",peer.toString()); + } + } + } catch(final MalformedURLException e) { + final Seed peerd = sb.peers.get(peer.hash); + if (peerd != null) { + peer = peerd; } + sb.peers.peerActions.interfaceDeparture(peer, challengeIP); + prop.put("table_comment",1); + prop.put("table_comment_status", "publish: malformed URL for peer '" + peer.getName() + "/" + post.get("peerHash") + "' : " + challengeURLStr); } prop.putHTML("table_peerHash",post.get("peerHash")); diff --git a/htroot/yacy/hello.java b/htroot/yacy/hello.java index 5bed791be..a153204e9 100644 --- a/htroot/yacy/hello.java +++ b/htroot/yacy/hello.java @@ -29,10 +29,12 @@ import java.io.IOException; import java.net.InetAddress; +import java.net.MalformedURLException; import java.util.Iterator; import java.util.Set; import java.util.concurrent.ConcurrentMap; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; @@ -43,6 +45,7 @@ import net.yacy.peers.Seed; import net.yacy.peers.graphics.ProfilingGraph; import net.yacy.search.EventTracker; import net.yacy.search.Switchboard; +import net.yacy.search.SwitchboardConstants; import net.yacy.server.serverCore; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -153,11 +156,14 @@ public final class hello { } final int connectedBefore = sb.peers.sizeConnected(); //ConcurrentLog.info("**hello-DEBUG**", "peer " + remoteSeed.getName() + " challenged us with IPs " + reportedips); + final boolean preferHttps = sb.getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED, + SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT); + final int totalTimeout = preferHttps ? 13000 : 6500; int callbackRemain = Math.min(5, reportedips.size()); - long callbackStart = System.currentTimeMillis(); - if (callbackRemain > 0 && reportedips.size() > 0) { + final long callbackStart = System.currentTimeMillis(); + if (callbackRemain > 0 && reportedips.size() > 0) { for (String reportedip: reportedips) { - int partialtimeout = ((int) (callbackStart + 6500 - System.currentTimeMillis())) / callbackRemain; // bad hack until a concurrent version is implemented + int partialtimeout = ((int) (callbackStart + totalTimeout - System.currentTimeMillis())) / callbackRemain; // bad hack until a concurrent version is implemented if (partialtimeout <= 0) break; //ConcurrentLog.info("**hello-DEBUG**", "reportedip = " + reportedip + " is handled"); if (Seed.isProperIP(reportedip)) { @@ -165,11 +171,24 @@ public final class hello { prop.put("yourip", reportedip); remoteSeed.setIP(reportedip); time = System.currentTimeMillis(); - callback = Protocol.queryRWICount(remoteSeed.getPublicAddress(reportedip), remoteSeed.hash, partialtimeout); + try { + MultiProtocolURL remoteBaseURL = remoteSeed.getPublicMultiprotocolURL(reportedip, preferHttps); + callback = Protocol.queryRWICount(remoteBaseURL, remoteSeed, partialtimeout); + if (callback[0] < 0 && remoteBaseURL.isHTTPS()) { + /* Failed using https : retry using http */ + remoteBaseURL = remoteSeed.getPublicMultiprotocolURL(reportedip, false); + callback = Protocol.queryRWICount(remoteBaseURL, remoteSeed, partialtimeout); + } + } catch(final MalformedURLException e) { + callback = new long[] {-1, -1}; + } //ConcurrentLog.info("**hello-DEBUG**", "reportedip = " + reportedip + " returns callback " + (callback == null ? "NULL" : callback[0])); time_backping = System.currentTimeMillis() - time; backping_method = "reportedip=" + reportedip; - if (callback[0] >= 0) { success = true; break; } + if (callback[0] >= 0) { + success = true; + break; + } if (--callbackRemain <= 0) break; // no more tries left / restrict to a limited number of ips } } diff --git a/source/net/yacy/peers/Network.java b/source/net/yacy/peers/Network.java index 32a26fc59..53bff58c6 100644 --- a/source/net/yacy/peers/Network.java +++ b/source/net/yacy/peers/Network.java @@ -54,6 +54,7 @@ import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.document.feed.RSSFeed; import net.yacy.cora.document.feed.RSSMessage; import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.protocol.Domains; import net.yacy.cora.util.ConcurrentLog; import net.yacy.peers.operation.yacySeedUploadFile; @@ -194,27 +195,51 @@ public class Network protected class publishThread extends Thread { - private Map result; private final Seed seed; public publishThread(final ThreadGroup tg, final Seed seed) { super(tg, "PublishSeed_" + seed.getName()); this.seed = seed; - this.result = null; } @Override public final void run() { + Map result = null; try { - for (String ip: this.seed.getIPs()) { - this.result = Protocol.hello(Network.this.sb.peers.mySeed(), Network.this.sb.peers.peerActions, this.seed.getPublicAddress(ip), this.seed.hash); - if ( this.result == null ) { - // no or wrong response, delete that address - final String cause = "peer ping to peer resulted in error response (added < 0)"; + final boolean preferHttps = Network.this.sb.getConfigBool( + SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED, + SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT); + for (final String ip: this.seed.getIPs()) { + try { + MultiProtocolURL targetBaseURL = this.seed.getPublicMultiprotocolURL(ip, preferHttps); + result = Protocol.hello(Network.this.sb.peers.mySeed(), Network.this.sb.peers.peerActions, targetBaseURL, this.seed.hash); + if (result == null && targetBaseURL.isHTTPS()) { + /* Failed with https : retry with http on the same address */ + targetBaseURL = this.seed.getPublicMultiprotocolURL(ip, false); + result = Protocol.hello(Network.this.sb.peers.mySeed(), Network.this.sb.peers.peerActions, + targetBaseURL, this.seed.hash); + if (result != null) { + /* Got a result using http : mark SSL as unavailable on the peer */ + log.info("publish: SSL/TLS unavailable on " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + + this.seed.getName() + "' : can be reached using http but not https on address " + + ip); + this.seed.setFlagSSLAvailable(false); + Network.this.sb.peers.updateConnected(this.seed); + } + } + if(result == null) { + // no or wrong response, delete that address + final String cause = "peer ping to peer resulted in error response (added < 0)"; + log.info("publish: disconnected " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' from " + this.seed.getIPs() + ": " + cause); + Network.this.sb.peers.peerActions.interfaceDeparture(this.seed, ip); + continue; + } + } catch(final MalformedURLException e) { + final String cause = "malformed peer URL"; log.info("publish: disconnected " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' from " + this.seed.getIPs() + ": " + cause); Network.this.sb.peers.peerActions.interfaceDeparture(this.seed, ip); - continue; - } + continue; + } // success! we have published our peer to a senior peer // update latest news from the other peer log.info("publish: handshaked "+ this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getIPs()); diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java index 94f9db8ed..8f5345681 100644 --- a/source/net/yacy/peers/Protocol.java +++ b/source/net/yacy/peers/Protocol.java @@ -58,7 +58,6 @@ import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -138,32 +137,37 @@ public final class Protocol { */ public static class Post { - public byte[] result; // contains the result from a successful post or null if no attempt was successful - public Set unsuccessfulAddresses; // contains a set of addresses which had been tested for submission was without success - public String successfulAddress; // contains the address which had been successfully used or null if no success with any Address + /** Contains the result from a successful post or null if no attempt was successful */ + private byte[] result; + /** + * @param targetBaseURL the base target URL + * @param targetHash the hash of the target peer + * @param path the path on the base URL + * @param parts the body content + * @param timeout the timeout in milliseconds + * @param httpFallback when true, retry as http when a https request failed + * @throws IOException + */ public Post( - final String targetAddress, - final String targetPeerHash, + final MultiProtocolURL targetBaseURL, + final String targetHash, final String path, final Map parts, final int timeout) throws IOException { final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent); httpClient.setTimout(timeout); - this.result = httpClient.POSTbytes( - new MultiProtocolURL("http://" + targetAddress + path), - Seed.b64Hash2hexHash(targetPeerHash) + ".yacyh", - parts, - false, true); - this.unsuccessfulAddresses = new HashSet<>(); - if (this.result == null) { - this.unsuccessfulAddresses.add(targetAddress); - this.successfulAddress = null; - } else { - this.successfulAddress = targetAddress; - } + MultiProtocolURL targetURL = new MultiProtocolURL(targetBaseURL, path); + this.result = httpClient.POSTbytes(targetURL, Seed.b64Hash2hexHash(targetHash) + ".yacyh", parts, false, + true); } - + + /** + * @return the result from a successful post or null if no attempt was successful + */ + public byte[] getResult() { + return this.result; + } } /** @@ -182,7 +186,7 @@ public final class Protocol { public static Map hello( final Seed mySeed, final PeerActions peerActions, - final String targetAddress, + final MultiProtocolURL targetBaseURL, final String targetHash) { Map result = null; @@ -202,7 +206,7 @@ public final class Protocol { final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 30000); content = httpClient.POSTbytes( - new MultiProtocolURL("http://" + targetAddress + "/yacy/hello.html"), + new MultiProtocolURL(targetBaseURL, "/yacy/hello.html"), Seed.b64Hash2hexHash(targetHash) + ".yacyh", parts, false, true); @@ -212,7 +216,7 @@ public final class Protocol { if ( Thread.currentThread().isInterrupted() ) {Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "' interrupted."); return null; } - Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "', peer " + targetAddress + "; exception: " + e.getMessage()); + Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "', peer " + targetBaseURL + "; exception: " + e.getMessage()); // try again (go into loop) result = null; } @@ -222,7 +226,7 @@ public final class Protocol { + ((result == null) ? "result null" : ("result=" + result.toString()))); return null; } - Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "' contacted peer at " + targetAddress + ", received " + ((content == null) ? "null" : content.length) + " bytes, time = " + responseTime + " milliseconds"); + Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "' contacted peer at " + targetBaseURL + ", received " + ((content == null) ? "null" : content.length) + " bytes, time = " + responseTime + " milliseconds"); // check consistency with expectation Seed otherPeer = null; @@ -233,7 +237,7 @@ public final class Protocol { } else { try { // patch the remote peer address to avoid that remote peers spoof the network with wrong addresses - String host = Domains.stripToHostName(targetAddress); + String host = targetBaseURL.getHost(); InetAddress ie = Domains.dnsResolve(host); otherPeer = Seed.genRemoteSeed(seed, false, ie.getHostAddress()); if ( !otherPeer.hash.equals(targetHash) ) { @@ -342,7 +346,7 @@ public final class Protocol { } else { try { if ( i == 1 ) { - String host = Domains.stripToHostName(targetAddress); + String host = targetBaseURL.getHost(); InetAddress ia = Domains.dnsResolve(host); if (ia == null) continue; host = ia.getHostAddress(); // the actual address of the target as we had been successful when contacting them is patched here @@ -364,23 +368,24 @@ public final class Protocol { return result; } - public static long[] queryRWICount(final String targetAddress, final String targetHash, int timeout) { + public static long[] queryRWICount(final MultiProtocolURL targetBaseURL, final Seed target, int timeout) { // prepare request final String salt = crypt.randomSalt(); // send request try { - final Map parts = basicRequestParts(Switchboard.getSwitchboard(), targetHash, salt); + final Map parts = basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt); parts.put("object", UTF8.StringBody("rwicount")); parts.put("env", UTF8.StringBody("")); - //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "posting request to " + targetAddress); - final Post post = new Post(targetAddress, targetHash, "/yacy/query.html", parts, timeout); - //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received CONTENT from requesting " + targetAddress + (post.result == null ? "NULL" : (": length = " + post.result.length))); + //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "posting request to " + targetBaseURL); + final Post post = new Post(targetBaseURL, target.hash, "/yacy/query.html", parts, timeout); + + //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received CONTENT from requesting " + targetBaseURL + (post.result == null ? "NULL" : (": length = " + post.result.length))); final Map result = FileUtils.table(post.result); if (result == null || result.isEmpty()) return new long[] {-1, -1}; - //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESULT from requesting " + targetAddress + " : result = " + result.toString()); + //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESULT from requesting " + targetBaseURL + " : result = " + result.toString()); final String resp = result.get("response"); - //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESPONSE from requesting " + targetAddress + " : response = " + resp); + //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESPONSE from requesting " + targetBaseURL + " : response = " + resp); if (resp == null) return new long[] {-1, -1}; String magic = result.get("magic"); if (magic == null) magic = "0"; @@ -390,7 +395,7 @@ public final class Protocol { return new long[] {-1, -1}; } } catch (final Exception e ) { - //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received EXCEPTION from requesting " + targetAddress + ": " + e.getMessage()); + //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received EXCEPTION from requesting " + targetBaseURL + ": " + e.getMessage()); if (Network.log.isFine()) Network.log.fine("yacyClient.queryRWICount error:" + e.getMessage()); return new long[] {-1, -1}; } @@ -518,7 +523,8 @@ public final class Protocol { Network.log.info("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + ")"); if(targetBaseURL.startsWith("https")) { /* First mark https unavailable on this peer before removing any interface */ - target.setFlagSSLAvailable(false); + target.setFlagSSLAvailable(false); + event.peers.updateConnected(target); } else { event.peers.peerActions.interfaceDeparture(target, ip); } @@ -608,7 +614,8 @@ public final class Protocol { Network.log.info("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + ")"); if(targetBaseURL.startsWith("https")) { /* First mark https unavailable on this peer before removing any interface */ - target.setFlagSSLAvailable(false); + target.setFlagSSLAvailable(false); + event.peers.updateConnected(target); } else { event.peers.peerActions.interfaceDeparture(target, ip); } @@ -1222,11 +1229,11 @@ public final class Protocol { if(targetBaseURL.startsWith("https")) { /* First mark https unavailable on this peer before removing anything else */ target.setFlagSSLAvailable(false); + event.peers.updateConnected(target); } else { target.setFlagSolrAvailable(false); } } - target.setFlagSolrAvailable(false || myseed); return -1; } } catch(InterruptedException e) { @@ -1477,26 +1484,34 @@ public final class Protocol { return true; } - public static Map permissionMessage(final String targetAddress, final String targetHash) { - // ask for allowed message size and attachment size - // if this replies null, the peer does not answer - + /** + * Post a request asking for allowed message size and attachment size to the + * target peer on the selected target ip. All parameters must not be null. + * + * @param targetBaseURL + * the public base URL of the target peer on one of its reported IP + * addresses in {@link Seed#getIPs()} + * @param target + * the target peer + * @param sb + * the switchboard instance + * @return the result of the request + * @throws IOException + * when the peer doesn't answer on this IP or any other error + * occurred + */ + public static Map permissionMessage(final MultiProtocolURL targetBaseURL, final Seed target, + final Switchboard sb) throws IOException { // prepare request final String salt = crypt.randomSalt(); // send request - try { - final Map parts = - basicRequestParts(Switchboard.getSwitchboard(), targetHash, salt); - parts.put("process", UTF8.StringBody("permission")); - final Post post = new Post(targetAddress, targetAddress, "/yacy/message.html", parts, 6000); - final Map result = FileUtils.table(post.result); - return result; - } catch (final Exception e ) { - // most probably a network time-out exception - Network.log.warn("yacyClient.permissionMessage error:" + e.getMessage()); - return null; - } + final Map parts = basicRequestParts(sb, target.hash, salt); + parts.put("process", UTF8.StringBody("permission")); + final Post post = new Post(targetBaseURL, target.hash, "/yacy/message.html", parts, 6000); + + final Map result = FileUtils.table(post.result); + return result; } public static Map crawlReceipt( @@ -1702,12 +1717,21 @@ public final class Protocol { final ReferenceContainerCache indexes, boolean gzipBody, final int timeout) { - for (String ip : targetSeed.getIPs()) { + final boolean preferHttps = Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED, SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT); + for (final String ip : targetSeed.getIPs()) { if (ip == null) { Network.log.warn("no address for transferRWI"); return null; } - final String address = targetSeed.getPublicAddress(ip); + MultiProtocolURL targetBaseURL = null; + try { + targetBaseURL = targetSeed.getPublicMultiprotocolURL(ip, preferHttps); + } catch(final MalformedURLException e) { + Network.log.info("yacyClient.transferRWI malformed target URL : " + targetBaseURL); + // disconnect unavailable peer ip + Switchboard.getSwitchboard().peers.peerActions.interfaceDeparture(targetSeed, ip); + continue; + } // prepare post values final String salt = crypt.randomSalt(); @@ -1746,12 +1770,25 @@ public final class Protocol { parts.put("entryc", UTF8.StringBody(Integer.toString(indexcount))); parts.put("indexes", UTF8.StringBody(entrypost.toString())); final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, timeout); - final byte[] content = - httpClient.POSTbytes( - new MultiProtocolURL("http://" + address + "/yacy/transferRWI.html"), - targetSeed.getHexHash() + ".yacyh", - parts, - gzipBody, true); + byte[] content = null; + try { + content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/transferRWI.html"), + targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true); + } catch(final IOException e) { + if(targetBaseURL.isHTTPS()) { + targetBaseURL = targetSeed.getPublicMultiprotocolURL(ip, false); + /* Failed with https : retry with http on the same address */ + content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/transferRWI.html"), + targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true); + if(content != null) { + /* Success with http : mark SSL as unavailable on the target peer */ + Network.log.info("yacyClient.transferRWI SSL unavailable on address " + ip); + Switchboard.getSwitchboard().peers.updateConnected(targetSeed); + } + } else { + throw e; + } + } final Iterator v = FileUtils.strings(content); // this should return a list of urlhashes that are unknown @@ -1761,7 +1798,7 @@ public final class Protocol { result.put(Seed.IP, ip); // add used ip to result for error handling (in case no "result" key was received) return result; } catch (final Exception e ) { - Network.log.info("yacyClient.transferRWI to " + address + " error: " + e.getMessage()); + Network.log.info("yacyClient.transferRWI to " + targetBaseURL + " error: " + e.getMessage()); // disconnect unavailable peer ip Switchboard.getSwitchboard().peers.peerActions.interfaceDeparture(targetSeed, ip); } @@ -1908,7 +1945,10 @@ public final class Protocol { final Map parts = basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt); parts.put("object", UTF8.StringBody("host")); - final Post post = new Post(target.getPublicAddress(target.getIP()), target.hash, "/yacy/idx.json", parts, 30000); + final String remoteBaseURL = target.getPublicURL(target.getIP(), + Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED, + SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT)); + final Post post = new Post(new MultiProtocolURL(remoteBaseURL), target.hash, "/yacy/idx.json", parts, 30000); if ( post.result == null || post.result.length == 0 ) { Network.log.warn("yacyClient.loadIDXHosts error: empty result"); return null; diff --git a/source/net/yacy/peers/Seed.java b/source/net/yacy/peers/Seed.java index 5ec1ce52c..562b0eb93 100644 --- a/source/net/yacy/peers/Seed.java +++ b/source/net/yacy/peers/Seed.java @@ -68,6 +68,7 @@ import net.yacy.cora.date.AbstractFormatter; import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.federate.yacy.Distribution; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.Digest; @@ -791,7 +792,7 @@ public class Seed implements Cloneable, Comparable, Comparator * @return an URL string for the given peer ip * @throws RuntimeException when the ip parameter is null */ - public final String getPublicURL(final String ip, final boolean preferHTTPS) { + public final String getPublicURL(final String ip, final boolean preferHTTPS) throws RuntimeException { if (ip == null) { throw new RuntimeException("ip == NULL"); // that should not happen in Peer-to-Peer mode (but can in Intranet mode) } @@ -822,6 +823,27 @@ public class Seed implements Cloneable, Comparable, Comparator } return sb.toString(); } + + /** + * Generate a public URL Multiprotocol instance using a given ip. This combines + * the ip with the http(s) port and encloses the ip with square brackets if the + * ip is of typeIPv6 + * + * @param ip + * a host name or ip address + * @param preferHTTPS + * when true and https is available on this Seed, use it as the + * scheme part of the url + * @return an MultiProtocolURL instance for the given peer ip + * @throws RuntimeException + * when the ip parameter is null + * @throws MalformedURLException + * when the ip and port could not make a well formed URL + */ + public final MultiProtocolURL getPublicMultiprotocolURL(final String ip, final boolean preferHTTPS) + throws RuntimeException, MalformedURLException { + return new MultiProtocolURL(getPublicURL(ip, preferHTTPS)); + } /** @return the port number of this seed or -1 if not present */ public final int getPort() { diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java index 88860ef77..71b91cab9 100644 --- a/source/net/yacy/search/SwitchboardConstants.java +++ b/source/net/yacy/search/SwitchboardConstants.java @@ -520,6 +520,13 @@ public final class SwitchboardConstants { public static final String NETWORK_BOOTSTRAP_SEEDLIST_STUB = "network.unit.bootstrap.seedlist"; public static final String NETWORK_SEARCHVERIFY = "network.unit.inspection.searchverify"; + + /** Key of the setting controlling whether https should be preferred for in-protocol operations when available on remote peers. + * A distinct general setting is available to control whether https sould be used for remote search queries : see {@link #REMOTESEARCH_HTTPS_PREFERRED} */ + public static final String NETWORK_PROTOCOL_HTTPS_PREFERRED = "network.unit.protocol.https.preferred"; + + /** Default setting value controlling whether https should be preferred for in-protocol operations when available on remote peers */ + public static final boolean NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT = false; /** * appearance