boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Started implementing optional https preference for protocol operations

Browse Source 
Introduced through the new configurable setting
network.unit.protocol.https.preferred, defaulting to false for now.

Let choose to prefer using https when available on remote peers to
perform YaCy protocol operations including notably hello or transferRWI.

Not yet implemented for every YaCy protocol operations.
pull/154/head
luccioman 7 years ago
parent 2bc61f5657
commit 17e004599d
8 changed files with 300 additions and 111 deletions
Show Stats Download Patch File Download Diff File

4
defaults/yacy.init
Unescape View File

@ -127,6 +127,10 @@ network.unit.domain.nocheck = false
# that means it is not usable in YaCy p2p-configurations, only in private portal configurations
network.unit.tenant.agent =
# Prefer https for in-protocol operations when available on remote peers
# A distinct general setting is available to control whether https sould be used for remote search queries : remotesearch.https.preferred
network.unit.protocol.https.preferred = false
# Update process properties
# The update server location is given in the network.unit.definition,
# but the settings for update processing and cycles are individual.

91
htroot/MessageSend_p.java
Unescape View File

@ -26,6 +26,7 @@
//if the shell's current path is HTROOT
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Date;
import java.util.Map;
@ -34,6 +35,7 @@ import org.apache.http.entity.mime.content.ContentBody;
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.peers.Network;
@ -42,6 +44,7 @@ import net.yacy.peers.Protocol.Post;
import net.yacy.peers.Seed;
import net.yacy.peers.SeedDB;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
import net.yacy.utils.crypt;
@ -77,8 +80,35 @@ public class MessageSend_p {
// first ask if the other peer is online, and also what kind of document it accepts
Seed seed = sb.peers.getConnected(ASCII.getBytes(hash));
if (seed != null) {
for (String ip : seed.getIPs()) {
final Map<String, String> result = Protocol.permissionMessage(seed.getPublicAddress(ip), hash);
for (final String ip : seed.getIPs()) {
Map<String, String> result = null;
MultiProtocolURL targetBaseURL = null;
final String targetBaseURLStr = seed.getPublicURL(ip,
sb.getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,
SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT));
try {
targetBaseURL = new MultiProtocolURL(targetBaseURLStr);
result = Protocol.permissionMessage(targetBaseURL, seed, sb);
} catch(final MalformedURLException e) {
Network.log.warn("yacyClient.permissionMessage malformed target peer URL :" + targetBaseURLStr);
} catch(final Exception e) {
// most probably a network time-out exception
Network.log.warn("yacyClient.permissionMessage error:" + e.getMessage());
if(targetBaseURL.isHTTPS()) {
try {
/* Request made over https : retry using http on the same IP as a fallback */
targetBaseURL = seed.getPublicMultiprotocolURL(ip, false);
result = Protocol.permissionMessage(targetBaseURL, seed, sb);
if(result != null) {
/* Got a successfull result with http : mark now SSl as not available ont the target peer */
seed.setFlagSSLAvailable(false);
sb.peers.updateConnected(seed);
}
} catch (final IOException e2) {
Network.log.warn("yacyClient.postMessage error:" + e2.getMessage());
}
}
}
//System.out.println("DEBUG: permission request result = " + result.toString());
String peerName;
Seed targetPeer = null;
@ -102,6 +132,7 @@ public class MessageSend_p {
sb.peers.peerActions.interfaceDeparture(targetPeer, ip);
}
} else {
prop.put("mode_permission", "1");
// write input form
@ -146,28 +177,56 @@ public class MessageSend_p {
final String salt = crypt.randomSalt();
// send request
final Map<String, ContentBody> parts = Protocol.basicRequestParts(Switchboard.getSwitchboard(), hash, salt);
final Map<String, ContentBody> parts = Protocol.basicRequestParts(sb, hash, salt);
parts.put("process", UTF8.StringBody("post"));
parts.put("myseed", UTF8.StringBody(seedDB.mySeed().genSeedStr(salt)));
parts.put("subject", UTF8.StringBody(subject));
parts.put("message", UTF8.StringBody(mb));
Seed seed = seedDB.getConnected(ASCII.getBytes(hash));
final Seed seed = seedDB.getConnected(ASCII.getBytes(hash));
boolean preferHttps = sb.getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,
SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT);
Post post1 = null;
for (String ip: seed.getIPs()) {
try {
post1 = new Post(seed.getPublicAddress(ip), hash, "/yacy/message.html", parts, 20000);
} catch (IOException e) {
Network.log.warn("yacyClient.postMessage error:" + e.getMessage());
post1 = null;
}
if (post1 != null) break;
seedDB.peerActions.interfaceDeparture(seed, ip);
for(final String ip : seed.getIPs()) {
MultiProtocolURL targetBaseURL = null;
try {
targetBaseURL = seed.getPublicMultiprotocolURL(ip, preferHttps);
post1 = new Post(targetBaseURL, seed.hash, "/yacy/message.html", parts, 20000);
} catch(final MalformedURLException e) {
Network.log.warn("yacyClient.postMessage malformed target peer URL when using ip " + ip);
} catch (final IOException e) {
Network.log.warn("yacyClient.postMessage error:" + e.getMessage());
if(targetBaseURL.isHTTPS()) {
try {
/* Request made over https : retry using http on the same IP as a fallback */
targetBaseURL = seed.getPublicMultiprotocolURL(ip, false);
post1 = new Post(targetBaseURL, seed.hash, "/yacy/message.html", parts, 20000);
if(post1 != null) {
/* Got a successfull result with http : mark now SSl as not available ont the target peer */
seed.setFlagSSLAvailable(false);
}
} catch (final IOException e2) {
Network.log.warn("yacyClient.postMessage error:" + e2.getMessage());
}
}
}
if (post1 != null) {
break;
}
seedDB.peerActions.interfaceDeparture(seed, ip);
}
final Map<String, String> result1 = post1 == null ? null : FileUtils.table(post1.result);
final Map<String, String> result1 = post1 == null ? null : FileUtils.table(post1.getResult());
final Map<String, String> result = result1;
//message has been sent
prop.put("mode_status_response", result.get("response"));
if(result != null) {
// message has been sent
prop.put("mode_status_response", result.get("response"));
} else {
prop.put("mode_status", "1");
// "unresolved pattern", the remote peer is alive but had an exception
prop.putXML("mode_status_message", message);
}
} catch (final NumberFormatException e) {
prop.put("mode_status", "1");

49
htroot/Network.java
Unescape View File

@ -27,6 +27,7 @@
// javac -classpath .:../classes Network.java
// if the shell's current path is HTROOT
import java.net.MalformedURLException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@ -39,6 +40,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.protocol.ClientIdentification;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
@ -223,29 +225,40 @@ public class Network {
map.put(Seed.IP, challengeIP);
map.put(Seed.PORT, challengePort);
Seed peer = post.get("peerHash") == null ? null : new Seed(post.get("peerHash"), map);
String challengeAddress = peer.getPublicAddress(challengeIP);
sb.updateMySeed();
Seed mySeed = sb.peers.mySeed();
final Map<String, String> response = Protocol.hello(mySeed, sb.peers.peerActions, challengeAddress, peer.hash);
if (response == null) {
Seed peerd = sb.peers.get(peer.hash);
if (peerd != null) peer = peerd;
sb.peers.peerActions.interfaceDeparture(peer, challengeIP);
prop.put("table_comment",1);
prop.put("table_comment_status", "publish: no response from peer '" + peer.getName() + "/" + post.get("peerHash") + "' from <a href=\"http://" + challengeAddress + "\" target=\"_blank\">" + challengeAddress + "</a>");
} else {
String yourtype = response.get("yourtype");
String yourip = response.get("yourip");
peer = sb.peers.getConnected(peer.hash);
if (peer == null) {
final String challengeURLStr = peer.getPublicURL(challengeIP, false);
try {
final MultiProtocolURL challengeURL = new MultiProtocolURL(challengeURLStr);
final Map<String, String> response = Protocol.hello(mySeed, sb.peers.peerActions, challengeURL, peer.hash);
if (response == null) {
Seed peerd = sb.peers.get(peer.hash);
if (peerd != null) peer = peerd;
sb.peers.peerActions.interfaceDeparture(peer, challengeIP);
prop.put("table_comment",1);
prop.put("table_comment_status","publish: disconnected peer 'UNKNOWN/" + post.get("peerHash") + "' from <a href=\"http://" + challengeAddress + "\" target=\"_blank\">" + challengeAddress + "</a>, yourtype = " + yourtype + ", yourip = " + yourip);
prop.put("table_comment_status", "publish: no response from peer '" + peer.getName() + "/" + post.get("peerHash") + "' from <a href=\"" + challengeURL + "\" target=\"_blank\">" + challengeURL + "</a>");
} else {
prop.put("table_comment",2);
prop.put("table_comment_status","publish: handshaked " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + peer.getName() + "' at <a href=\"http://" + challengeAddress + "\" target=\"_blank\">" + challengeAddress + "</a>, yourtype = " + yourtype + ", yourip = " + yourip);
prop.putHTML("table_comment_details",peer.toString());
String yourtype = response.get("yourtype");
String yourip = response.get("yourip");
peer = sb.peers.getConnected(peer.hash);
if (peer == null) {
prop.put("table_comment",1);
prop.put("table_comment_status","publish: disconnected peer 'UNKNOWN/" + post.get("peerHash") + "' from <a href=\"" + challengeURL + "\" target=\"_blank\">" + challengeURL + "</a>, yourtype = " + yourtype + ", yourip = " + yourip);
} else {
prop.put("table_comment",2);
prop.put("table_comment_status","publish: handshaked " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + peer.getName() + "' at <a href=\"" + challengeURL + "\" target=\"_blank\">" + challengeURL + "</a>, yourtype = " + yourtype + ", yourip = " + yourip);
prop.putHTML("table_comment_details",peer.toString());
}
}
} catch(final MalformedURLException e) {
final Seed peerd = sb.peers.get(peer.hash);
if (peerd != null) {
peer = peerd;
}
sb.peers.peerActions.interfaceDeparture(peer, challengeIP);
prop.put("table_comment",1);
prop.put("table_comment_status", "publish: malformed URL for peer '" + peer.getName() + "/" + post.get("peerHash") + "' : " + challengeURLStr);
}
prop.putHTML("table_peerHash",post.get("peerHash"));

29
htroot/yacy/hello.java
Unescape View File

@ -29,10 +29,12 @@
import java.io.IOException;
import java.net.InetAddress;
import java.net.MalformedURLException;
import java.util.Iterator;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
@ -43,6 +45,7 @@ import net.yacy.peers.Seed;
import net.yacy.peers.graphics.ProfilingGraph;
import net.yacy.search.EventTracker;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.server.serverCore;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
@ -153,11 +156,14 @@ public final class hello {
}
final int connectedBefore = sb.peers.sizeConnected();
//ConcurrentLog.info("**hello-DEBUG**", "peer " + remoteSeed.getName() + " challenged us with IPs " + reportedips);
final boolean preferHttps = sb.getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,
SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT);
final int totalTimeout = preferHttps ? 13000 : 6500;
int callbackRemain = Math.min(5, reportedips.size());
long callbackStart = System.currentTimeMillis();
if (callbackRemain > 0 && reportedips.size() > 0) {
final long callbackStart = System.currentTimeMillis();
if (callbackRemain > 0 && reportedips.size() > 0) {
for (String reportedip: reportedips) {
int partialtimeout = ((int) (callbackStart + 6500 - System.currentTimeMillis())) / callbackRemain; // bad hack until a concurrent version is implemented
int partialtimeout = ((int) (callbackStart + totalTimeout - System.currentTimeMillis())) / callbackRemain; // bad hack until a concurrent version is implemented
if (partialtimeout <= 0) break;
//ConcurrentLog.info("**hello-DEBUG**", "reportedip = " + reportedip + " is handled");
if (Seed.isProperIP(reportedip)) {
@ -165,11 +171,24 @@ public final class hello {
prop.put("yourip", reportedip);
remoteSeed.setIP(reportedip);
time = System.currentTimeMillis();
callback = Protocol.queryRWICount(remoteSeed.getPublicAddress(reportedip), remoteSeed.hash, partialtimeout);
try {
MultiProtocolURL remoteBaseURL = remoteSeed.getPublicMultiprotocolURL(reportedip, preferHttps);
callback = Protocol.queryRWICount(remoteBaseURL, remoteSeed, partialtimeout);
if (callback[0] < 0 && remoteBaseURL.isHTTPS()) {
/* Failed using https : retry using http */
remoteBaseURL = remoteSeed.getPublicMultiprotocolURL(reportedip, false);
callback = Protocol.queryRWICount(remoteBaseURL, remoteSeed, partialtimeout);
}
} catch(final MalformedURLException e) {
callback = new long[] {-1, -1};
}
//ConcurrentLog.info("**hello-DEBUG**", "reportedip = " + reportedip + " returns callback " + (callback == null ? "NULL" : callback[0]));
time_backping = System.currentTimeMillis() - time;
backping_method = "reportedip=" + reportedip;
if (callback[0] >= 0) { success = true; break; }
if (callback[0] >= 0) {
success = true;
break;
}
if (--callbackRemain <= 0) break; // no more tries left / restrict to a limited number of ips
}
}

43
source/net/yacy/peers/Network.java
Unescape View File

@ -54,6 +54,7 @@ import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.feed.RSSFeed;
import net.yacy.cora.document.feed.RSSMessage;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.peers.operation.yacySeedUploadFile;
@ -194,27 +195,51 @@ public class Network
protected class publishThread extends Thread
{
private Map<String, String> result;
private final Seed seed;
public publishThread(final ThreadGroup tg, final Seed seed) {
super(tg, "PublishSeed_" + seed.getName());
this.seed = seed;
this.result = null;
}
@Override
public final void run() {
Map<String, String> result = null;
try {
for (String ip: this.seed.getIPs()) {
this.result = Protocol.hello(Network.this.sb.peers.mySeed(), Network.this.sb.peers.peerActions, this.seed.getPublicAddress(ip), this.seed.hash);
if ( this.result == null ) {
// no or wrong response, delete that address
final String cause = "peer ping to peer resulted in error response (added < 0)";
final boolean preferHttps = Network.this.sb.getConfigBool(
SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,
SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT);
for (final String ip: this.seed.getIPs()) {
try {
MultiProtocolURL targetBaseURL = this.seed.getPublicMultiprotocolURL(ip, preferHttps);
result = Protocol.hello(Network.this.sb.peers.mySeed(), Network.this.sb.peers.peerActions, targetBaseURL, this.seed.hash);
if (result == null && targetBaseURL.isHTTPS()) {
/* Failed with https : retry with http on the same address */
targetBaseURL = this.seed.getPublicMultiprotocolURL(ip, false);
result = Protocol.hello(Network.this.sb.peers.mySeed(), Network.this.sb.peers.peerActions,
targetBaseURL, this.seed.hash);
if (result != null) {
/* Got a result using http : mark SSL as unavailable on the peer */
log.info("publish: SSL/TLS unavailable on " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '"
+ this.seed.getName() + "' : can be reached using http but not https on address "
+ ip);
this.seed.setFlagSSLAvailable(false);
Network.this.sb.peers.updateConnected(this.seed);
}
}
if(result == null) {
// no or wrong response, delete that address
final String cause = "peer ping to peer resulted in error response (added < 0)";
log.info("publish: disconnected " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' from " + this.seed.getIPs() + ": " + cause);
Network.this.sb.peers.peerActions.interfaceDeparture(this.seed, ip);
continue;
}
} catch(final MalformedURLException e) {
final String cause = "malformed peer URL";
log.info("publish: disconnected " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' from " + this.seed.getIPs() + ": " + cause);
Network.this.sb.peers.peerActions.interfaceDeparture(this.seed, ip);
continue;
}
continue;
}
// success! we have published our peer to a senior peer
// update latest news from the other peer
log.info("publish: handshaked "+ this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getIPs());

164
source/net/yacy/peers/Protocol.java
Unescape View File

@ -58,7 +58,6 @@ import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@ -138,32 +137,37 @@ public final class Protocol {
*/
public static class Post {
public byte[] result; // contains the result from a successful post or null if no attempt was successful
public Set<String> unsuccessfulAddresses; // contains a set of addresses which had been tested for submission was without success
public String successfulAddress; // contains the address which had been successfully used or null if no success with any Address
/** Contains the result from a successful post or null if no attempt was successful */
private byte[] result;
/**
* @param targetBaseURL the base target URL
* @param targetHash the hash of the target peer
* @param path the path on the base URL
* @param parts the body content
* @param timeout the timeout in milliseconds
* @param httpFallback when true, retry as http when a https request failed
* @throws IOException
*/
public Post(
final String targetAddress,
final String targetPeerHash,
final MultiProtocolURL targetBaseURL,
final String targetHash,
final String path,
final Map<String, ContentBody> parts,
final int timeout) throws IOException {
final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
httpClient.setTimout(timeout);
this.result = httpClient.POSTbytes(
new MultiProtocolURL("http://" + targetAddress + path),
Seed.b64Hash2hexHash(targetPeerHash) + ".yacyh",
parts,
false, true);
this.unsuccessfulAddresses = new HashSet<>();
if (this.result == null) {
this.unsuccessfulAddresses.add(targetAddress);
this.successfulAddress = null;
} else {
this.successfulAddress = targetAddress;
}
MultiProtocolURL targetURL = new MultiProtocolURL(targetBaseURL, path);
this.result = httpClient.POSTbytes(targetURL, Seed.b64Hash2hexHash(targetHash) + ".yacyh", parts, false,
true);
}
/**
* @return the result from a successful post or null if no attempt was successful
*/
public byte[] getResult() {
return this.result;
}
}
/**
@ -182,7 +186,7 @@ public final class Protocol {
public static Map<String, String> hello(
final Seed mySeed,
final PeerActions peerActions,
final String targetAddress,
final MultiProtocolURL targetBaseURL,
final String targetHash) {
Map<String, String> result = null;
@ -202,7 +206,7 @@ public final class Protocol {
final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 30000);
content =
httpClient.POSTbytes(
new MultiProtocolURL("http://" + targetAddress + "/yacy/hello.html"),
new MultiProtocolURL(targetBaseURL, "/yacy/hello.html"),
Seed.b64Hash2hexHash(targetHash) + ".yacyh",
parts,
false, true);
@ -212,7 +216,7 @@ public final class Protocol {
if ( Thread.currentThread().isInterrupted() ) {Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "' interrupted.");
return null;
}
Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "', peer " + targetAddress + "; exception: " + e.getMessage());
Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "', peer " + targetBaseURL + "; exception: " + e.getMessage());
// try again (go into loop)
result = null;
}
@ -222,7 +226,7 @@ public final class Protocol {
+ ((result == null) ? "result null" : ("result=" + result.toString())));
return null;
}
Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "' contacted peer at " + targetAddress + ", received " + ((content == null) ? "null" : content.length) + " bytes, time = " + responseTime + " milliseconds");
Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "' contacted peer at " + targetBaseURL + ", received " + ((content == null) ? "null" : content.length) + " bytes, time = " + responseTime + " milliseconds");
// check consistency with expectation
Seed otherPeer = null;
@ -233,7 +237,7 @@ public final class Protocol {
} else {
try {
// patch the remote peer address to avoid that remote peers spoof the network with wrong addresses
String host = Domains.stripToHostName(targetAddress);
String host = targetBaseURL.getHost();
InetAddress ie = Domains.dnsResolve(host);
otherPeer = Seed.genRemoteSeed(seed, false, ie.getHostAddress());
if ( !otherPeer.hash.equals(targetHash) ) {
@ -342,7 +346,7 @@ public final class Protocol {
} else {
try {
if ( i == 1 ) {
String host = Domains.stripToHostName(targetAddress);
String host = targetBaseURL.getHost();
InetAddress ia = Domains.dnsResolve(host);
if (ia == null) continue;
host = ia.getHostAddress(); // the actual address of the target as we had been successful when contacting them is patched here
@ -364,23 +368,24 @@ public final class Protocol {
return result;
}
public static long[] queryRWICount(final String targetAddress, final String targetHash, int timeout) {
public static long[] queryRWICount(final MultiProtocolURL targetBaseURL, final Seed target, int timeout) {
// prepare request
final String salt = crypt.randomSalt();
// send request
try {
final Map<String, ContentBody> parts = basicRequestParts(Switchboard.getSwitchboard(), targetHash, salt);
final Map<String, ContentBody> parts = basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt);
parts.put("object", UTF8.StringBody("rwicount"));
parts.put("env", UTF8.StringBody(""));
//ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "posting request to " + targetAddress);
final Post post = new Post(targetAddress, targetHash, "/yacy/query.html", parts, timeout);
//ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received CONTENT from requesting " + targetAddress + (post.result == null ? "NULL" : (": length = " + post.result.length)));
//ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "posting request to " + targetBaseURL);
final Post post = new Post(targetBaseURL, target.hash, "/yacy/query.html", parts, timeout);
//ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received CONTENT from requesting " + targetBaseURL + (post.result == null ? "NULL" : (": length = " + post.result.length)));
final Map<String, String> result = FileUtils.table(post.result);
if (result == null || result.isEmpty()) return new long[] {-1, -1};
//ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESULT from requesting " + targetAddress + " : result = " + result.toString());
//ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESULT from requesting " + targetBaseURL + " : result = " + result.toString());
final String resp = result.get("response");
//ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESPONSE from requesting " + targetAddress + " : response = " + resp);
//ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESPONSE from requesting " + targetBaseURL + " : response = " + resp);
if (resp == null) return new long[] {-1, -1};
String magic = result.get("magic");
if (magic == null) magic = "0";
@ -390,7 +395,7 @@ public final class Protocol {
return new long[] {-1, -1};
}
} catch (final Exception e ) {
//ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received EXCEPTION from requesting " + targetAddress + ": " + e.getMessage());
//ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received EXCEPTION from requesting " + targetBaseURL + ": " + e.getMessage());
if (Network.log.isFine()) Network.log.fine("yacyClient.queryRWICount error:" + e.getMessage());
return new long[] {-1, -1};
}
@ -518,7 +523,8 @@ public final class Protocol {
Network.log.info("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + ")");
if(targetBaseURL.startsWith("https")) {
/* First mark https unavailable on this peer before removing any interface */
target.setFlagSSLAvailable(false);
target.setFlagSSLAvailable(false);
event.peers.updateConnected(target);
} else {
event.peers.peerActions.interfaceDeparture(target, ip);
}
@ -608,7 +614,8 @@ public final class Protocol {
Network.log.info("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + ")");
if(targetBaseURL.startsWith("https")) {
/* First mark https unavailable on this peer before removing any interface */
target.setFlagSSLAvailable(false);
target.setFlagSSLAvailable(false);
event.peers.updateConnected(target);
} else {
event.peers.peerActions.interfaceDeparture(target, ip);
}
@ -1222,11 +1229,11 @@ public final class Protocol {
if(targetBaseURL.startsWith("https")) {
/* First mark https unavailable on this peer before removing anything else */
target.setFlagSSLAvailable(false);
event.peers.updateConnected(target);
} else {
target.setFlagSolrAvailable(false);
}
}
target.setFlagSolrAvailable(false || myseed);
return -1;
}
} catch(InterruptedException e) {
@ -1477,26 +1484,34 @@ public final class Protocol {
return true;
}
public static Map<String, String> permissionMessage(final String targetAddress, final String targetHash) {
// ask for allowed message size and attachment size
// if this replies null, the peer does not answer
/**
* Post a request asking for allowed message size and attachment size to the
* target peer on the selected target ip. All parameters must not be null.
*
* @param targetBaseURL
* the public base URL of the target peer on one of its reported IP
* addresses in {@link Seed#getIPs()}
* @param target
* the target peer
* @param sb
* the switchboard instance
* @return the result of the request
* @throws IOException
* when the peer doesn't answer on this IP or any other error
* occurred
*/
public static Map<String, String> permissionMessage(final MultiProtocolURL targetBaseURL, final Seed target,
final Switchboard sb) throws IOException {
// prepare request
final String salt = crypt.randomSalt();
// send request
try {
final Map<String, ContentBody> parts =
basicRequestParts(Switchboard.getSwitchboard(), targetHash, salt);
parts.put("process", UTF8.StringBody("permission"));
final Post post = new Post(targetAddress, targetAddress, "/yacy/message.html", parts, 6000);
final Map<String, String> result = FileUtils.table(post.result);
return result;
} catch (final Exception e ) {
// most probably a network time-out exception
Network.log.warn("yacyClient.permissionMessage error:" + e.getMessage());
return null;
}
final Map<String, ContentBody> parts = basicRequestParts(sb, target.hash, salt);
parts.put("process", UTF8.StringBody("permission"));
final Post post = new Post(targetBaseURL, target.hash, "/yacy/message.html", parts, 6000);
final Map<String, String> result = FileUtils.table(post.result);
return result;
}
public static Map<String, String> crawlReceipt(
@ -1702,12 +1717,21 @@ public final class Protocol {
final ReferenceContainerCache<WordReference> indexes,
boolean gzipBody,
final int timeout) {
for (String ip : targetSeed.getIPs()) {
final boolean preferHttps = Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED, SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT);
for (final String ip : targetSeed.getIPs()) {
if (ip == null) {
Network.log.warn("no address for transferRWI");
return null;
}
final String address = targetSeed.getPublicAddress(ip);
MultiProtocolURL targetBaseURL = null;
try {
targetBaseURL = targetSeed.getPublicMultiprotocolURL(ip, preferHttps);
} catch(final MalformedURLException e) {
Network.log.info("yacyClient.transferRWI malformed target URL : " + targetBaseURL);
// disconnect unavailable peer ip
Switchboard.getSwitchboard().peers.peerActions.interfaceDeparture(targetSeed, ip);
continue;
}
// prepare post values
final String salt = crypt.randomSalt();
@ -1746,12 +1770,25 @@ public final class Protocol {
parts.put("entryc", UTF8.StringBody(Integer.toString(indexcount)));
parts.put("indexes", UTF8.StringBody(entrypost.toString()));
final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, timeout);
final byte[] content =
httpClient.POSTbytes(
new MultiProtocolURL("http://" + address + "/yacy/transferRWI.html"),
targetSeed.getHexHash() + ".yacyh",
parts,
gzipBody, true);
byte[] content = null;
try {
content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/transferRWI.html"),
targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true);
} catch(final IOException e) {
if(targetBaseURL.isHTTPS()) {
targetBaseURL = targetSeed.getPublicMultiprotocolURL(ip, false);
/* Failed with https : retry with http on the same address */
content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/transferRWI.html"),
targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true);
if(content != null) {
/* Success with http : mark SSL as unavailable on the target peer */
Network.log.info("yacyClient.transferRWI SSL unavailable on address " + ip);
Switchboard.getSwitchboard().peers.updateConnected(targetSeed);
}
} else {
throw e;
}
}
final Iterator<String> v = FileUtils.strings(content);
// this should return a list of urlhashes that are unknown
@ -1761,7 +1798,7 @@ public final class Protocol {
result.put(Seed.IP, ip); // add used ip to result for error handling (in case no "result" key was received)
return result;
} catch (final Exception e ) {
Network.log.info("yacyClient.transferRWI to " + address + " error: " + e.getMessage());
Network.log.info("yacyClient.transferRWI to " + targetBaseURL + " error: " + e.getMessage());
// disconnect unavailable peer ip
Switchboard.getSwitchboard().peers.peerActions.interfaceDeparture(targetSeed, ip);
}
@ -1908,7 +1945,10 @@ public final class Protocol {
final Map<String, ContentBody> parts =
basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt);
parts.put("object", UTF8.StringBody("host"));
final Post post = new Post(target.getPublicAddress(target.getIP()), target.hash, "/yacy/idx.json", parts, 30000);
final String remoteBaseURL = target.getPublicURL(target.getIP(),
Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,
SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT));
final Post post = new Post(new MultiProtocolURL(remoteBaseURL), target.hash, "/yacy/idx.json", parts, 30000);
if ( post.result == null || post.result.length == 0 ) {
Network.log.warn("yacyClient.loadIDXHosts error: empty result");
return null;

24
source/net/yacy/peers/Seed.java
Unescape View File

@ -68,6 +68,7 @@ import net.yacy.cora.date.AbstractFormatter;
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.federate.yacy.Distribution;
import net.yacy.cora.order.Base64Order;
import net.yacy.cora.order.Digest;
@ -791,7 +792,7 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
* @return an URL string for the given peer ip
* @throws RuntimeException when the ip parameter is null
*/
public final String getPublicURL(final String ip, final boolean preferHTTPS) {
public final String getPublicURL(final String ip, final boolean preferHTTPS) throws RuntimeException {
if (ip == null) {
throw new RuntimeException("ip == NULL"); // that should not happen in Peer-to-Peer mode (but can in Intranet mode)
}
@ -822,6 +823,27 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
}
return sb.toString();
}
/**
* Generate a public URL Multiprotocol instance using a given ip. This combines
* the ip with the http(s) port and encloses the ip with square brackets if the
* ip is of typeIPv6
*
* @param ip
* a host name or ip address
* @param preferHTTPS
* when true and https is available on this Seed, use it as the
* scheme part of the url
* @return an MultiProtocolURL instance for the given peer ip
* @throws RuntimeException
* when the ip parameter is null
* @throws MalformedURLException
* when the ip and port could not make a well formed URL
*/
public final MultiProtocolURL getPublicMultiprotocolURL(final String ip, final boolean preferHTTPS)
throws RuntimeException, MalformedURLException {
return new MultiProtocolURL(getPublicURL(ip, preferHTTPS));
}
/** @return the port number of this seed or <code>-1</code> if not present */
public final int getPort() {

7
source/net/yacy/search/SwitchboardConstants.java
Unescape View File

@ -520,6 +520,13 @@ public final class SwitchboardConstants {
public static final String NETWORK_BOOTSTRAP_SEEDLIST_STUB = "network.unit.bootstrap.seedlist";
public static final String NETWORK_SEARCHVERIFY = "network.unit.inspection.searchverify";
/** Key of the setting controlling whether https should be preferred for in-protocol operations when available on remote peers.
* A distinct general setting is available to control whether https sould be used for remote search queries : see {@link #REMOTESEARCH_HTTPS_PREFERRED} */
public static final String NETWORK_PROTOCOL_HTTPS_PREFERRED = "network.unit.protocol.https.preferred";
/** Default setting value controlling whether https should be preferred for in-protocol operations when available on remote peers */
public static final boolean NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT = false;
/**
* appearance

Write Preview
Loading…
Cancel
Save