boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Made possible to use https for remote search on peers with SSL enabled.

Browse Source 
Default is still http to prevent any regressions, but a new setting is
available to choose https as the preferred protocol to perform remote
searches. 
New configuration setting 'remotesearch.https.preferred' is manually
editable in yacy.conf file or in Advanced Properties page
(/ConfigProperties_p.html).
Should be enabled as default in the future for improved privacy. 
Https could also eventually be used for other peers communications.
pull/149/head
luccioman 7 years ago
parent 01dca12d05
commit f01aac31fd
5 changed files with 243 additions and 98 deletions
Show Stats Download Patch File Download Diff File

2
defaults/yacy.init
Unescape View File

@ -925,6 +925,8 @@ remotesearch.result.store=true
remotesearch.result.store.maxsize=-1 remotesearch.result.store.maxsize=-1
remotesearch.maxload.rwi=8.0 remotesearch.maxload.rwi=8.0
remotesearch.maxload.solr=4.0 remotesearch.maxload.solr=4.0
# When available on the target peer, whether https should be preferred to perform remote search
remotesearch.https.preferred = false
# Control whether remote Solr instances responses should be binary encoded. Responses are transferred as XML when set to false. # Control whether remote Solr instances responses should be binary encoded. Responses are transferred as XML when set to false.
remote.solr.binaryResponse.enabled=true remote.solr.binaryResponse.enabled=true

209
source/net/yacy/cora/federate/solr/instance/RemoteInstance.java
Unescape View File

@ -28,15 +28,9 @@ import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import net.yacy.cora.document.id.MultiProtocolURL; import javax.net.ssl.SSLContext;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.util.CommonPattern;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.search.schema.CollectionSchema;
import net.yacy.search.schema.WebgraphSchema;
import org.apache.commons.lang.StringUtils;
import org.apache.http.Header; import org.apache.http.Header;
import org.apache.http.HeaderElement; import org.apache.http.HeaderElement;
import org.apache.http.HttpEntity; import org.apache.http.HttpEntity;
@ -48,18 +42,35 @@ import org.apache.http.HttpResponseInterceptor;
import org.apache.http.auth.AuthScope; import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.AuthCache; import org.apache.http.client.AuthCache;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.GzipDecompressingEntity; import org.apache.http.client.entity.GzipDecompressingEntity;
import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.ssl.AllowAllHostnameVerifier;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
import org.apache.http.impl.auth.BasicScheme; import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.protocol.HTTP; import org.apache.http.protocol.HTTP;
import org.apache.http.protocol.HttpContext; import org.apache.http.protocol.HttpContext;
import org.apache.http.ssl.SSLContextBuilder;
import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient; import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.util.CommonPattern;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.search.schema.CollectionSchema;
import net.yacy.search.schema.WebgraphSchema;
@SuppressWarnings("deprecation") @SuppressWarnings("deprecation")
public class RemoteInstance implements SolrInstance { public class RemoteInstance implements SolrInstance {
private String solrurl; private String solrurl;
private final Object client; // not declared as org.apache.http.impl.client.DefaultHttpClient to avoid warnings during compilation. TODO: switch to org.apache.http.impl.client.HttpClientBuilder private final HttpClient client;
private final String defaultCoreName; private final String defaultCoreName;
private final ConcurrentUpdateSolrClient defaultServer; private final ConcurrentUpdateSolrClient defaultServer;
private final Collection<String> coreNames; private final Collection<String> coreNames;
@ -129,63 +140,108 @@ public class RemoteInstance implements SolrInstance {
} }
} }
if (solraccount.length() > 0) { if (solraccount.length() > 0) {
org.apache.http.impl.conn.PoolingClientConnectionManager cm = new org.apache.http.impl.conn.PoolingClientConnectionManager(); // try also: ThreadSafeClientConnManager /* Note : optionally trusting self-signed certificate on an external remote Solr may be considered for convenience */
cm.setMaxTotal(100); this.client = buildCustomHttpClient(timeout, u, solraccount, solrpw, host, false);
cm.setDefaultMaxPerRoute(100); } else if(u.isHTTPS()){
/* Here we must trust self-signed certificates as most peers with SSL enabled use such certificates */
this.client = new org.apache.http.impl.client.DefaultHttpClient(cm) { this.client = buildCustomHttpClient(timeout, u, solraccount, solrpw, host, true);
@Override
protected HttpContext createHttpContext() {
HttpContext context = super.createHttpContext();
AuthCache authCache = new org.apache.http.impl.client.BasicAuthCache();
BasicScheme basicAuth = new BasicScheme();
HttpHost targetHost = new HttpHost(u.getHost(), u.getPort(), u.getProtocol());
authCache.put(targetHost, basicAuth);
context.setAttribute(org.apache.http.client.protocol.HttpClientContext.AUTH_CACHE, authCache);
this.setHttpRequestRetryHandler(new org.apache.http.impl.client.DefaultHttpRequestRetryHandler(0, false)); // no retries needed; we expect connections to fail; therefore we should not retry
return context;
}
};
org.apache.http.params.HttpParams params = ((org.apache.http.impl.client.DefaultHttpClient) this.client).getParams();
org.apache.http.params.HttpConnectionParams.setConnectionTimeout(params, timeout);
org.apache.http.params.HttpConnectionParams.setSoTimeout(params, timeout);
((org.apache.http.impl.client.DefaultHttpClient) this.client).addRequestInterceptor(new HttpRequestInterceptor() {
@Override
public void process(final HttpRequest request, final HttpContext context) throws IOException {
if (!request.containsHeader(HeaderFramework.ACCEPT_ENCODING)) request.addHeader(HeaderFramework.ACCEPT_ENCODING, HeaderFramework.CONTENT_ENCODING_GZIP);
if (!request.containsHeader(HTTP.CONN_DIRECTIVE)) request.addHeader(HTTP.CONN_DIRECTIVE, "close"); // prevent CLOSE_WAIT
}
});
((org.apache.http.impl.client.DefaultHttpClient) this.client).addResponseInterceptor(new HttpResponseInterceptor() {
@Override
public void process(final HttpResponse response, final HttpContext context) throws IOException {
HttpEntity entity = response.getEntity();
if (entity != null) {
Header ceheader = entity.getContentEncoding();
if (ceheader != null) {
HeaderElement[] codecs = ceheader.getElements();
for (HeaderElement codec : codecs) {
if (codec.getName().equalsIgnoreCase(HeaderFramework.CONTENT_ENCODING_GZIP)) {
response.setEntity(new GzipDecompressingEntity(response.getEntity()));
return;
}
}
}
}
}
});
org.apache.http.impl.client.BasicCredentialsProvider credsProvider = new org.apache.http.impl.client.BasicCredentialsProvider();
credsProvider.setCredentials(new AuthScope(host, AuthScope.ANY_PORT), new UsernamePasswordCredentials(solraccount, solrpw));
((org.apache.http.impl.client.DefaultHttpClient) this.client).setCredentialsProvider(credsProvider);
} else { } else {
this.client = null; // The default HttpSolrClient will be used
this.client = null;
} }
this.defaultServer = (ConcurrentUpdateSolrClient) getServer(this.defaultCoreName); this.defaultServer = (ConcurrentUpdateSolrClient) getServer(this.defaultCoreName);
if (this.defaultServer == null) throw new IOException("cannot connect to url " + url + " and connect core " + defaultCoreName); if (this.defaultServer == null) throw new IOException("cannot connect to url " + url + " and connect core " + defaultCoreName);
} }
/**
* @param solraccount eventual user name used to authenticate on the target Solr
* @param solraccount eventual password used to authenticate on the target Solr
* @param trustSelfSignedCertificates when true, https connections to an host rpviding a self-signed certificate are accepted
* @return a new apache HttpClient instance usable as a custom http client by SolrJ
*/
private static HttpClient buildCustomHttpClient(final int timeout, final MultiProtocolURL u, final String solraccount, final String solrpw,
final String host, final boolean trustSelfSignedCertificates) {
/* Important note : deprecated use of Apache classes is required because SolrJ still use them internally (see HttpClientUtil).
* Upgrade only when Solr implementation will become compatible */
org.apache.http.impl.conn.PoolingClientConnectionManager cm;
SchemeRegistry registry = null;
if(trustSelfSignedCertificates) {
SSLContext sslContext;
try {
sslContext = SSLContextBuilder.create().loadTrustMaterial(TrustSelfSignedStrategy.INSTANCE).build();
registry = new SchemeRegistry();
registry.register(
new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
registry.register(
new Scheme("https", 443, new SSLSocketFactory(sslContext, AllowAllHostnameVerifier.INSTANCE)));
} catch (final Exception e) {
// Should not happen
ConcurrentLog.warn("RemoteInstance", "Error when initializing SSL context trusting self-signed certificates.", e);
registry = null;
}
}
if(registry != null) {
cm = new org.apache.http.impl.conn.PoolingClientConnectionManager(registry);
} else {
cm = new org.apache.http.impl.conn.PoolingClientConnectionManager(); // try also: ThreadSafeClientConnManager
}
cm.setMaxTotal(100);
cm.setDefaultMaxPerRoute(100);
org.apache.http.impl.client.DefaultHttpClient result = new org.apache.http.impl.client.DefaultHttpClient(cm) {
@Override
protected HttpContext createHttpContext() {
HttpContext context = super.createHttpContext();
AuthCache authCache = new org.apache.http.impl.client.BasicAuthCache();
BasicScheme basicAuth = new BasicScheme();
HttpHost targetHost = new HttpHost(u.getHost(), u.getPort(), u.getProtocol());
authCache.put(targetHost, basicAuth);
context.setAttribute(org.apache.http.client.protocol.HttpClientContext.AUTH_CACHE, authCache);
this.setHttpRequestRetryHandler(new org.apache.http.impl.client.DefaultHttpRequestRetryHandler(0, false)); // no retries needed; we expect connections to fail; therefore we should not retry
return context;
}
};
org.apache.http.params.HttpParams params = result.getParams();
org.apache.http.params.HttpConnectionParams.setConnectionTimeout(params, timeout);
org.apache.http.params.HttpConnectionParams.setSoTimeout(params, timeout);
result.addRequestInterceptor(new HttpRequestInterceptor() {
@Override
public void process(final HttpRequest request, final HttpContext context) throws IOException {
if (!request.containsHeader(HeaderFramework.ACCEPT_ENCODING)) request.addHeader(HeaderFramework.ACCEPT_ENCODING, HeaderFramework.CONTENT_ENCODING_GZIP);
if (!request.containsHeader(HTTP.CONN_DIRECTIVE)) request.addHeader(HTTP.CONN_DIRECTIVE, "close"); // prevent CLOSE_WAIT
}
});
result.addResponseInterceptor(new HttpResponseInterceptor() {
@Override
public void process(final HttpResponse response, final HttpContext context) throws IOException {
HttpEntity entity = response.getEntity();
if (entity != null) {
Header ceheader = entity.getContentEncoding();
if (ceheader != null) {
HeaderElement[] codecs = ceheader.getElements();
for (HeaderElement codec : codecs) {
if (codec.getName().equalsIgnoreCase(HeaderFramework.CONTENT_ENCODING_GZIP)) {
response.setEntity(new GzipDecompressingEntity(response.getEntity()));
return;
}
}
}
}
}
});
if(solraccount != null && !solraccount.isEmpty()) {
org.apache.http.impl.client.BasicCredentialsProvider credsProvider = new org.apache.http.impl.client.BasicCredentialsProvider();
credsProvider.setCredentials(new AuthScope(host, AuthScope.ANY_PORT), new UsernamePasswordCredentials(solraccount, solrpw));
result.setCredentialsProvider(credsProvider);
}
return result;
}
@Override @Override
public int hashCode() { public int hashCode() {
return this.solrurl.hashCode(); return this.solrurl.hashCode();
@ -228,22 +284,29 @@ public class RemoteInstance implements SolrInstance {
ConcurrentUpdateSolrClient s = this.server.get(name); ConcurrentUpdateSolrClient s = this.server.get(name);
if (s != null) return s; if (s != null) return s;
// create new http server // create new http server
final MultiProtocolURL u;
try {
u = new MultiProtocolURL(this.solrurl + name);
} catch (final MalformedURLException e) {
return null;
}
if (this.client != null) { if (this.client != null) {
final MultiProtocolURL u; final String solrServerURL;
try { if(StringUtils.isNotEmpty(u.getUserInfo())) {
u = new MultiProtocolURL(this.solrurl + name); /* Remove user authentication info from the URL, as authentication will be handled by the custom http client */
} catch (final MalformedURLException e) { String host = u.getHost();
return null; int port = u.getPort();
} String solrpath = u.getPath();
String host = u.getHost(); solrServerURL = u.getProtocol() + "://" + host + ":" + port + solrpath;
int port = u.getPort(); ConcurrentLog.info("RemoteSolrConnector", "connecting Solr authenticated with url : " + u);
String solrpath = u.getPath(); } else {
String p = "http://" + host + ":" + port + solrpath; solrServerURL = u.toString();
ConcurrentLog.info("RemoteSolrConnector", "connecting Solr authenticated with url:" + p); ConcurrentLog.info("RemoteSolrConnector", "connecting Solr with url : " + u);
s = new ConcurrentUpdateSolrClient(p, ((org.apache.http.impl.client.DefaultHttpClient) this.client), 10, Runtime.getRuntime().availableProcessors()); }
s = new ConcurrentUpdateSolrClient(solrServerURL, this.client, 10, Runtime.getRuntime().availableProcessors());
} else { } else {
ConcurrentLog.info("RemoteSolrConnector", "connecting Solr with url:" + this.solrurl + name); ConcurrentLog.info("RemoteSolrConnector", "connecting Solr with url : " + this.solrurl + name);
s = new ConcurrentUpdateSolrClient(this.solrurl + name, queueSizeByMemory(), Runtime.getRuntime().availableProcessors()); s = new ConcurrentUpdateSolrClient(u.toString(), queueSizeByMemory(), Runtime.getRuntime().availableProcessors());
} }
//s.setAllowCompression(true); //s.setAllowCompression(true);
s.setSoTimeout(this.timeout); s.setSoTimeout(this.timeout);

81
source/net/yacy/peers/Protocol.java
Unescape View File

@ -487,8 +487,14 @@ public final class Protocol {
SearchResult result = null; SearchResult result = null;
for (String ip: target.getIPs()) { for (String ip: target.getIPs()) {
//if (ip.indexOf(':') >= 0) System.out.println("Search target: IPv6: " + ip); //if (ip.indexOf(':') >= 0) System.out.println("Search target: IPv6: " + ip);
String clusteraddress = target.getPublicAddress(ip); final String targetBaseURL;
if (target.clash(event.peers.mySeed().getIPs())) clusteraddress = "localhost:" + event.peers.mySeed().getPort(); if (target.clash(event.peers.mySeed().getIPs())) {
targetBaseURL = "http://localhost:" + event.peers.mySeed().getPort();
} else {
targetBaseURL = target.getPublicURL(ip,
Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED,
SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED_DEFAULT));
}
try { try {
result = result =
new SearchResult( new SearchResult(
@ -504,13 +510,18 @@ public final class Protocol {
maxDistance, maxDistance,
partitions, partitions,
target.getHexHash() + ".yacyh", target.getHexHash() + ".yacyh",
clusteraddress, targetBaseURL,
secondarySearchSuperviser secondarySearchSuperviser
); );
break; break;
} catch (final IOException e ) { } catch (final IOException e ) {
Network.log.info("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + ")"); Network.log.info("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + ")");
event.peers.peerActions.interfaceDeparture(target, ip); if(targetBaseURL.startsWith("https")) {
/* First mark https unavailable on this peer before removing any interface */
target.setFlagSSLAvailable(false);
} else {
event.peers.peerActions.interfaceDeparture(target, ip);
}
return -1; return -1;
} }
} }
@ -571,6 +582,9 @@ public final class Protocol {
event.addExpectedRemoteReferences(count); event.addExpectedRemoteReferences(count);
SearchResult result = null; SearchResult result = null;
for (String ip: target.getIPs()) { for (String ip: target.getIPs()) {
final String targetBaseURL = target.getPublicURL(ip,
Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED,
SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED_DEFAULT));
try { try {
result = result =
new SearchResult( new SearchResult(
@ -586,13 +600,18 @@ public final class Protocol {
maxDistance, maxDistance,
partitions, partitions,
target.getHexHash() + ".yacyh", target.getHexHash() + ".yacyh",
target.getPublicAddress(ip), targetBaseURL,
null null
); );
break; break;
} catch (final IOException e ) { } catch (final IOException e ) {
Network.log.info("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + ")"); Network.log.info("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + ")");
event.peers.peerActions.interfaceDeparture(target, ip); if(targetBaseURL.startsWith("https")) {
/* First mark https unavailable on this peer before removing any interface */
target.setFlagSSLAvailable(false);
} else {
event.peers.peerActions.interfaceDeparture(target, ip);
}
return -1; return -1;
} }
} }
@ -845,7 +864,7 @@ public final class Protocol {
final int maxDistance, final int maxDistance,
final int partitions, final int partitions,
final String hostname, final String hostname,
final String hostaddress, final String targetBaseURL,
final SecondarySearchSuperviser secondarySearchSuperviser final SecondarySearchSuperviser secondarySearchSuperviser
) throws IOException { ) throws IOException {
// send a search request to peer with remote Hash // send a search request to peer with remote Hash
@ -902,8 +921,7 @@ public final class Protocol {
} }
final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 8000); final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 8000);
//System.out.println("Protocol: http://" + hostaddress + "/yacy/search.html" + requestPartsToString(parts)); // DEBUG byte[] a = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL + "/yacy/search.html"), hostname, parts, false, true);
byte[] a = httpClient.POSTbytes(new MultiProtocolURL("http://" + hostaddress + "/yacy/search.html"), hostname, parts, false, true);
if (a != null && a.length > 200000) { if (a != null && a.length > 200000) {
// there is something wrong. This is too large, maybe a hack on the other side? // there is something wrong. This is too large, maybe a hack on the other side?
a = null; a = null;
@ -986,8 +1004,8 @@ public final class Protocol {
/** The solr query to run */ /** The solr query to run */
private final SolrQuery solrQuery; private final SolrQuery solrQuery;
/** The instance address */ /** The instance base URL */
private final String address; private final String targetBaseURL;
/** The target seed information */ /** The target seed information */
private final Seed target; private final Seed target;
@ -1010,15 +1028,15 @@ public final class Protocol {
/** /**
* Constructor. All parameters are required to not be null. * Constructor. All parameters are required to not be null.
* @param solrQuery the Solr query to run * @param solrQuery the Solr query to run
* @param address the instance address : host name or IP + the eventual port * @param targetBaseURL the instance base URL : http(s) + host name or IP + the eventual port
* @param target the remote target seed information * @param target the remote target seed information
* @param timeout the request timeout in milliseconds * @param timeout the request timeout in milliseconds
*/ */
protected SolrRequestTask(final SolrQuery solrQuery, final String address, final Seed target, protected SolrRequestTask(final SolrQuery solrQuery, final String targetBaseURL, final Seed target,
final boolean mySeed, final int timeout, final QueryResponse[] rsp, final SolrDocumentList[] docList) { final boolean mySeed, final int timeout, final QueryResponse[] rsp, final SolrDocumentList[] docList) {
super("Protocol.solrQuery(" + solrQuery.getQuery() + " to " + target.hash + ")"); super("Protocol.solrQuery(" + solrQuery.getQuery() + " to " + target.hash + ")");
this.solrQuery = solrQuery; this.solrQuery = solrQuery;
this.address = address; this.targetBaseURL = targetBaseURL;
this.target = target; this.target = target;
this.mySeed = mySeed; this.mySeed = mySeed;
this.timeout = timeout; this.timeout = timeout;
@ -1040,13 +1058,13 @@ public final class Protocol {
/* Strip too large details to avoid polluting this log with complete remote stack traces */ /* Strip too large details to avoid polluting this log with complete remote stack traces */
message = message.substring(0, MAX_ERROR_MESSAGE_LENGTH) + "..."; message = message.substring(0, MAX_ERROR_MESSAGE_LENGTH) + "...";
} }
log.fine(messageBegin + " at " + this.address + " : " + message); log.fine(messageBegin + " at " + this.targetBaseURL + " : " + message);
} }
@Override @Override
public void run() { public void run() {
try { try {
this.instance = new RemoteInstance("http://" + this.address, null, "solr", this.timeout); // this is a 'patch configuration' which considers 'solr' as default collection this.instance = new RemoteInstance(this.targetBaseURL, null, "solr", this.timeout); // this is a 'patch configuration' which considers 'solr' as default collection
try { try {
boolean useBinaryResponseWriter = SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT; boolean useBinaryResponseWriter = SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT;
if (Switchboard.getSwitchboard() != null) { if (Switchboard.getSwitchboard() != null) {
@ -1170,13 +1188,20 @@ public final class Protocol {
} else { } else {
try { try {
final boolean myseed = target == event.peers.mySeed(); final boolean myseed = target == event.peers.mySeed();
final String targetBaseURL;
if(myseed) {
targetBaseURL = "http://localhost:" + target.getPort();
} else {
targetBaseURL = target.getPublicURL(ip,
Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED,
SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED_DEFAULT));
}
if (!myseed && !target.getFlagSolrAvailable()) { // skip if peer.dna has flag that last try resulted in error if (!myseed && !target.getFlagSolrAvailable()) { // skip if peer.dna has flag that last try resulted in error
Network.log.info("SEARCH skip (solr), remote Solr interface not accessible, peer=" + target.getName()); Network.log.info("SEARCH skip (solr), remote Solr interface not accessible, peer=" + target.getName());
return -1; return -1;
} }
final String address = myseed ? "localhost:" + target.getPort() : target.getPublicAddress(ip);
final int solrtimeout = Switchboard.getSwitchboard().getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 6000); final int solrtimeout = Switchboard.getSwitchboard().getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 6000);
SolrRequestTask remoteRequest = new SolrRequestTask(solrQuery, address, target, myseed, solrtimeout, rsp, docList); SolrRequestTask remoteRequest = new SolrRequestTask(solrQuery, targetBaseURL, target, myseed, solrtimeout, rsp, docList);
remoteRequest.start(); remoteRequest.start();
remoteRequest.join(solrtimeout); // just wait until timeout appears remoteRequest.join(solrtimeout); // just wait until timeout appears
if (remoteRequest.isAlive()) { if (remoteRequest.isAlive()) {
@ -1190,6 +1215,20 @@ public final class Protocol {
target.setFlagSolrAvailable(false || myseed); target.setFlagSolrAvailable(false || myseed);
return -1; // give up, leave remoteRequest abandoned. return -1; // give up, leave remoteRequest abandoned.
} }
if (rsp[0] == null || docList[0] == null) {
Network.log.info("SEARCH failed (solr), remote Peer: " + target.getName() + "/" + target.getPublicAddress(ip) + " returned null");
if(!myseed) {
if(targetBaseURL.startsWith("https")) {
/* First mark https unavailable on this peer before removing anything else */
target.setFlagSSLAvailable(false);
} else {
target.setFlagSolrAvailable(false);
}
}
target.setFlagSolrAvailable(false || myseed);
return -1;
}
} catch(InterruptedException e) { } catch(InterruptedException e) {
/* Current thread might be interrupted by SearchEvent.cleanup() : /* Current thread might be interrupted by SearchEvent.cleanup() :
* we must not in that case mark the target as not available but rather transmit the exception to the caller (likely RemoteSearch.solrRemoteSearch) */ * we must not in that case mark the target as not available but rather transmit the exception to the caller (likely RemoteSearch.solrRemoteSearch) */
@ -1201,12 +1240,6 @@ public final class Protocol {
} }
} }
if (rsp[0] == null || docList[0] == null) {
Network.log.info("SEARCH failed (solr), remote Peer: " + target.getName() + "/" + target.getPublicAddress(ip) + " returned null");
target.setFlagSolrAvailable(false || localsearch);
return -1;
}
// evaluate facets // evaluate facets
if(useSolrFacets) { if(useSolrFacets) {
for (String field: event.query.facetfields) { for (String field: event.query.facetfields) {

43
source/net/yacy/peers/Seed.java
Unescape View File

@ -756,10 +756,11 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
} }
/** /**
* generate a public address using a given ip. This combines the ip with the port and encloses the ip * generate a public address using a given ip. This combines the ip with the http port and encloses the ip
* with square brackets if the ip is of typeIPv6 * with square brackets if the ip is of typeIPv6
* @param ip * @param ip
* @return an address string which can be used as host:port part of an url (if no port avail returns just host) * @return an address string which can be used as host:port part of an url (if no port avail returns just host)
* @ŧhrows RuntimeException when the ip parameter is null
*/ */
public final String getPublicAddress(final String ip) { public final String getPublicAddress(final String ip) {
if (ip == null) throw new RuntimeException("ip == NULL"); // that should not happen in Peer-to-Peer mode (but can in Intranet mode) if (ip == null) throw new RuntimeException("ip == NULL"); // that should not happen in Peer-to-Peer mode (but can in Intranet mode)
@ -781,6 +782,46 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
} }
return sb.toString(); return sb.toString();
} }
/**
* Generate a public URL using a given ip. This combines the ip with the http(s) port and encloses the ip
* with square brackets if the ip is of typeIPv6
* @param ip a host name or ip address
* @param preferHTTPS when true and https is available on this Seed, use it as the scheme part of the url
* @return an URL string for the given peer ip
* @throws RuntimeException when the ip parameter is null
*/
public final String getPublicURL(final String ip, final boolean preferHTTPS) {
if (ip == null) {
throw new RuntimeException("ip == NULL"); // that should not happen in Peer-to-Peer mode (but can in Intranet mode)
}
final String scheme;
final String port;
if(preferHTTPS && getFlagSSLAvailable()) {
scheme = "https://";
port = this.dna.get(Seed.PORTSSL);
} else {
scheme = "http://";
port = this.dna.get(Seed.PORT);
}
final StringBuilder sb = new StringBuilder(scheme.length() + ip.length() + 8); // / = surplus for port
sb.append(scheme);
if (ip.indexOf(':') >= 0) {
if (!ip.startsWith("[")) sb.append('[');
sb.append(ip);
if (!ip.endsWith("]")) sb.append(']');
} else {
sb.append(ip);
}
if (port == null || port.length() < 2 || port.length() > 5) {
//just skip port if peer didn't report it..... may finally depart
Network.log.severe(preferHTTPS ? "https" : "http " + " port not wellformed for peer" + this.getName() + ": " + port == null ? "null" : port);
} else {
sb.append(':');
sb.append(port);
}
return sb.toString();
}
/** @return the port number of this seed or <code>-1</code> if not present */ /** @return the port number of this seed or <code>-1</code> if not present */
public final int getPort() { public final int getPort() {

6
source/net/yacy/search/SwitchboardConstants.java
Unescape View File

@ -290,6 +290,12 @@ public final class SwitchboardConstants {
/** Default maximum system load allowing remote Solr searches */ /** Default maximum system load allowing remote Solr searches */
public static final float REMOTESEARCH_MAXLOAD_SOLR_DEFAULT = 4.0f; public static final float REMOTESEARCH_MAXLOAD_SOLR_DEFAULT = 4.0f;
/** Key of the setting controlling whether https should be preferred for remote searches, when available on the target peer */
public static final String REMOTESEARCH_HTTPS_PREFERRED = "remotesearch.https.preferred";
/** Default setting value controlling whether https should be preferred for remote searches, when available on the target peer */
public static final boolean REMOTESEARCH_HTTPS_PREFERRED_DEFAULT = false;
/** /**
* Setting key to configure whether responses from remote Solr instances * Setting key to configure whether responses from remote Solr instances
* should be binary encoded : * should be binary encoded :

Write Preview
Loading…
Cancel
Save