boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Optionally allow external Solr server with self-signed certificate

Browse Source 
This is necessary when you want to attach to a dedicated external Solr
server protected with basic http authentication and requested over https
but having only a self-signed certificate.
pull/137/head
luccioman 7 years ago
parent 1e4ceaac3f
commit 69690c13a0
10 changed files with 90 additions and 12 deletions
Show Stats Download Patch File Download Diff File

2
defaults/yacy.init
Unescape View File

@ -1285,6 +1285,8 @@ federated.service.solr.indexing.sharding = MODULO_HOST_MD5
federated.service.solr.indexing.lazy = true
federated.service.solr.indexing.timeout = 60000
federated.service.solr.indexing.writeEnabled = true
# Set to true when TLS is enabled with a self-signed certificated on a main external Solr with authentication required ( its URL looks like https://user:password@localhost:8984/solr/)
federated.service.solr.indexing.authenticated.allowSelfSigned = false
# temporary definition of backend services to use.
# After the migration a rwi+solr combination is used, the solr contains the content of the previously used metadata-db.

4
htroot/IndexFederated_p.html
Unescape View File

@ -36,6 +36,10 @@
<dt>Use remote Solr server(s)&nbsp;<input type="checkbox" name="solr.indexing.solrremote" id="solr_indexing_solrremote" #(solr.indexing.solrremote.checked)#:: checked="checked"#(/solr.indexing.solrremote.checked)# onclick="if(!document.getElementById('config').solr_indexing_solrremote.checked) {document.getElementById('config').core_service_fulltext.checked = true;}"/></dt>
<dd>It's easy to <a href="http://www.yacy-websearch.net/wiki/index.php/Dev:Solr" target="_blank">attach an external Solr to YaCy</a>.
This external Solr can be used instead the internal Solr. It can also be used additionally to the internal Solr, then both Solr indexes are mirrored.
</dd>
<dt>Allow self-signed certificates <input type="checkbox" name="federated.service.solr.indexing.authenticated.allowSelfSigned" #(federated.service.solr.indexing.authenticated.allowSelfSigned.checked)#:: checked="checked"#(/federated.service.solr.indexing.authenticated.allowSelfSigned.checked)#/></dt>
<dd>Tick this when the remote Solr server is password protected and is requested over HTTPS but provides only a self-signed certificate (not a validated one by an official Certificate Authority). The Solr URL could be for example something like <i>https://user:password@localhost:8984/solr</i>.
</dd>
#(table)#::

12
htroot/IndexFederated_p.java
Unescape View File

@ -104,6 +104,10 @@ public class IndexFederated_p {
String solrurls = post.get("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr"));
final boolean solrRemoteIsOnAfterwards = post.getBoolean("solr.indexing.solrremote") & solrurls.length() > 0;
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, solrRemoteIsOnAfterwards);
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED, post
.getBoolean(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED));
final BufferedReader r = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(UTF8.getBytes(solrurls))));
final StringBuilder s = new StringBuilder();
String s0;
@ -142,10 +146,13 @@ public class IndexFederated_p {
// switch on
final boolean usesolr = sb.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) & solrurls.length() > 0;
final int solrtimeout = sb.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 10000);
final boolean trustSelfSignedOnAuthenticatedServer = Switchboard.getSwitchboard().getConfigBool(
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);
try {
if (usesolr) {
ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout);
ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout, trustSelfSignedOnAuthenticatedServer);
sb.index.fulltext().connectRemoteSolr(instances, shardMethod, writeEnabled);
} else {
sb.index.fulltext().disconnectRemoteSolr();
@ -191,6 +198,9 @@ public class IndexFederated_p {
prop.put(SwitchboardConstants.CORE_SERVICE_CITATION + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, false) ? 1 : 0);
prop.put(SwitchboardConstants.CORE_SERVICE_WEBGRAPH + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, false) ? 1 : 0);
prop.put("solr.indexing.solrremote.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) ? 1 : 0);
prop.put(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED + ".checked",
env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT));
prop.put("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr").replace(",", "\n"));
String thisShardingMethodName = env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name());
int mc = 0;

8
source/net/yacy/cora/federate/SolrFederateSearchConnector.java
Unescape View File

@ -96,7 +96,13 @@ public class SolrFederateSearchConnector extends AbstractFederateSearchConnector
msp.add(CommonParams.QT, "/"); // important to override default append of /select
msp.add(CommonParams.ROWS, Integer.toString(query.itemsPerPage));
try {
RemoteInstance instance = new RemoteInstance(baseurl, remotecorename, corename, 20000);
boolean trustSelfSignedOnAuthenticatedServer = SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT;
if (Switchboard.getSwitchboard() != null) {
trustSelfSignedOnAuthenticatedServer = Switchboard.getSwitchboard().getConfigBool(
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);
}
RemoteInstance instance = new RemoteInstance(baseurl, remotecorename, corename, 20000, trustSelfSignedOnAuthenticatedServer);
try {
boolean useBinaryResponseWriter = SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT;
if (Switchboard.getSwitchboard() != null) {

2
source/net/yacy/cora/federate/solr/connector/RemoteSolrConnector.java
Unescape View File

@ -135,7 +135,7 @@ public class RemoteSolrConnector extends SolrServerConnector implements SolrConn
public static void main(final String args[]) {
RemoteSolrConnector solr;
try {
RemoteInstance instance = new RemoteInstance("http://127.0.0.1:8983/solr/", null, "collection1", 10000);
RemoteInstance instance = new RemoteInstance("http://127.0.0.1:8983/solr/", null, "collection1", 10000, false);
ArrayList<RemoteInstance> instances = new ArrayList<RemoteInstance>();
instances.add(instance);
solr = new RemoteSolrConnector(new ShardInstance(instances, ShardSelection.Method.MODULO_HOST_MD5, true), true, "solr");

42
source/net/yacy/cora/federate/solr/instance/RemoteInstance.java
Unescape View File

@ -77,18 +77,51 @@ public class RemoteInstance implements SolrInstance {
private final Map<String, ConcurrentUpdateSolrClient> server;
private final int timeout;
public static ArrayList<RemoteInstance> getShardInstances(final String urlList, Collection<String> coreNames, String defaultCoreName, final int timeout) throws IOException {
/**
* @param urlList
* the list of URLs of remote Solr shard instances. Must not be null.
* @param coreNames
* the Solr core names for the main collection and the webgraph
* @param defaultCoreName
* the core name of the main collection
* @param timeout
* the connection timeout in milliseconds
* @param trustSelfSignedOnAuthenticatedServer
* when true, self-signed certificates are accepcted for an https
* connection to a remote server with authentication credentials
* @throws IOException
* when a connection could not be opened to a remote Solr instance
*/
public static ArrayList<RemoteInstance> getShardInstances(final String urlList, Collection<String> coreNames,
String defaultCoreName, final int timeout, final boolean trustSelfSignedOnAuthenticatedServer)
throws IOException {
urlList.replace(' ', ',');
String[] urls = CommonPattern.COMMA.split(urlList);
ArrayList<RemoteInstance> instances = new ArrayList<RemoteInstance>();
for (final String u: urls) {
RemoteInstance instance = new RemoteInstance(u, coreNames, defaultCoreName, timeout);
RemoteInstance instance = new RemoteInstance(u, coreNames, defaultCoreName, timeout, trustSelfSignedOnAuthenticatedServer);
instances.add(instance);
}
return instances;
}
public RemoteInstance(final String url, final Collection<String> coreNames, final String defaultCoreName, final int timeout) throws IOException {
/**
* @param url
* the remote Solr URL. A default localhost URL is assumed when null.
* @param coreNames
* the Solr core names for the main collection and the webgraph
* @param defaultCoreName
* the core name of the main collection
* @param timeout
* the connection timeout in milliseconds
* @param trustSelfSignedOnAuthenticatedServer
* when true, self-signed certificates are accepcted for an https
* connection to a remote server with authentication credentials
* @throws IOException
* when a connection could not be opened to the remote Solr instance
*/
public RemoteInstance(final String url, final Collection<String> coreNames, final String defaultCoreName,
final int timeout, final boolean trustSelfSignedOnAuthenticatedServer) throws IOException {
this.timeout = timeout;
this.server= new HashMap<String, ConcurrentUpdateSolrClient>();
this.solrurl = url == null ? "http://127.0.0.1:8983/solr/" : url; // that should work for the example configuration of solr 4.x.x
@ -140,8 +173,7 @@ public class RemoteInstance implements SolrInstance {
}
}
if (solraccount.length() > 0) {
/* Note : optionally trusting self-signed certificate on an external remote Solr may be considered for convenience */
this.client = buildCustomHttpClient(timeout, u, solraccount, solrpw, host, false);
this.client = buildCustomHttpClient(timeout, u, solraccount, solrpw, host, trustSelfSignedOnAuthenticatedServer);
} else if(u.isHTTPS()){
/* Here we must trust self-signed certificates as most peers with SSL enabled use such certificates */
this.client = buildCustomHttpClient(timeout, u, solraccount, solrpw, host, true);

9
source/net/yacy/peers/Protocol.java
Unescape View File

@ -1102,7 +1102,14 @@ public final class Protocol {
@Override
public void run() {
try {
this.instance = new RemoteInstance(this.targetBaseURL, null, "solr", this.timeout); // this is a 'patch configuration' which considers 'solr' as default collection
boolean trustSelfSignedOnAuthenticatedServer = SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT;
if (Switchboard.getSwitchboard() != null) {
trustSelfSignedOnAuthenticatedServer = Switchboard.getSwitchboard().getConfigBool(
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);
}
this.instance = new RemoteInstance(this.targetBaseURL, null, "solr", this.timeout, trustSelfSignedOnAuthenticatedServer); // this is a 'patch configuration' which considers 'solr' as default collection
try {
boolean useBinaryResponseWriter = SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT;
if (Switchboard.getSwitchboard() != null) {

6
source/net/yacy/search/AutoSearch.java
Unescape View File

@ -255,7 +255,11 @@ public class AutoSearch extends AbstractBusyThread {
solrQuery.set(CommonParams.ROWS, sb.getConfig(SwitchboardConstants.REMOTESEARCH_MAXCOUNT_USER, "20"));
this.setName("Protocol.solrQuery(" + solrQuery.getQuery() + " to " + seed.hash + ")");
try {
RemoteInstance instance = new RemoteInstance("http://" + seed.getPublicAddress(seed.getIP()) + "/solr/", null, null, 10000); // this is a 'patch configuration' which considers 'solr' as default collection
final boolean trustSelfSignedOnAuthenticatedServer = this.sb.getConfigBool(
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);
RemoteInstance instance = new RemoteInstance("http://" + seed.getPublicAddress(seed.getIP()) + "/solr/", null, null, 10000, trustSelfSignedOnAuthenticatedServer); // this is a 'patch configuration' which considers 'solr' as default collection
try {
SolrConnector solrConnector = new RemoteSolrConnector(instance,
sb.getConfigBool(SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED,

10
source/net/yacy/search/Switchboard.java
Unescape View File

@ -585,10 +585,13 @@ public final class Switchboard extends serverSwitch {
final boolean usesolr = getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) & solrurls.length() > 0;
final int solrtimeout = getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 60000);
final boolean writeEnabled = getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_WRITEENABLED, true);
final boolean trustSelfSignedOnAuthenticatedServer = Switchboard.getSwitchboard().getConfigBool(
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);
if (usesolr && solrurls != null && solrurls.length() > 0) {
try {
ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout);
ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout, trustSelfSignedOnAuthenticatedServer);
String shardMethodName = getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name());
ShardSelection.Method shardMethod = ShardSelection.Method.valueOf(shardMethodName);
this.index.fulltext().connectRemoteSolr(instances, shardMethod, writeEnabled);
@ -1486,10 +1489,13 @@ public final class Switchboard extends serverSwitch {
final boolean usesolr = getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) & solrurls.length() > 0;
final int solrtimeout = getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 60000);
final boolean writeEnabled = getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_WRITEENABLED, true);
final boolean trustSelfSignedOnAuthenticatedServer = Switchboard.getSwitchboard().getConfigBool(
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);
if (usesolr && solrurls != null && solrurls.length() > 0) {
try {
ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout);
ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout, trustSelfSignedOnAuthenticatedServer);
String shardMethodName = getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name());
ShardSelection.Method shardMethod = ShardSelection.Method.valueOf(shardMethodName);
this.index.fulltext().connectRemoteSolr(instances, shardMethod, writeEnabled);

7
source/net/yacy/search/SwitchboardConstants.java
Unescape View File

@ -316,6 +316,13 @@ public final class SwitchboardConstants {
public static final String FEDERATED_SERVICE_SOLR_INDEXING_LAZY = "federated.service.solr.indexing.lazy";
public static final String FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT = "federated.service.solr.indexing.timeout";
public static final String FEDERATED_SERVICE_SOLR_INDEXING_WRITEENABLED = "federated.service.solr.indexing.writeEnabled";
/** Setting key controlling whether a self-signed certificate is acceptable from a remote Solr instance requested with authentication credentials.
* This has no impact on connections to remote Solr instances used in p2p search for which self-signed certificates are always accepted. */
public static final String FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED = "federated.service.solr.indexing.authenticated.allowSelfSigned";
/** Default value controlling whether a self-signed certificate is acceptable from a remote Solr instance with authentication credentials. */
public static final boolean FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT = false;
public static final String CORE_SERVICE_FULLTEXT = "core.service.fulltext";
public static final String CORE_SERVICE_RWI = "core.service.rwi.tmp";

Write Preview
Loading…
Cancel
Save