Optionally allow external Solr server with self-signed certificate

This is necessary when you want to attach to a dedicated external Solr
server protected with basic http authentication and requested over https
but having only a self-signed certificate.

defaults/yacy.init

@@ -1285,6 +1285,8 @@ federated.service.solr.indexing.sharding = MODULO_HOST_MD5
 federated.service.solr.indexing.lazy = true
 federated.service.solr.indexing.timeout = 60000
 federated.service.solr.indexing.writeEnabled = true
+# Set to true when TLS is enabled with a self-signed certificated on a main external Solr with authentication required ( its URL looks like https://user:password@localhost:8984/solr/) 
+federated.service.solr.indexing.authenticated.allowSelfSigned = false
 
 # temporary definition of backend services to use.
 # After the migration a rwi+solr combination is used, the solr contains the content of the previously used metadata-db.

htroot/IndexFederated_p.html

@@ -36,6 +36,10 @@
       <dt>Use remote Solr server(s)&nbsp;<input type="checkbox" name="solr.indexing.solrremote" id="solr_indexing_solrremote" #(solr.indexing.solrremote.checked)#:: checked="checked"#(/solr.indexing.solrremote.checked)# onclick="if(!document.getElementById('config').solr_indexing_solrremote.checked) {document.getElementById('config').core_service_fulltext.checked = true;}"/></dt>
       <dd>It's easy to <a href="http://www.yacy-websearch.net/wiki/index.php/Dev:Solr" target="_blank">attach an external Solr to YaCy</a>.
       This external Solr can be used instead the internal Solr. It can also be used additionally to the internal Solr, then both Solr indexes are mirrored. 
+      </dd>
+      
+      <dt>Allow self-signed certificates <input type="checkbox" name="federated.service.solr.indexing.authenticated.allowSelfSigned" #(federated.service.solr.indexing.authenticated.allowSelfSigned.checked)#:: checked="checked"#(/federated.service.solr.indexing.authenticated.allowSelfSigned.checked)#/></dt>
+      <dd>Tick this when the remote Solr server is password protected and is requested over HTTPS but provides only a self-signed certificate (not a validated one by an official Certificate Authority). The Solr URL could be for example something like <i>https://user:password@localhost:8984/solr</i>.
       </dd>	
       
       #(table)#::

htroot/IndexFederated_p.java

@@ -104,6 +104,10 @@ public class IndexFederated_p {
             String solrurls = post.get("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr"));
             final boolean solrRemoteIsOnAfterwards = post.getBoolean("solr.indexing.solrremote") & solrurls.length() > 0;
             env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, solrRemoteIsOnAfterwards);
+            
+			env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED, post
+					.getBoolean(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED));
+            
             final BufferedReader r = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(UTF8.getBytes(solrurls))));
             final StringBuilder s = new StringBuilder();
             String s0;
@@ -142,10 +146,13 @@ public class IndexFederated_p {
                 // switch on
                 final boolean usesolr = sb.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) & solrurls.length() > 0;
                 final int solrtimeout = sb.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 10000);
+        		final boolean trustSelfSignedOnAuthenticatedServer = Switchboard.getSwitchboard().getConfigBool(
+        				SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
+        				SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);
                 
                 try {
                     if (usesolr) {
-                        ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout);
+                        ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout, trustSelfSignedOnAuthenticatedServer);
                         sb.index.fulltext().connectRemoteSolr(instances, shardMethod, writeEnabled);
                     } else {
                         sb.index.fulltext().disconnectRemoteSolr();
@@ -191,6 +198,9 @@ public class IndexFederated_p {
         prop.put(SwitchboardConstants.CORE_SERVICE_CITATION + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, false) ? 1 : 0);
         prop.put(SwitchboardConstants.CORE_SERVICE_WEBGRAPH + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, false) ? 1 : 0);
         prop.put("solr.indexing.solrremote.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) ? 1 : 0);
+		prop.put(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED + ".checked",
+				env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
+						SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT));
         prop.put("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr").replace(",", "\n"));        
         String thisShardingMethodName = env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name());
         int mc = 0;

source/net/yacy/cora/federate/SolrFederateSearchConnector.java

@@ -96,7 +96,13 @@ public class SolrFederateSearchConnector extends AbstractFederateSearchConnector
         msp.add(CommonParams.QT, "/"); // important to override default append of /select
         msp.add(CommonParams.ROWS, Integer.toString(query.itemsPerPage));
         try {
-            RemoteInstance instance = new RemoteInstance(baseurl, remotecorename, corename, 20000);
+			boolean trustSelfSignedOnAuthenticatedServer = SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT;
+			if (Switchboard.getSwitchboard() != null) {
+				trustSelfSignedOnAuthenticatedServer = Switchboard.getSwitchboard().getConfigBool(
+						SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
+						SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);
+			}
+            RemoteInstance instance = new RemoteInstance(baseurl, remotecorename, corename, 20000, trustSelfSignedOnAuthenticatedServer);
             try {
 				boolean useBinaryResponseWriter = SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT;
 				if (Switchboard.getSwitchboard() != null) {

source/net/yacy/cora/federate/solr/connector/RemoteSolrConnector.java

@@ -135,7 +135,7 @@ public class RemoteSolrConnector extends SolrServerConnector implements SolrConn
     public static void main(final String args[]) {
         RemoteSolrConnector solr;
         try {
-            RemoteInstance instance = new RemoteInstance("http://127.0.0.1:8983/solr/", null, "collection1", 10000);
+            RemoteInstance instance = new RemoteInstance("http://127.0.0.1:8983/solr/", null, "collection1", 10000, false);
             ArrayList<RemoteInstance> instances = new ArrayList<RemoteInstance>();
             instances.add(instance);
             solr = new RemoteSolrConnector(new ShardInstance(instances, ShardSelection.Method.MODULO_HOST_MD5, true), true, "solr");

source/net/yacy/cora/federate/solr/instance/RemoteInstance.java

@@ -77,18 +77,51 @@ public class RemoteInstance implements SolrInstance {
     private final Map<String, ConcurrentUpdateSolrClient> server;
     private final int timeout;
     
-    public static ArrayList<RemoteInstance> getShardInstances(final String urlList, Collection<String> coreNames, String defaultCoreName, final int timeout) throws IOException {
+	/**
+	 * @param urlList
+	 *            the list of URLs of remote Solr shard instances. Must not be null.
+	 * @param coreNames
+	 *            the Solr core names for the main collection and the webgraph
+	 * @param defaultCoreName
+	 *            the core name of the main collection
+	 * @param timeout
+	 *            the connection timeout in milliseconds
+	 * @param trustSelfSignedOnAuthenticatedServer
+	 *            when true, self-signed certificates are accepcted for an https
+	 *            connection to a remote server with authentication credentials
+	 * @throws IOException
+	 *             when a connection could not be opened to a remote Solr instance
+	 */
+	public static ArrayList<RemoteInstance> getShardInstances(final String urlList, Collection<String> coreNames,
+			String defaultCoreName, final int timeout, final boolean trustSelfSignedOnAuthenticatedServer)
+			throws IOException {
         urlList.replace(' ', ',');
         String[] urls = CommonPattern.COMMA.split(urlList);
         ArrayList<RemoteInstance> instances = new ArrayList<RemoteInstance>();
         for (final String u: urls) {
-            RemoteInstance instance = new RemoteInstance(u, coreNames, defaultCoreName, timeout);
+            RemoteInstance instance = new RemoteInstance(u, coreNames, defaultCoreName, timeout, trustSelfSignedOnAuthenticatedServer);
             instances.add(instance);
         }
         return instances;
     }
     
-    public RemoteInstance(final String url, final Collection<String> coreNames, final String defaultCoreName, final int timeout) throws IOException {
+	/**
+	 * @param url
+	 *            the remote Solr URL. A default localhost URL is assumed when null.
+	 * @param coreNames
+	 *            the Solr core names for the main collection and the webgraph
+	 * @param defaultCoreName
+	 *            the core name of the main collection
+	 * @param timeout
+	 *            the connection timeout in milliseconds
+	 * @param trustSelfSignedOnAuthenticatedServer
+	 *            when true, self-signed certificates are accepcted for an https
+	 *            connection to a remote server with authentication credentials
+	 * @throws IOException
+	 *             when a connection could not be opened to the remote Solr instance
+	 */
+	public RemoteInstance(final String url, final Collection<String> coreNames, final String defaultCoreName,
+			final int timeout, final boolean trustSelfSignedOnAuthenticatedServer) throws IOException {
         this.timeout = timeout;
         this.server= new HashMap<String, ConcurrentUpdateSolrClient>();
         this.solrurl = url == null ? "http://127.0.0.1:8983/solr/" : url; // that should work for the example configuration of solr 4.x.x
@@ -140,8 +173,7 @@ public class RemoteInstance implements SolrInstance {
             }
         }
         if (solraccount.length() > 0) {
-            /* Note : optionally trusting self-signed certificate on an external remote Solr may be considered for convenience */
+            this.client = buildCustomHttpClient(timeout, u, solraccount, solrpw, host, trustSelfSignedOnAuthenticatedServer);
-            this.client = buildCustomHttpClient(timeout, u, solraccount, solrpw, host, false);
         } else if(u.isHTTPS()){
         	/* Here we must trust self-signed certificates as most peers with SSL enabled use such certificates */
         	this.client = buildCustomHttpClient(timeout, u, solraccount, solrpw, host, true);

source/net/yacy/peers/Protocol.java

@@ -1102,7 +1102,14 @@ public final class Protocol {
         @Override
         public void run() {
             try {
-                this.instance = new RemoteInstance(this.targetBaseURL, null, "solr", this.timeout); // this is a 'patch configuration' which considers 'solr' as default collection
+    			boolean trustSelfSignedOnAuthenticatedServer = SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT;
+    			if (Switchboard.getSwitchboard() != null) {
+    				trustSelfSignedOnAuthenticatedServer = Switchboard.getSwitchboard().getConfigBool(
+    						SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
+    						SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);
+    			}
+				
+                this.instance = new RemoteInstance(this.targetBaseURL, null, "solr", this.timeout, trustSelfSignedOnAuthenticatedServer); // this is a 'patch configuration' which considers 'solr' as default collection
                 try {
 					boolean useBinaryResponseWriter = SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT;
 					if (Switchboard.getSwitchboard() != null) {

source/net/yacy/search/AutoSearch.java

@@ -255,7 +255,11 @@ public class AutoSearch extends AbstractBusyThread {
             solrQuery.set(CommonParams.ROWS, sb.getConfig(SwitchboardConstants.REMOTESEARCH_MAXCOUNT_USER, "20"));
             this.setName("Protocol.solrQuery(" + solrQuery.getQuery() + " to " + seed.hash + ")");
             try {
-                RemoteInstance instance = new RemoteInstance("http://" + seed.getPublicAddress(seed.getIP()) + "/solr/", null, null, 10000); // this is a 'patch configuration' which considers 'solr' as default collection
+				final boolean trustSelfSignedOnAuthenticatedServer = this.sb.getConfigBool(
+						SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
+						SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);
+        		
+                RemoteInstance instance = new RemoteInstance("http://" + seed.getPublicAddress(seed.getIP()) + "/solr/", null, null, 10000, trustSelfSignedOnAuthenticatedServer); // this is a 'patch configuration' which considers 'solr' as default collection
                 try {
 					SolrConnector solrConnector = new RemoteSolrConnector(instance,
 							sb.getConfigBool(SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED,

source/net/yacy/search/Switchboard.java

@@ -585,10 +585,13 @@ public final class Switchboard extends serverSwitch {
         final boolean usesolr = getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) & solrurls.length() > 0;
         final int solrtimeout = getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 60000);
         final boolean writeEnabled = getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_WRITEENABLED, true);
+		final boolean trustSelfSignedOnAuthenticatedServer = Switchboard.getSwitchboard().getConfigBool(
+				SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
+				SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);
 
         if (usesolr && solrurls != null && solrurls.length() > 0) {
             try {
-                ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout);
+                ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout, trustSelfSignedOnAuthenticatedServer);
                 String shardMethodName = getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name());
                 ShardSelection.Method shardMethod = ShardSelection.Method.valueOf(shardMethodName);
                 this.index.fulltext().connectRemoteSolr(instances, shardMethod, writeEnabled);
@@ -1486,10 +1489,13 @@ public final class Switchboard extends serverSwitch {
             final boolean usesolr = getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) & solrurls.length() > 0;
             final int solrtimeout = getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 60000);
             final boolean writeEnabled = getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_WRITEENABLED, true);
+			final boolean trustSelfSignedOnAuthenticatedServer = Switchboard.getSwitchboard().getConfigBool(
+					SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
+					SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);
 
             if (usesolr && solrurls != null && solrurls.length() > 0) {
                 try {
-                    ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout);
+                    ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout, trustSelfSignedOnAuthenticatedServer);
                     String shardMethodName = getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name());
                     ShardSelection.Method shardMethod = ShardSelection.Method.valueOf(shardMethodName);
                     this.index.fulltext().connectRemoteSolr(instances, shardMethod, writeEnabled);

source/net/yacy/search/SwitchboardConstants.java

@@ -316,6 +316,13 @@ public final class SwitchboardConstants {
     public static final String FEDERATED_SERVICE_SOLR_INDEXING_LAZY         = "federated.service.solr.indexing.lazy";
     public static final String FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT      = "federated.service.solr.indexing.timeout";
     public static final String FEDERATED_SERVICE_SOLR_INDEXING_WRITEENABLED = "federated.service.solr.indexing.writeEnabled";
+    
+    /** Setting key controlling whether a self-signed certificate is acceptable from a remote Solr instance requested with authentication credentials. 
+     * This has no impact on connections to remote Solr instances used in p2p search for which self-signed certificates are always accepted. */
+    public static final String FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED = "federated.service.solr.indexing.authenticated.allowSelfSigned";
+
+    /** Default value controlling whether a self-signed certificate is acceptable from a remote Solr instance with authentication credentials. */
+    public static final boolean FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT = false;
 
     public static final String CORE_SERVICE_FULLTEXT            = "core.service.fulltext";
     public static final String CORE_SERVICE_RWI                 = "core.service.rwi.tmp";