diff --git a/defaults/yacy.init b/defaults/yacy.init index 755102985..1b8cc3cb1 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -227,6 +227,19 @@ http.outgoing.pool.general.maxTotal = 200 # maximum number of simultaneously open outgoing HTTP connections in the remote Solr pool (net.yacy.cora.federate.solr.instance.RemoteInstance) http.outgoing.pool.remoteSolr.maxTotal = 100 + +# TLS Server Name Indication (SNI) extension support for outgoing HTTP connections +# Must be enabled to load some https URLs (for websites deployed with different certificats and host names on the same shared IP address), otherwise loading fails with errors such as "Received fatal alert: handshake_failure" +# It can be necessary to disable it to load some https URLs served by old and misconfigured web servers, otherwise loading fails with the exception javax.net.ssl.SSLProtocolException: "handshake alert: unrecognized_name" +# In 2019, this last error should occur less frequently as in the early days of the extension implementation as its standard definition has evolved from RFC 4366 to RFC 6066 which revised how server implementations should handle the unrecognized_name(112) alert. +# Can also be configured with the JVM option jsse.enableSNIExtension, but in that case a server restart is required when you want to modify the setting and it is not customizable per http client (general or for remote Solr) + +# Set to true to enable TLS Server Name Indication (SNI) extension on outgoing HTTP connections in the general http client (net.yacy.cora.protocol.http.HTTPClient) +http.outgoing.general.tls.sniExtension.enabled = true + +# Set to true to enable TLS Server Name Indication (SNI) extension on outgoing HTTP connections in the remote Solr http client (net.yacy.cora.federate.solr.instance.RemoteInstance) +http.outgoing.remoteSolr.tls.sniExtension.enabled = true + # default root path for the file server # may be overridden by the htdocs parameter # users shall be encouraged to use the htdocs path for individual content, diff --git a/htroot/SettingsAck_p.html b/htroot/SettingsAck_p.html index 3c67481cc..9e91137ed 100644 --- a/htroot/SettingsAck_p.html +++ b/htroot/SettingsAck_p.html @@ -210,6 +210,8 @@ ::

Compression settings have been saved.

+ :: +

HTTP client settings have been saved.

#(/info)#

diff --git a/htroot/SettingsAck_p.java b/htroot/SettingsAck_p.java index 4eed26a48..745367801 100644 --- a/htroot/SettingsAck_p.java +++ b/htroot/SettingsAck_p.java @@ -642,6 +642,25 @@ public class SettingsAck_p { prop.put("info", "35"); return prop; } + + // HTTP client settings + if (post.containsKey("httpClientSettings")) { + // set backlink + prop.put("needsRestart_referer", "Settings_p.html?page=httpClient"); + + if(System.getProperty("jsse.enableSNIExtension") == null) { + /* Only apply custom SNI extension settings when the JVM system option jsse.enableSNIExtension is not defined */ + env.setConfig(SwitchboardConstants.HTTP_OUTGOING_GENERAL_TLS_SNI_EXTENSION_ENABLED, + post.containsKey(SwitchboardConstants.HTTP_OUTGOING_GENERAL_TLS_SNI_EXTENSION_ENABLED)); + + env.setConfig(SwitchboardConstants.HTTP_OUTGOING_REMOTE_SOLR_TLS_SNI_EXTENSION_ENABLED, + post.containsKey(SwitchboardConstants.HTTP_OUTGOING_REMOTE_SOLR_TLS_SNI_EXTENSION_ENABLED)); + } + sb.initOutgoingConnectionSettings(); + + prop.put("info", "38"); + return prop; + } // nothing made prop.put("info", "1");//no information submitted diff --git a/htroot/Settings_HttpClient.inc b/htroot/Settings_HttpClient.inc new file mode 100644 index 000000000..464a1d830 --- /dev/null +++ b/htroot/Settings_HttpClient.inc @@ -0,0 +1,72 @@ +

HTTP client settings

+ +

+ You can configure here some advanced settings of the clients used by YaCy to handle outgoing HTTP connections. +

+ +

+ About Server Name Indication (SNI) : + this extension to the TLS protocol must be enabled to load some https URLs (for websites deployed with different certificats and host names on the same shared IP address), otherwise loading fails with errors such as "Received fatal alert: handshake_failure". + But it can be necessary to disable it in order to load some https URLs served by old and misconfigured web servers, otherwise loading fails with the exception javax.net.ssl.SSLProtocolException: "handshake alert: unrecognized_name". + Controlling SNI extension activation can also be done with the JVM option jsse.enableSNIExtension, but in that case a server restart is required when you want to modify the setting and it is not customizable per http client (general or for remote Solr). +

+ +
+ +
+ General HTTP client + +

Configuration settings for the main HTTP client, used notably to crawl websites and communicate with other YaCy peers.

+ + #(http.outgoing.general.tls.sniExtension.enabled.readonly)#:: + + #(/http.outgoing.general.tls.sniExtension.enabled.readonly)# + +
+
+
+ +
+
+
+
+ +
+ Remote Solr HTTP client + +

Configuration settings for the specific HTTP client dedicated to communications with remote Solr servers (located on other YaCy peers or eventually owned by this one when it is configured to use a remote Solr index).

+ + #(http.outgoing.remoteSolr.tls.sniExtension.enabled.readonly)#:: + + #(/http.outgoing.remoteSolr.tls.sniExtension.enabled.readonly)# + +
+
+
+ +
+
+
+
+ +
+ + Changes will take effect immediately. +
+
\ No newline at end of file diff --git a/htroot/Settings_p.html b/htroot/Settings_p.html index 8892e1422..076bd8d13 100644 --- a/htroot/Settings_p.html +++ b/htroot/Settings_p.html @@ -29,6 +29,7 @@
  • URL/Web Proxy Access Settings
  • Remote Proxy (optional)
  • Debug/Analysis Settings
    • +
  • HTTP client Settings
    • diff --git a/htroot/Settings_p.java b/htroot/Settings_p.java index 20d853896..4f66d296c 100644 --- a/htroot/Settings_p.java +++ b/htroot/Settings_p.java @@ -25,7 +25,9 @@ import java.util.HashMap; import java.util.Iterator; +import net.yacy.cora.federate.solr.instance.RemoteInstance; import net.yacy.cora.protocol.RequestHeader; +import net.yacy.cora.protocol.http.HTTPClient; import net.yacy.data.TransactionManager; import net.yacy.http.ReferrerPolicy; import net.yacy.peers.Network; @@ -81,6 +83,8 @@ public final class Settings_p { prop.put("settingsTables", "Settings_Crawler.inc"); } else if (page.equals("debug")) { prop.put("settingsTables", "Settings_Debug.inc"); + } else if (page.equals("httpClient")) { + prop.put("settingsTables", "Settings_HttpClient.inc"); } else { prop.put("settingsTables", ""); } @@ -238,6 +242,23 @@ public final class Settings_p { prop.put("solrBinaryResponseChecked", env.getConfigBool(SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED, SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT) ? 1 : 0); + // HTTP client + prop.put(SwitchboardConstants.HTTP_OUTGOING_GENERAL_TLS_SNI_EXTENSION_ENABLED, + env.getConfigBool(SwitchboardConstants.HTTP_OUTGOING_GENERAL_TLS_SNI_EXTENSION_ENABLED, + HTTPClient.ENABLE_SNI_EXTENSION_DEFAULT)); + prop.put(SwitchboardConstants.HTTP_OUTGOING_REMOTE_SOLR_TLS_SNI_EXTENSION_ENABLED, + env.getConfigBool(SwitchboardConstants.HTTP_OUTGOING_REMOTE_SOLR_TLS_SNI_EXTENSION_ENABLED, + RemoteInstance.ENABLE_SNI_EXTENSION_DEFAULT)); + final String jvmSniExtensionSetting = System.getProperty("jsse.enableSNIExtension"); + prop.put(SwitchboardConstants.HTTP_OUTGOING_GENERAL_TLS_SNI_EXTENSION_ENABLED + ".readonly", + jvmSniExtensionSetting != null); + prop.put(SwitchboardConstants.HTTP_OUTGOING_GENERAL_TLS_SNI_EXTENSION_ENABLED + ".readonly_jvmSettingValue", + jvmSniExtensionSetting); + prop.put(SwitchboardConstants.HTTP_OUTGOING_REMOTE_SOLR_TLS_SNI_EXTENSION_ENABLED + ".readonly", + jvmSniExtensionSetting != null); + prop.put(SwitchboardConstants.HTTP_OUTGOING_REMOTE_SOLR_TLS_SNI_EXTENSION_ENABLED + ".readonly_jvmSettingValue", + jvmSniExtensionSetting); + /* For easier user understanding, the following flags controlling data sources selection * are rendered in the UI as checkboxes corresponding to enabled value when ticked */ prop.put("searchLocalDHTChecked", !env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_DHT_OFF, false) ? 1 : 0); diff --git a/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java b/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java index 5671836de..f4bbe8e5b 100644 --- a/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java +++ b/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java @@ -24,11 +24,15 @@ import java.io.IOException; import java.net.MalformedURLException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLParameters; +import javax.net.ssl.SSLSocket; import org.apache.commons.lang.StringUtils; import org.apache.http.Header; @@ -93,6 +97,15 @@ public class RemoteInstance implements SolrInstance { /** The connection manager holding the HTTP connections pool shared between remote Solr clients. */ public static final org.apache.http.impl.conn.PoolingClientConnectionManager CONNECTION_MANAGER = buildConnectionManager(); + /** Default setting to apply when the JVM system option jsse.enableSNIExtension is not defined */ + public static final boolean ENABLE_SNI_EXTENSION_DEFAULT = true; + + /** When true, Server Name Indication (SNI) extension is enabled on outgoing TLS connections. + * @see RFC 6066 definition + * @see JDK 1.7 bug on "unrecognized_name" warning for SNI */ + public static final AtomicBoolean ENABLE_SNI_EXTENSION = new AtomicBoolean( + Boolean.parseBoolean(System.getProperty("jsse.enableSNIExtension", Boolean.toString(ENABLE_SNI_EXTENSION_DEFAULT)))); + /** * Background daemon thread evicting expired idle connections from the pool. * This may be eventually already done by the pool itself on connection request, @@ -354,7 +367,18 @@ public class RemoteInstance implements SolrInstance { registry = new SchemeRegistry(); registry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); registry.register( - new Scheme("https", 443, new SSLSocketFactory(sslContext, AllowAllHostnameVerifier.INSTANCE))); + new Scheme("https", 443, new SSLSocketFactory(sslContext, AllowAllHostnameVerifier.INSTANCE) { + @Override + protected void prepareSocket(SSLSocket socket) throws IOException { + if(!ENABLE_SNI_EXTENSION.get()) { + /* Set the SSLParameters server names to empty so we don't use SNI extension. + * See https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html#ClientSNIExamples */ + final SSLParameters sslParams = socket.getSSLParameters(); + sslParams.setServerNames(Collections.emptyList()); + socket.setSSLParameters(sslParams); + } + } + })); } catch (final Exception e) { // Should not happen ConcurrentLog.warn("RemoteInstance", diff --git a/source/net/yacy/cora/protocol/http/HTTPClient.java b/source/net/yacy/cora/protocol/http/HTTPClient.java index ff131999b..1e5a66f2c 100644 --- a/source/net/yacy/cora/protocol/http/HTTPClient.java +++ b/source/net/yacy/cora/protocol/http/HTTPClient.java @@ -34,6 +34,7 @@ import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; +import java.util.Collections; import java.util.Locale; import java.util.Map; import java.util.Map.Entry; @@ -44,8 +45,11 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLParameters; +import javax.net.ssl.SSLSocket; import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; @@ -132,6 +136,15 @@ public class HTTPClient { /** The connection manager holding the configured connection pool for this client */ public static final PoolingHttpClientConnectionManager CONNECTION_MANAGER = initPoolingConnectionManager(); + /** Default setting to apply when the JVM system option jsse.enableSNIExtension is not defined */ + public static final boolean ENABLE_SNI_EXTENSION_DEFAULT = true; + + /** When true, Server Name Indication (SNI) extension is enabled on outgoing TLS connections. + * @see RFC 6066 definition + * @see JDK 1.7 bug on "unrecognized_name" warning for SNI */ + public static final AtomicBoolean ENABLE_SNI_EXTENSION = new AtomicBoolean( + Boolean.parseBoolean(System.getProperty("jsse.enableSNIExtension", Boolean.toString(ENABLE_SNI_EXTENSION_DEFAULT)))); + /** * Background daemon thread evicting expired idle connections from the pool. * This may be eventually already done by the pool itself on connection request, @@ -1086,10 +1099,21 @@ public class HTTPClient { // e.printStackTrace(); } - final SSLConnectionSocketFactory sslSF = new SSLConnectionSocketFactory( + return new SSLConnectionSocketFactory( sslContext, - new NoopHostnameVerifier()); - return sslSF; + new NoopHostnameVerifier()) { + + @Override + protected void prepareSocket(SSLSocket socket) throws IOException { + if(!ENABLE_SNI_EXTENSION.get()) { + /* Set the SSLParameters server names to empty so we don't use SNI extension. + * See https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html#ClientSNIExamples */ + final SSLParameters sslParams = socket.getSSLParameters(); + sslParams.setServerNames(Collections.emptyList()); + socket.setSSLParameters(sslParams); + } + } + }; } /** diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 6d834dca3..f436b6e66 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -477,6 +477,9 @@ public final class Switchboard extends serverSwitch { final int wordCacheMaxCount = (int) getConfigLong(SwitchboardConstants.WORDCACHE_MAX_COUNT, 20000); setConfig(SwitchboardConstants.WORDCACHE_MAX_COUNT, Integer.toString(wordCacheMaxCount)); + /* Init outgoing connections clients with user defined settings */ + initOutgoingConnectionSettings(); + /* Init outgoing connections pools with user defined settings */ initOutgoingConnectionPools(); @@ -1279,6 +1282,22 @@ public final class Switchboard extends serverSwitch { this.log.config("Finished Switchboard Initialization"); } + + /** + * Initialize outgoing connections custom settings + */ + public void initOutgoingConnectionSettings() { + final String systemEnableSniExt = System.getProperty("jsse.enableSNIExtension"); + if(systemEnableSniExt == null) { + /* Only apply custom configuration when the JVM system option jsse.enableSNIExtension is not defined */ + HTTPClient.ENABLE_SNI_EXTENSION + .set(getConfigBool(SwitchboardConstants.HTTP_OUTGOING_GENERAL_TLS_SNI_EXTENSION_ENABLED, + HTTPClient.ENABLE_SNI_EXTENSION_DEFAULT)); + + RemoteInstance.ENABLE_SNI_EXTENSION.set(getConfigBool(SwitchboardConstants.HTTP_OUTGOING_REMOTE_SOLR_TLS_SNI_EXTENSION_ENABLED, + RemoteInstance.ENABLE_SNI_EXTENSION_DEFAULT)); + } + } /** * Initialize outgoing connections pools with user defined settings diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java index b745edda9..4f9f9daa0 100644 --- a/source/net/yacy/search/SwitchboardConstants.java +++ b/source/net/yacy/search/SwitchboardConstants.java @@ -509,6 +509,13 @@ public final class SwitchboardConstants { /** Default setting value controlling the maximum number of simultaneously open outgoing HTTP connections in the remote Solr pool */ public static final int HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL_DEFAULT = 100; + + /** Key of the setting controlling whether TLS Server Name Indication (SNI) extension is enabled on outgoing HTTP connections in the general http client (net.yacy.cora.protocol.http.HTTPClient) */ + public static final String HTTP_OUTGOING_GENERAL_TLS_SNI_EXTENSION_ENABLED = "http.outgoing.general.tls.sniExtension.enabled"; + + /** Key of the setting controlling whether TLS Server Name Indication (SNI) extension is enabled on outgoing HTTP connections in the remote Solr http client (net.yacy.cora.federate.solr.instance.RemoteInstance) */ + public static final String HTTP_OUTGOING_REMOTE_SOLR_TLS_SNI_EXTENSION_ENABLED = "http.outgoing.remoteSolr.tls.sniExtension.enabled"; + /* diff --git a/startYACY.bat b/startYACY.bat index 776acccc0..0c72bd11d 100644 --- a/startYACY.bat +++ b/startYACY.bat @@ -18,8 +18,7 @@ if exist DATA\SETTINGS\httpProxy.conf GoTo :RENAMEINDEX if exist DATA\SETTINGS\yacy.conf GoTo :GETSTARTOPTS :STARTJAVA -set javacmd=%javacmd% -Djava.awt.headless=true -Dsolr.directoryFactory=solr.MMapDirectoryFactory -Dfile.encoding=UTF-8 -Djsse.enableSNIExtension=false -Rem -Djsse.enableSNIExtension=false fix a ssl problem in Java 1.7, see http://teknosrc.com/javax-net-ssl-sslprotocolexception-handshake-alert-unrecognized_name-solved/ +set javacmd=%javacmd% -Djava.awt.headless=true -Dsolr.directoryFactory=solr.MMapDirectoryFactory -Dfile.encoding=UTF-8 Rem Starting YaCy Echo Generated classpath:%CLASSPATH% diff --git a/startYACY.sh b/startYACY.sh index 876f5df14..3dea48cca 100755 --- a/startYACY.sh +++ b/startYACY.sh @@ -6,8 +6,7 @@ PIDFILE="yacy.pid" OS="`uname`" #get javastart args -JAVA_ARGS="-server -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djsse.enableSNIExtension=false"; -#rem -Djsse.enableSNIExtension=false fix a ssl problem in Java 1.7, see http://teknosrc.com/javax-net-ssl-sslprotocolexception-handshake-alert-unrecognized_name-solved/ +JAVA_ARGS="-server -Djava.awt.headless=true -Dfile.encoding=UTF-8"; #JAVA_ARGS="-verbose:gc -XX:+PrintGCTimeStamps -XX:+PrintGCDetails $JAVA_ARGS"; diff --git a/startYACY_debug.bat b/startYACY_debug.bat index a1ed0b0f4..bbce5915d 100644 --- a/startYACY_debug.bat +++ b/startYACY_debug.bat @@ -17,8 +17,7 @@ if exist DATA\SETTINGS\httpProxy.conf GoTo :RENAMEINDEX if exist DATA\SETTINGS\yacy.conf GoTo :GETSTARTOPTS :STARTJAVA -set javacmd=%javacmd% -XX:-UseGCOverheadLimit -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djsse.enableSNIExtension=false -Rem -Djsse.enableSNIExtension=false fix a ssl problem in Java 1.7, see http://teknosrc.com/javax-net-ssl-sslprotocolexception-handshake-alert-unrecognized_name-solved/ +set javacmd=%javacmd% -XX:-UseGCOverheadLimit -Djava.awt.headless=true -Dfile.encoding=UTF-8 Rem Starting YaCy Echo Generated classpath:%CLASSPATH%