Made SNI extension user configurable without the need for server restart

TLS Server Name Indication (SNI) extension activation can now be
configured with the new Settings_p.html?page=httpClient administration
page.
SNI extension is also now enabled by default, as in 2019 the
unrecognized_name(112) alert is more properly handled by major web
servers TLS implementations, following the RFC 6066 standard.

Related YaCy issues : #153 #189 and #272
JDK 1.7 bug :
https://bugs.java.com/bugdatabase/view_bug.do?bug_id=7127374
Apache httpd issue :
https://bz.apache.org/bugzilla/show_bug.cgi?id=56241
RFC 6066 : https://tools.ietf.org/html/rfc6066#section-3
pull/303/head
luccioman 6 years ago
parent e90405b6f0
commit a5771b1f14

@ -227,6 +227,19 @@ http.outgoing.pool.general.maxTotal = 200
# maximum number of simultaneously open outgoing HTTP connections in the remote Solr pool (net.yacy.cora.federate.solr.instance.RemoteInstance)
http.outgoing.pool.remoteSolr.maxTotal = 100
# TLS Server Name Indication (SNI) extension support for outgoing HTTP connections
# Must be enabled to load some https URLs (for websites deployed with different certificats and host names on the same shared IP address), otherwise loading fails with errors such as "Received fatal alert: handshake_failure"
# It can be necessary to disable it to load some https URLs served by old and misconfigured web servers, otherwise loading fails with the exception javax.net.ssl.SSLProtocolException: "handshake alert: unrecognized_name"
# In 2019, this last error should occur less frequently as in the early days of the extension implementation as its standard definition has evolved from RFC 4366 to RFC 6066 which revised how server implementations should handle the unrecognized_name(112) alert.
# Can also be configured with the JVM option jsse.enableSNIExtension, but in that case a server restart is required when you want to modify the setting and it is not customizable per http client (general or for remote Solr)
# Set to true to enable TLS Server Name Indication (SNI) extension on outgoing HTTP connections in the general http client (net.yacy.cora.protocol.http.HTTPClient)
http.outgoing.general.tls.sniExtension.enabled = true
# Set to true to enable TLS Server Name Indication (SNI) extension on outgoing HTTP connections in the remote Solr http client (net.yacy.cora.federate.solr.instance.RemoteInstance)
http.outgoing.remoteSolr.tls.sniExtension.enabled = true
# default root path for the file server
# may be overridden by the htdocs parameter
# users shall be encouraged to use the htdocs path for individual content,

@ -210,6 +210,8 @@
</table>
::<!-- 37: Compression settings changed -->
<p>Compression settings have been saved.</p>
::<!-- 38: HTTP client settings changed -->
<p>HTTP client settings have been saved.</p>
#(/info)#
<p></p>

@ -642,6 +642,25 @@ public class SettingsAck_p {
prop.put("info", "35");
return prop;
}
// HTTP client settings
if (post.containsKey("httpClientSettings")) {
// set backlink
prop.put("needsRestart_referer", "Settings_p.html?page=httpClient");
if(System.getProperty("jsse.enableSNIExtension") == null) {
/* Only apply custom SNI extension settings when the JVM system option jsse.enableSNIExtension is not defined */
env.setConfig(SwitchboardConstants.HTTP_OUTGOING_GENERAL_TLS_SNI_EXTENSION_ENABLED,
post.containsKey(SwitchboardConstants.HTTP_OUTGOING_GENERAL_TLS_SNI_EXTENSION_ENABLED));
env.setConfig(SwitchboardConstants.HTTP_OUTGOING_REMOTE_SOLR_TLS_SNI_EXTENSION_ENABLED,
post.containsKey(SwitchboardConstants.HTTP_OUTGOING_REMOTE_SOLR_TLS_SNI_EXTENSION_ENABLED));
}
sb.initOutgoingConnectionSettings();
prop.put("info", "38");
return prop;
}
// nothing made
prop.put("info", "1");//no information submitted

@ -0,0 +1,72 @@
<h3>HTTP client settings</h3>
<p>
You can configure here some advanced settings of the clients used by YaCy to handle outgoing HTTP connections.
<p>
<p id="sniExtensionInfo">
About Server Name Indication (SNI) :
this extension to the <abbr title="Transport Layer Security">TLS</abbr> protocol must be enabled to load some https URLs (for websites deployed with different certificats and host names on the same shared IP address), otherwise loading fails with errors such as <i>"Received fatal alert: handshake_failure"</i>.
But it can be necessary to disable it in order to load some https URLs served by old and misconfigured web servers, otherwise loading fails with the exception <i>javax.net.ssl.SSLProtocolException: "handshake alert: unrecognized_name"</i>.
Controlling <abbr title="Server Name Indication">SNI</abbr> extension activation can also be done with the JVM option <i>jsse.enableSNIExtension</i>, but in that case a server restart is required when you want to modify the setting and it is not customizable per http client (general or for remote Solr).
</p>
<form action="SettingsAck_p.html" method="post" enctype="multipart/form-data" class="form-horizontal">
<input type="hidden" name="transactionToken" value="#[transactionToken]#"/>
<fieldset>
<legend>General HTTP client</legend>
<p>Configuration settings for the main HTTP client, used notably to crawl websites and communicate with other YaCy peers.</p>
#(http.outgoing.general.tls.sniExtension.enabled.readonly)#::
<p class="alert alert-info" role="alert">
SNI extension support can not be defined here as it is currently configured by the JVM option <code>-Djsse.enableSNIExtension=#[jvmSettingValue]#</code>.
</p>
#(/http.outgoing.general.tls.sniExtension.enabled.readonly)#
<div class="form-group">
<div class="col-sm-4">
<div class="checkbox">
<label>
<input name="http.outgoing.general.tls.sniExtension.enabled" id="http.outgoing.general.tls.sniExtension.enabled"
type="checkbox" #(http.outgoing.general.tls.sniExtension.enabled)#::checked="checked"#(/http.outgoing.general.tls.sniExtension.enabled)#
#(http.outgoing.general.tls.sniExtension.enabled.readonly)#::disabled="disabled"#(/http.outgoing.general.tls.sniExtension.enabled.readonly)#
aria-describedby="sniExtensionInfo"/>
Enable <abbr title="Server Name Indication">SNI</abbr> extension to <abbr title="Transport Layer Security">TLS</abbr>
</label>
</div>
</div>
</div>
</fieldset>
<fieldset>
<legend>Remote Solr HTTP client</legend>
<p>Configuration settings for the specific HTTP client dedicated to communications with remote Solr servers (located on other YaCy peers or eventually owned by this one when it is configured to use a remote Solr index).</p>
#(http.outgoing.remoteSolr.tls.sniExtension.enabled.readonly)#::
<p class="alert alert-info" role="alert">
SNI extension support can not be defined here as it is currently configured by the JVM option <code>-Djsse.enableSNIExtension=#[jvmSettingValue]#</code>.
</p>
#(/http.outgoing.remoteSolr.tls.sniExtension.enabled.readonly)#
<div class="form-group">
<div class="col-sm-4">
<div class="checkbox">
<label>
<input name="http.outgoing.remoteSolr.tls.sniExtension.enabled" id="http.outgoing.remoteSolr.tls.sniExtension.enabled"
type="checkbox" #(http.outgoing.remoteSolr.tls.sniExtension.enabled)#::checked="checked"#(/http.outgoing.remoteSolr.tls.sniExtension.enabled)#
#(http.outgoing.remoteSolr.tls.sniExtension.enabled.readonly)#::disabled="disabled"#(/http.outgoing.remoteSolr.tls.sniExtension.enabled.readonly)#
aria-describedby="sniExtensionInfo"/>
Enable <abbr title="Server Name Indication">SNI</abbr> extension to <abbr title="Transport Layer Security">TLS</abbr>
</label>
</div>
</div>
</div>
</fieldset>
<div class="col-sm-6">
<input type="submit" class="btn btn-primary" name="httpClientSettings" value="Submit" aria-describedby="submitInfo"/>
<em id="submitInfo">Changes will take effect immediately.</em>
</div>
</form>

@ -29,6 +29,7 @@
<li><a href="?page=UrlProxyAccess">URL/Web Proxy Access Settings</a></li>
<li><a href="?page=proxy">Remote Proxy (optional)</a></li>
<li><a href="?page=debug">Debug/Analysis Settings</a></li>
<li><a href="?page=httpClient">HTTP client Settings</a></li>
</ul>
</td>
</tr>

@ -25,7 +25,9 @@
import java.util.HashMap;
import java.util.Iterator;
import net.yacy.cora.federate.solr.instance.RemoteInstance;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.http.HTTPClient;
import net.yacy.data.TransactionManager;
import net.yacy.http.ReferrerPolicy;
import net.yacy.peers.Network;
@ -81,6 +83,8 @@ public final class Settings_p {
prop.put("settingsTables", "Settings_Crawler.inc");
} else if (page.equals("debug")) {
prop.put("settingsTables", "Settings_Debug.inc");
} else if (page.equals("httpClient")) {
prop.put("settingsTables", "Settings_HttpClient.inc");
} else {
prop.put("settingsTables", "");
}
@ -238,6 +242,23 @@ public final class Settings_p {
prop.put("solrBinaryResponseChecked", env.getConfigBool(SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED,
SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT) ? 1 : 0);
// HTTP client
prop.put(SwitchboardConstants.HTTP_OUTGOING_GENERAL_TLS_SNI_EXTENSION_ENABLED,
env.getConfigBool(SwitchboardConstants.HTTP_OUTGOING_GENERAL_TLS_SNI_EXTENSION_ENABLED,
HTTPClient.ENABLE_SNI_EXTENSION_DEFAULT));
prop.put(SwitchboardConstants.HTTP_OUTGOING_REMOTE_SOLR_TLS_SNI_EXTENSION_ENABLED,
env.getConfigBool(SwitchboardConstants.HTTP_OUTGOING_REMOTE_SOLR_TLS_SNI_EXTENSION_ENABLED,
RemoteInstance.ENABLE_SNI_EXTENSION_DEFAULT));
final String jvmSniExtensionSetting = System.getProperty("jsse.enableSNIExtension");
prop.put(SwitchboardConstants.HTTP_OUTGOING_GENERAL_TLS_SNI_EXTENSION_ENABLED + ".readonly",
jvmSniExtensionSetting != null);
prop.put(SwitchboardConstants.HTTP_OUTGOING_GENERAL_TLS_SNI_EXTENSION_ENABLED + ".readonly_jvmSettingValue",
jvmSniExtensionSetting);
prop.put(SwitchboardConstants.HTTP_OUTGOING_REMOTE_SOLR_TLS_SNI_EXTENSION_ENABLED + ".readonly",
jvmSniExtensionSetting != null);
prop.put(SwitchboardConstants.HTTP_OUTGOING_REMOTE_SOLR_TLS_SNI_EXTENSION_ENABLED + ".readonly_jvmSettingValue",
jvmSniExtensionSetting);
/* For easier user understanding, the following flags controlling data sources selection
* are rendered in the UI as checkboxes corresponding to enabled value when ticked */
prop.put("searchLocalDHTChecked", !env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_DHT_OFF, false) ? 1 : 0);

@ -24,11 +24,15 @@ import java.io.IOException;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLParameters;
import javax.net.ssl.SSLSocket;
import org.apache.commons.lang.StringUtils;
import org.apache.http.Header;
@ -93,6 +97,15 @@ public class RemoteInstance implements SolrInstance {
/** The connection manager holding the HTTP connections pool shared between remote Solr clients. */
public static final org.apache.http.impl.conn.PoolingClientConnectionManager CONNECTION_MANAGER = buildConnectionManager();
/** Default setting to apply when the JVM system option jsse.enableSNIExtension is not defined */
public static final boolean ENABLE_SNI_EXTENSION_DEFAULT = true;
/** When true, Server Name Indication (SNI) extension is enabled on outgoing TLS connections.
* @see <a href="https://tools.ietf.org/html/rfc6066#section-3">RFC 6066 definition</a>
* @see <a href="https://bugs.java.com/bugdatabase/view_bug.do?bug_id=7127374">JDK 1.7 bug</a> on "unrecognized_name" warning for SNI */
public static final AtomicBoolean ENABLE_SNI_EXTENSION = new AtomicBoolean(
Boolean.parseBoolean(System.getProperty("jsse.enableSNIExtension", Boolean.toString(ENABLE_SNI_EXTENSION_DEFAULT))));
/**
* Background daemon thread evicting expired idle connections from the pool.
* This may be eventually already done by the pool itself on connection request,
@ -354,7 +367,18 @@ public class RemoteInstance implements SolrInstance {
registry = new SchemeRegistry();
registry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
registry.register(
new Scheme("https", 443, new SSLSocketFactory(sslContext, AllowAllHostnameVerifier.INSTANCE)));
new Scheme("https", 443, new SSLSocketFactory(sslContext, AllowAllHostnameVerifier.INSTANCE) {
@Override
protected void prepareSocket(SSLSocket socket) throws IOException {
if(!ENABLE_SNI_EXTENSION.get()) {
/* Set the SSLParameters server names to empty so we don't use SNI extension.
* See https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html#ClientSNIExamples */
final SSLParameters sslParams = socket.getSSLParameters();
sslParams.setServerNames(Collections.emptyList());
socket.setSSLParameters(sslParams);
}
}
}));
} catch (final Exception e) {
// Should not happen
ConcurrentLog.warn("RemoteInstance",

@ -34,6 +34,7 @@ import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.Collections;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
@ -44,8 +45,11 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.FutureTask;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLParameters;
import javax.net.ssl.SSLSocket;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
@ -132,6 +136,15 @@ public class HTTPClient {
/** The connection manager holding the configured connection pool for this client */
public static final PoolingHttpClientConnectionManager CONNECTION_MANAGER = initPoolingConnectionManager();
/** Default setting to apply when the JVM system option jsse.enableSNIExtension is not defined */
public static final boolean ENABLE_SNI_EXTENSION_DEFAULT = true;
/** When true, Server Name Indication (SNI) extension is enabled on outgoing TLS connections.
* @see <a href="https://tools.ietf.org/html/rfc6066#section-3">RFC 6066 definition</a>
* @see <a href="https://bugs.java.com/bugdatabase/view_bug.do?bug_id=7127374">JDK 1.7 bug</a> on "unrecognized_name" warning for SNI */
public static final AtomicBoolean ENABLE_SNI_EXTENSION = new AtomicBoolean(
Boolean.parseBoolean(System.getProperty("jsse.enableSNIExtension", Boolean.toString(ENABLE_SNI_EXTENSION_DEFAULT))));
/**
* Background daemon thread evicting expired idle connections from the pool.
* This may be eventually already done by the pool itself on connection request,
@ -1086,10 +1099,21 @@ public class HTTPClient {
// e.printStackTrace();
}
final SSLConnectionSocketFactory sslSF = new SSLConnectionSocketFactory(
return new SSLConnectionSocketFactory(
sslContext,
new NoopHostnameVerifier());
return sslSF;
new NoopHostnameVerifier()) {
@Override
protected void prepareSocket(SSLSocket socket) throws IOException {
if(!ENABLE_SNI_EXTENSION.get()) {
/* Set the SSLParameters server names to empty so we don't use SNI extension.
* See https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html#ClientSNIExamples */
final SSLParameters sslParams = socket.getSSLParameters();
sslParams.setServerNames(Collections.emptyList());
socket.setSSLParameters(sslParams);
}
}
};
}
/**

@ -477,6 +477,9 @@ public final class Switchboard extends serverSwitch {
final int wordCacheMaxCount = (int) getConfigLong(SwitchboardConstants.WORDCACHE_MAX_COUNT, 20000);
setConfig(SwitchboardConstants.WORDCACHE_MAX_COUNT, Integer.toString(wordCacheMaxCount));
/* Init outgoing connections clients with user defined settings */
initOutgoingConnectionSettings();
/* Init outgoing connections pools with user defined settings */
initOutgoingConnectionPools();
@ -1279,6 +1282,22 @@ public final class Switchboard extends serverSwitch {
this.log.config("Finished Switchboard Initialization");
}
/**
* Initialize outgoing connections custom settings
*/
public void initOutgoingConnectionSettings() {
final String systemEnableSniExt = System.getProperty("jsse.enableSNIExtension");
if(systemEnableSniExt == null) {
/* Only apply custom configuration when the JVM system option jsse.enableSNIExtension is not defined */
HTTPClient.ENABLE_SNI_EXTENSION
.set(getConfigBool(SwitchboardConstants.HTTP_OUTGOING_GENERAL_TLS_SNI_EXTENSION_ENABLED,
HTTPClient.ENABLE_SNI_EXTENSION_DEFAULT));
RemoteInstance.ENABLE_SNI_EXTENSION.set(getConfigBool(SwitchboardConstants.HTTP_OUTGOING_REMOTE_SOLR_TLS_SNI_EXTENSION_ENABLED,
RemoteInstance.ENABLE_SNI_EXTENSION_DEFAULT));
}
}
/**
* Initialize outgoing connections pools with user defined settings

@ -509,6 +509,13 @@ public final class SwitchboardConstants {
/** Default setting value controlling the maximum number of simultaneously open outgoing HTTP connections in the remote Solr pool */
public static final int HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL_DEFAULT = 100;
/** Key of the setting controlling whether TLS Server Name Indication (SNI) extension is enabled on outgoing HTTP connections in the general http client (net.yacy.cora.protocol.http.HTTPClient) */
public static final String HTTP_OUTGOING_GENERAL_TLS_SNI_EXTENSION_ENABLED = "http.outgoing.general.tls.sniExtension.enabled";
/** Key of the setting controlling whether TLS Server Name Indication (SNI) extension is enabled on outgoing HTTP connections in the remote Solr http client (net.yacy.cora.federate.solr.instance.RemoteInstance) */
public static final String HTTP_OUTGOING_REMOTE_SOLR_TLS_SNI_EXTENSION_ENABLED = "http.outgoing.remoteSolr.tls.sniExtension.enabled";
/*

@ -18,8 +18,7 @@ if exist DATA\SETTINGS\httpProxy.conf GoTo :RENAMEINDEX
if exist DATA\SETTINGS\yacy.conf GoTo :GETSTARTOPTS
:STARTJAVA
set javacmd=%javacmd% -Djava.awt.headless=true -Dsolr.directoryFactory=solr.MMapDirectoryFactory -Dfile.encoding=UTF-8 -Djsse.enableSNIExtension=false
Rem -Djsse.enableSNIExtension=false fix a ssl problem in Java 1.7, see http://teknosrc.com/javax-net-ssl-sslprotocolexception-handshake-alert-unrecognized_name-solved/
set javacmd=%javacmd% -Djava.awt.headless=true -Dsolr.directoryFactory=solr.MMapDirectoryFactory -Dfile.encoding=UTF-8
Rem Starting YaCy
Echo Generated classpath:%CLASSPATH%

@ -6,8 +6,7 @@ PIDFILE="yacy.pid"
OS="`uname`"
#get javastart args
JAVA_ARGS="-server -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djsse.enableSNIExtension=false";
#rem -Djsse.enableSNIExtension=false fix a ssl problem in Java 1.7, see http://teknosrc.com/javax-net-ssl-sslprotocolexception-handshake-alert-unrecognized_name-solved/
JAVA_ARGS="-server -Djava.awt.headless=true -Dfile.encoding=UTF-8";
#JAVA_ARGS="-verbose:gc -XX:+PrintGCTimeStamps -XX:+PrintGCDetails $JAVA_ARGS";

@ -17,8 +17,7 @@ if exist DATA\SETTINGS\httpProxy.conf GoTo :RENAMEINDEX
if exist DATA\SETTINGS\yacy.conf GoTo :GETSTARTOPTS
:STARTJAVA
set javacmd=%javacmd% -XX:-UseGCOverheadLimit -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djsse.enableSNIExtension=false
Rem -Djsse.enableSNIExtension=false fix a ssl problem in Java 1.7, see http://teknosrc.com/javax-net-ssl-sslprotocolexception-handshake-alert-unrecognized_name-solved/
set javacmd=%javacmd% -XX:-UseGCOverheadLimit -Djava.awt.headless=true -Dfile.encoding=UTF-8
Rem Starting YaCy
Echo Generated classpath:%CLASSPATH%

Loading…
Cancel
Save