boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Use the heavy ConcurrentUpdateSolrClient only when necessary

Browse Source 
Prefer the lightweight HttpSolrClient when no updates are performed on
the remote Solr instance, as recommended by Solr documentation itself.
pull/183/head
luccioman 7 years ago
parent bd4cfeda3f
commit 40e8c7b89b
4 changed files with 74 additions and 41 deletions
Show Stats Download Patch File Download Diff File

3
source/net/yacy/cora/federate/SolrFederateSearchConnector.java
Unescape View File

@ -102,7 +102,8 @@ public class SolrFederateSearchConnector extends AbstractFederateSearchConnector
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);
}
RemoteInstance instance = new RemoteInstance(baseurl, remotecorename, corename, 20000, trustSelfSignedOnAuthenticatedServer);
RemoteInstance instance = new RemoteInstance(baseurl, remotecorename, corename, 20000,
trustSelfSignedOnAuthenticatedServer, Long.MAX_VALUE, false);
try {
boolean useBinaryResponseWriter = SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT;
if (Switchboard.getSwitchboard() != null) {

108
source/net/yacy/cora/federate/solr/instance/RemoteInstance.java
Unescape View File

@ -61,6 +61,7 @@ import org.apache.http.ssl.SSLContextBuilder;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.update.UpdateShardHandler.IdleConnectionsEvictor;
@ -109,14 +110,33 @@ public class RemoteInstance implements SolrInstance {
/** A custom scheme registry allowing https connections to servers using self-signed certificate */
private static final SchemeRegistry SCHEME_REGISTRY = buildTrustSelfSignedSchemeRegistry();
/** Solr server URL */
private String solrurl;
/** HTTP client used to request the Solr server */
private final HttpClient client;
/** Default Solr core name */
private final String defaultCoreName;
private final ConcurrentUpdateSolrClient defaultServer;
/** Solr client for the default core */
private final SolrClient defaultServer;
/** Solr core names for the main collection and the webgraph */
private final Collection<String> coreNames;
private final Map<String, ConcurrentUpdateSolrClient> server;
/** Map from Solr core names to SolrClient instances */
private final Map<String, SolrClient> server;
/** Connection timeout in milliseconds */
private final int timeout;
/**
* When true, the instance will be used for update operations. The Solr client
* is adjusted for better performance of multiple updates.
*/
private final boolean concurrentUpdates;
/**
* @param urlList
* the list of URLs of remote Solr shard instances. Must not be null.
@ -146,6 +166,8 @@ public class RemoteInstance implements SolrInstance {
}
/**
* Build a new instance optimized for concurrent updates, with no limit on responses size.
*
* @param url
* the remote Solr URL. A default localhost URL is assumed when null.
* @param coreNames
@ -162,7 +184,7 @@ public class RemoteInstance implements SolrInstance {
*/
public RemoteInstance(final String url, final Collection<String> coreNames, final String defaultCoreName,
final int timeout, final boolean trustSelfSignedOnAuthenticatedServer) throws IOException {
this(url, coreNames, defaultCoreName, timeout, trustSelfSignedOnAuthenticatedServer, Long.MAX_VALUE);
this(url, coreNames, defaultCoreName, timeout, trustSelfSignedOnAuthenticatedServer, Long.MAX_VALUE, true);
}
/**
@ -181,13 +203,18 @@ public class RemoteInstance implements SolrInstance {
* maximum acceptable decompressed size in bytes for a response from
* the remote Solr server. Negative value or Long.MAX_VALUE means no
* limit.
* @param concurrentUpdates
* when true, the instance will be used for update operations. The
* Solr client is adjusted for better performance of multiple
* updates.
* @throws IOException
* when a connection could not be opened to the remote Solr instance
*/
public RemoteInstance(final String url, final Collection<String> coreNames, final String defaultCoreName,
final int timeout, final boolean trustSelfSignedOnAuthenticatedServer, final long maxBytesPerResponse) throws IOException {
final int timeout, final boolean trustSelfSignedOnAuthenticatedServer, final long maxBytesPerResponse, final boolean concurrentUpdates) throws IOException {
this.timeout = timeout;
this.server= new HashMap<String, ConcurrentUpdateSolrClient>();
this.concurrentUpdates = concurrentUpdates;
this.server= new HashMap<String, SolrClient>();
this.solrurl = url == null ? "http://127.0.0.1:8983/solr/" : url; // that should work for the example configuration of solr 4.x.x
this.coreNames = coreNames == null ? new ArrayList<String>() : coreNames;
if (this.coreNames.size() == 0) {
@ -248,6 +275,13 @@ public class RemoteInstance implements SolrInstance {
params.set(HttpClientUtil.PROP_FOLLOW_REDIRECTS, false);
/* Accept gzip compression of responses to reduce network usage */
params.set(HttpClientUtil.PROP_ALLOW_COMPRESSION, true);
/* Set the maximum time to establish a connection to the remote server */
params.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, this.timeout);
/* Set the maximum time between data packets reception once a connection has been established */
params.set(HttpClientUtil.PROP_SO_TIMEOUT, this.timeout);
this.client = HttpClientUtil.createClient(params, CONNECTION_MANAGER);
if(this.client instanceof DefaultHttpClient) {
if(this.client.getParams() != null) {
@ -266,7 +300,7 @@ public class RemoteInstance implements SolrInstance {
}
}
this.defaultServer = (ConcurrentUpdateSolrClient) getServer(this.defaultCoreName);
this.defaultServer = getServer(this.defaultCoreName);
if (this.defaultServer == null) throw new IOException("cannot connect to url " + url + " and connect core " + defaultCoreName);
}
@ -469,10 +503,13 @@ public class RemoteInstance implements SolrInstance {
return this.defaultServer;
}
/**
* @param name the name of the Solr core
*/
@Override
public SolrClient getServer(String name) {
public SolrClient getServer(final String name) {
// try to get the server from the cache
ConcurrentUpdateSolrClient s = this.server.get(name);
SolrClient s = this.server.get(name);
if (s != null) return s;
// create new http server
final MultiProtocolURL u;
@ -481,37 +518,29 @@ public class RemoteInstance implements SolrInstance {
} catch (final MalformedURLException e) {
return null;
}
if (this.client != null) {
final String solrServerURL;
if(StringUtils.isNotEmpty(u.getUserInfo())) {
/* Remove user authentication info from the URL, as authentication will be handled by the custom http client */
String host = u.getHost();
int port = u.getPort();
String solrpath = u.getPath();
solrServerURL = u.getProtocol() + "://" + host + ":" + port + solrpath;
ConcurrentLog.info("RemoteSolrConnector", "connecting Solr authenticated with url : " + u);
} else {
solrServerURL = u.toString();
ConcurrentLog.info("RemoteSolrConnector", "connecting Solr with url : " + u);
}
ConcurrentUpdateSolrClient.Builder builder = new ConcurrentUpdateSolrClient.Builder(solrServerURL);
final String solrServerURL;
if(StringUtils.isNotEmpty(u.getUserInfo())) {
/* Remove user authentication info from the URL, as authentication will be handled by the custom http client */
String host = u.getHost();
int port = u.getPort();
String solrpath = u.getPath();
solrServerURL = u.getProtocol() + "://" + host + ":" + port + solrpath;
ConcurrentLog.info("RemoteSolrConnector", "connecting Solr authenticated with url : " + u);
} else {
solrServerURL = u.toString();
ConcurrentLog.info("RemoteSolrConnector", "connecting Solr with url : " + u);
}
if(this.concurrentUpdates) {
final ConcurrentUpdateSolrClient.Builder builder = new ConcurrentUpdateSolrClient.Builder(solrServerURL);
builder.withHttpClient(this.client);
builder.withQueueSize(queueSizeByMemory());
builder.withThreadCount(Runtime.getRuntime().availableProcessors());
s = builder.build();
builder.withQueueSize(queueSizeByMemory());
builder.withThreadCount(Runtime.getRuntime().availableProcessors());
s = builder.build();
} else {
ConcurrentLog.info("RemoteSolrConnector", "connecting Solr with url : " + this.solrurl + name);
ConcurrentUpdateSolrClient.Builder builder = new ConcurrentUpdateSolrClient.Builder(u.toString());
builder.withQueueSize(queueSizeByMemory());
builder.withThreadCount(Runtime.getRuntime().availableProcessors());
s = builder.build();
final HttpSolrClient.Builder builder = new HttpSolrClient.Builder(solrServerURL);
builder.withHttpClient(this.client);
s = builder.build();
}
//s.setAllowCompression(true);
/* Set the maximum time to establish a connection to the remote server */
s.setConnectionTimeout(this.timeout);
/* Set the maximum time between data packets reception one a connection has been established */
s.setSoTimeout(this.timeout);
//s.setMaxRetries(1); // Solr-Doc: No more than 1 recommended (depreciated)
this.server.put(name, s);
return s;
}
@ -524,13 +553,16 @@ public class RemoteInstance implements SolrInstance {
*/
@Override
public void close() {
for (final ConcurrentUpdateSolrClient solrClient : this.server.values()) {
for (final SolrClient solrClient : this.server.values()) {
/*
* Close every open Solr client : this is important as it shutdowns client's
* internal asynchronous tasks executor. To release the common connection
* manager, see closeConnectionManager().
*/
solrClient.close();
try {
solrClient.close();
} catch (final IOException ignored) {
}
}
}

2
source/net/yacy/peers/Protocol.java
Unescape View File

@ -1115,7 +1115,7 @@ public final class Protocol {
* - the remote peer is a malicious one and would like to trigger a deny of service */
final long maxBytesPerResponse = MemoryControl.available() / 4;
this.instance = new RemoteInstance(this.targetBaseURL, null, "solr", this.timeout, trustSelfSignedOnAuthenticatedServer, maxBytesPerResponse); // this is a 'patch configuration' which considers 'solr' as default collection
this.instance = new RemoteInstance(this.targetBaseURL, null, "solr", this.timeout, trustSelfSignedOnAuthenticatedServer, maxBytesPerResponse, false); // this is a 'patch configuration' which considers 'solr' as default collection
try {
boolean useBinaryResponseWriter = SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT;
if (Switchboard.getSwitchboard() != null) {

2
source/net/yacy/search/AutoSearch.java
Unescape View File

@ -270,7 +270,7 @@ public class AutoSearch extends AbstractBusyThread {
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);
RemoteInstance instance = new RemoteInstance(targetBaseURL + "/solr/", null, null, 10000, trustSelfSignedOnAuthenticatedServer); // this is a 'patch configuration' which considers 'solr' as default collection
RemoteInstance instance = new RemoteInstance(targetBaseURL + "/solr/", null, null, 10000, trustSelfSignedOnAuthenticatedServer, Long.MAX_VALUE, false); // this is a 'patch configuration' which considers 'solr' as default collection
try {
SolrConnector solrConnector = new RemoteSolrConnector(instance,
sb.getConfigBool(SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED,

Write Preview
Loading…
Cancel
Save