From d3a88eaecbedfbfed47f38833a77f9f543c1bd21 Mon Sep 17 00:00:00 2001 From: orbiter Date: Mon, 24 Feb 2014 20:26:02 +0100 Subject: [PATCH] introducing ConcurrentUpdateSolrServer for remote solr servers. Scaling of write buffers and update queue size is made according to assigned memory. --- .../solr/instance/InstanceMirror.java | 2 +- .../solr/instance/RemoteInstance.java | 26 +++++++++++-------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/source/net/yacy/cora/federate/solr/instance/InstanceMirror.java b/source/net/yacy/cora/federate/solr/instance/InstanceMirror.java index e87287398..3642fa145 100644 --- a/source/net/yacy/cora/federate/solr/instance/InstanceMirror.java +++ b/source/net/yacy/cora/federate/solr/instance/InstanceMirror.java @@ -161,7 +161,7 @@ public class InstanceMirror { if (msc != null) return msc; EmbeddedSolrConnector esc = getEmbeddedConnector(corename); RemoteSolrConnector rsc = getRemoteConnector(corename); - msc = new ConcurrentUpdateSolrConnector(new MirrorSolrConnector(esc, rsc), 100, 100000, Runtime.getRuntime().availableProcessors()); + msc = new ConcurrentUpdateSolrConnector(new MirrorSolrConnector(esc, rsc), RemoteInstance.queueSizeByMemory(), 100000, Runtime.getRuntime().availableProcessors()); this.mirrorConnectorCache.put(corename, msc); return msc; } diff --git a/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java b/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java index 4f058dc62..9ca938a6e 100644 --- a/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java +++ b/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java @@ -31,6 +31,7 @@ import java.util.Map; import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.protocol.Domains; import net.yacy.cora.util.ConcurrentLog; +import net.yacy.kelondro.util.MemoryControl; import net.yacy.search.schema.CollectionSchema; import net.yacy.search.schema.WebgraphSchema; @@ -49,7 +50,7 @@ import org.apache.http.client.entity.GzipDecompressingEntity; import org.apache.http.impl.auth.BasicScheme; import org.apache.http.protocol.HttpContext; import org.apache.solr.client.solrj.SolrServer; -import org.apache.solr.client.solrj.impl.HttpSolrServer; +import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrServer; @SuppressWarnings("deprecation") public class RemoteInstance implements SolrInstance { @@ -57,9 +58,9 @@ public class RemoteInstance implements SolrInstance { private String solrurl; private final Object client; // not declared as org.apache.http.impl.client.DefaultHttpClient to avoid warnings during compilation. TODO: switch to org.apache.http.impl.client.HttpClientBuilder private final String defaultCoreName; - private final HttpSolrServer defaultServer; + private final ConcurrentUpdateSolrServer defaultServer; private final Collection coreNames; - private final Map server; + private final Map server; private final int timeout; public static ArrayList getShardInstances(final String urlList, Collection coreNames, String defaultCoreName, final int timeout) throws IOException { @@ -75,7 +76,7 @@ public class RemoteInstance implements SolrInstance { public RemoteInstance(final String url, final Collection coreNames, final String defaultCoreName, final int timeout) throws IOException { this.timeout = timeout; - this.server= new HashMap(); + this.server= new HashMap(); this.solrurl = url == null ? "http://127.0.0.1:8983/solr/" : url; // that should work for the example configuration of solr 4.x.x this.coreNames = coreNames == null ? new ArrayList() : coreNames; if (this.coreNames.size() == 0) { @@ -178,7 +179,7 @@ public class RemoteInstance implements SolrInstance { this.client = null; } - this.defaultServer = (HttpSolrServer) getServer(this.defaultCoreName); + this.defaultServer = (ConcurrentUpdateSolrServer) getServer(this.defaultCoreName); if (this.defaultServer == null) throw new IOException("cannot connect to url " + url + " and connect core " + defaultCoreName); } @@ -211,7 +212,7 @@ public class RemoteInstance implements SolrInstance { @Override public SolrServer getServer(String name) { // try to get the server from the cache - HttpSolrServer s = this.server.get(name); + ConcurrentUpdateSolrServer s = this.server.get(name); if (s != null) return s; // create new http server if (this.client != null) { @@ -226,14 +227,14 @@ public class RemoteInstance implements SolrInstance { String solrpath = u.getPath(); String p = "http://" + host + ":" + port + solrpath; ConcurrentLog.info("RemoteSolrConnector", "connecting Solr authenticated with url:" + p); - s = new HttpSolrServer(p, ((org.apache.http.impl.client.DefaultHttpClient) this.client)); + s = new ConcurrentUpdateSolrServer(p, ((org.apache.http.impl.client.DefaultHttpClient) this.client), 10, Runtime.getRuntime().availableProcessors()); } else { ConcurrentLog.info("RemoteSolrConnector", "connecting Solr with url:" + this.solrurl + name); - s = new HttpSolrServer(this.solrurl + name); + s = new ConcurrentUpdateSolrServer(this.solrurl + name, queueSizeByMemory(), Runtime.getRuntime().availableProcessors()); } - s.setAllowCompression(true); - s.setConnectionTimeout(this.timeout); - s.setMaxRetries(1); // Solr-Doc: No more than 1 recommended (depreciated) + //s.setAllowCompression(true); + s.setSoTimeout(this.timeout); + //s.setMaxRetries(1); // Solr-Doc: No more than 1 recommended (depreciated) s.setSoTimeout(this.timeout); this.server.put(name, s); return s; @@ -244,4 +245,7 @@ public class RemoteInstance implements SolrInstance { if (this.client != null) ((org.apache.http.impl.client.DefaultHttpClient) this.client).getConnectionManager().shutdown(); } + public static int queueSizeByMemory() { + return (int) Math.max(1, MemoryControl.maxMemory() / 1024 / 1024 / 12); + } }