boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

introducing ConcurrentUpdateSolrServer for remote solr servers.

Browse Source 
Scaling of write buffers and update queue size is made according to
assigned memory.
pull/1/head
orbiter 11 years ago
parent c3f6c06f2c
commit d3a88eaecb
2 changed files with 16 additions and 12 deletions
Show Stats Download Patch File Download Diff File

2
source/net/yacy/cora/federate/solr/instance/InstanceMirror.java
Unescape View File

@ -161,7 +161,7 @@ public class InstanceMirror {
if (msc != null) return msc;
EmbeddedSolrConnector esc = getEmbeddedConnector(corename);
RemoteSolrConnector rsc = getRemoteConnector(corename);
msc = new ConcurrentUpdateSolrConnector(new MirrorSolrConnector(esc, rsc), 100, 100000, Runtime.getRuntime().availableProcessors());
msc = new ConcurrentUpdateSolrConnector(new MirrorSolrConnector(esc, rsc), RemoteInstance.queueSizeByMemory(), 100000, Runtime.getRuntime().availableProcessors());
this.mirrorConnectorCache.put(corename, msc);
return msc;
}

26
source/net/yacy/cora/federate/solr/instance/RemoteInstance.java
Unescape View File

@ -31,6 +31,7 @@ import java.util.Map;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.search.schema.CollectionSchema;
import net.yacy.search.schema.WebgraphSchema;
@ -49,7 +50,7 @@ import org.apache.http.client.entity.GzipDecompressingEntity;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.protocol.HttpContext;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrServer;
@SuppressWarnings("deprecation")
public class RemoteInstance implements SolrInstance {
@ -57,9 +58,9 @@ public class RemoteInstance implements SolrInstance {
private String solrurl;
private final Object client; // not declared as org.apache.http.impl.client.DefaultHttpClient to avoid warnings during compilation. TODO: switch to org.apache.http.impl.client.HttpClientBuilder
private final String defaultCoreName;
private final HttpSolrServer defaultServer;
private final ConcurrentUpdateSolrServer defaultServer;
private final Collection<String> coreNames;
private final Map<String, HttpSolrServer> server;
private final Map<String, ConcurrentUpdateSolrServer> server;
private final int timeout;
public static ArrayList<RemoteInstance> getShardInstances(final String urlList, Collection<String> coreNames, String defaultCoreName, final int timeout) throws IOException {
@ -75,7 +76,7 @@ public class RemoteInstance implements SolrInstance {
public RemoteInstance(final String url, final Collection<String> coreNames, final String defaultCoreName, final int timeout) throws IOException {
this.timeout = timeout;
this.server= new HashMap<String, HttpSolrServer>();
this.server= new HashMap<String, ConcurrentUpdateSolrServer>();
this.solrurl = url == null ? "http://127.0.0.1:8983/solr/" : url; // that should work for the example configuration of solr 4.x.x
this.coreNames = coreNames == null ? new ArrayList<String>() : coreNames;
if (this.coreNames.size() == 0) {
@ -178,7 +179,7 @@ public class RemoteInstance implements SolrInstance {
this.client = null;
}
this.defaultServer = (HttpSolrServer) getServer(this.defaultCoreName);
this.defaultServer = (ConcurrentUpdateSolrServer) getServer(this.defaultCoreName);
if (this.defaultServer == null) throw new IOException("cannot connect to url " + url + " and connect core " + defaultCoreName);
}
@ -211,7 +212,7 @@ public class RemoteInstance implements SolrInstance {
@Override
public SolrServer getServer(String name) {
// try to get the server from the cache
HttpSolrServer s = this.server.get(name);
ConcurrentUpdateSolrServer s = this.server.get(name);
if (s != null) return s;
// create new http server
if (this.client != null) {
@ -226,14 +227,14 @@ public class RemoteInstance implements SolrInstance {
String solrpath = u.getPath();
String p = "http://" + host + ":" + port + solrpath;
ConcurrentLog.info("RemoteSolrConnector", "connecting Solr authenticated with url:" + p);
s = new HttpSolrServer(p, ((org.apache.http.impl.client.DefaultHttpClient) this.client));
s = new ConcurrentUpdateSolrServer(p, ((org.apache.http.impl.client.DefaultHttpClient) this.client), 10, Runtime.getRuntime().availableProcessors());
} else {
ConcurrentLog.info("RemoteSolrConnector", "connecting Solr with url:" + this.solrurl + name);
s = new HttpSolrServer(this.solrurl + name);
s = new ConcurrentUpdateSolrServer(this.solrurl + name, queueSizeByMemory(), Runtime.getRuntime().availableProcessors());
}
s.setAllowCompression(true);
s.setConnectionTimeout(this.timeout);
s.setMaxRetries(1); // Solr-Doc: No more than 1 recommended (depreciated)
//s.setAllowCompression(true);
s.setSoTimeout(this.timeout);
//s.setMaxRetries(1); // Solr-Doc: No more than 1 recommended (depreciated)
s.setSoTimeout(this.timeout);
this.server.put(name, s);
return s;
@ -244,4 +245,7 @@ public class RemoteInstance implements SolrInstance {
if (this.client != null) ((org.apache.http.impl.client.DefaultHttpClient) this.client).getConnectionManager().shutdown();
}
public static int queueSizeByMemory() {
return (int) Math.max(1, MemoryControl.maxMemory() / 1024 / 1024 / 12);
}
}

Write Preview
Loading…
Cancel
Save