boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added SolrCloud access mode and configuration

Browse Source
pull/1/head
Michael Peter Christen 11 years ago
parent 8514bffc22
commit d07cdd8c3b
6 changed files with 123 additions and 72 deletions
Show Stats Download Patch File Download Diff File

7
htroot/IndexFederated_p.html
Unescape View File

@ -64,7 +64,12 @@
When a search request is made, all servers are accessed synchronously and the result is combined.</dd> When a search request is made, all servers are accessed synchronously and the result is combined.</dd>
<dt class="TableCellDark"></dt> <dt class="TableCellDark"></dt>
<dd>Sharding Method<br/><input type="text" size="50" maxlength="50" value="#[solr.indexing.sharding]#" name="solr.indexing.sharding" id="solr.indexing.sharding" disabled="disabled"/></dd> <dd>Sharding Method<br/>
<select id="solr.indexing.sharding" name="solr.indexing.sharding">
#{solr.indexing.sharding.methods}#
<option value="#[method]#" #(selected)#::selected="selected"#(/selected)#>#[method]#: #[description]#</option>
#{/solr.indexing.sharding.methods}#
</select></dd>
<dt class="TableCellDark"></dt> <dt class="TableCellDark"></dt>
<dd><input type="checkbox" name="solr.indexing.solrremote.writeenabled" id="solr_indexing_solrremote_writeenabled" #(solr.indexing.solrremote.writeenabled.checked)#:: checked="checked"#(/solr.indexing.solrremote.writeenabled.checked)#/> write-enabled (if unchecked, the remote server(s) will only be used as search peers)</dd> <dd><input type="checkbox" name="solr.indexing.solrremote.writeenabled" id="solr_indexing_solrremote_writeenabled" #(solr.indexing.solrremote.writeenabled.checked)#:: checked="checked"#(/solr.indexing.solrremote.writeenabled.checked)#/> write-enabled (if unchecked, the remote server(s) will only be used as search peers)</dd>

19
htroot/IndexFederated_p.java
Unescape View File

@ -28,6 +28,7 @@ import org.apache.solr.common.SolrException;
import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.federate.solr.connector.RemoteSolrConnector; import net.yacy.cora.federate.solr.connector.RemoteSolrConnector;
import net.yacy.cora.federate.solr.connector.ShardSelection;
import net.yacy.cora.federate.solr.connector.SolrConnector; import net.yacy.cora.federate.solr.connector.SolrConnector;
import net.yacy.cora.federate.solr.instance.RemoteInstance; import net.yacy.cora.federate.solr.instance.RemoteInstance;
import net.yacy.cora.federate.solr.instance.ShardInstance; import net.yacy.cora.federate.solr.instance.ShardInstance;
@ -110,7 +111,9 @@ public class IndexFederated_p {
} }
solrurls = s.toString().trim(); solrurls = s.toString().trim();
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, solrurls); env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, solrurls);
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, post.get("solr.indexing.sharding", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, "modulo-host-md5"))); String shardMethodName = post.get("solr.indexing.sharding", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name()));
ShardSelection.Method shardMethod = ShardSelection.Method.valueOf(shardMethodName);
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, shardMethod.name());
if (solrRemoteWasOn && !solrRemoteIsOnAfterwards) { if (solrRemoteWasOn && !solrRemoteIsOnAfterwards) {
// switch off // switch off
@ -133,7 +136,7 @@ public class IndexFederated_p {
try { try {
if (usesolr) { if (usesolr) {
ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout); ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout);
sb.index.fulltext().connectRemoteSolr(instances, writeEnabled); sb.index.fulltext().connectRemoteSolr(instances, shardMethod, writeEnabled);
} else { } else {
sb.index.fulltext().disconnectRemoteSolr(); sb.index.fulltext().disconnectRemoteSolr();
} }
@ -175,8 +178,16 @@ public class IndexFederated_p {
prop.put(SwitchboardConstants.CORE_SERVICE_CITATION + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, false) ? 1 : 0); prop.put(SwitchboardConstants.CORE_SERVICE_CITATION + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, false) ? 1 : 0);
prop.put(SwitchboardConstants.CORE_SERVICE_WEBGRAPH + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, false) ? 1 : 0); prop.put(SwitchboardConstants.CORE_SERVICE_WEBGRAPH + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, false) ? 1 : 0);
prop.put("solr.indexing.solrremote.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) ? 1 : 0); prop.put("solr.indexing.solrremote.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) ? 1 : 0);
prop.put("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr").replace(",", "\n")); prop.put("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr").replace(",", "\n"));
prop.put("solr.indexing.sharding", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, "modulo-host-md5")); String thisShardingMethodName = env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name());
int mc = 0;
for (ShardSelection.Method method: ShardSelection.Method.values()) {
prop.put("solr.indexing.sharding.methods_" + mc + "_method", method.name());
prop.put("solr.indexing.sharding.methods_" + mc + "_description", method.description);
prop.put("solr.indexing.sharding.methods_" + mc + "_selected", method.name().equals(thisShardingMethodName) ? 1 : 0);
mc++;
}
prop.put("solr.indexing.sharding.methods", mc);
prop.put("solr.indexing.solrremote.writeenabled.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_WRITEENABLED, true)); prop.put("solr.indexing.solrremote.writeenabled.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_WRITEENABLED, true));
prop.put("solr.indexing.lazy.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, true) ? 1 : 0); prop.put("solr.indexing.lazy.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, true) ? 1 : 0);

86
source/net/yacy/cora/federate/solr/connector/ShardSelection.java
Unescape View File

@ -21,91 +21,115 @@
package net.yacy.cora.federate.solr.connector; package net.yacy.cora.federate.solr.connector;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.security.MessageDigest; import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.ConcurrentLog;
import net.yacy.search.schema.CollectionSchema; import net.yacy.search.schema.CollectionSchema;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField; import org.apache.solr.common.SolrInputField;
public class ShardSelection { public class ShardSelection implements Iterable<SolrServer> {
private final Method method; // the sharding method private final Method method; // the sharding method
private final AtomicLong chardID; // the next id that shall be given away private final AtomicLong shardID; // the next id that shall be given away
private final int dimension; // the number of chards private final int dimension; // the number of chards
private final ArrayList<SolrServer> server;
public enum Method { public enum Method {
MODULO_HOST_MD5, ROUND_ROBIN; MODULO_HOST_MD5("hash-based calculation of storage targets, select all for retrieval"),
ROUND_ROBIN("round-robin of storage targets, select all for retrieval"),
SOLRCLOUD("round-robin of storage targets and round-robin for retrieval");
public final String description;
private Method(final String description) {
this.description = description;
}
} }
public ShardSelection(final Method method, final int dimension) { public ShardSelection(final ArrayList<SolrServer> server, final Method method) {
this.server = server;
this.method = method; this.method = method;
this.dimension = dimension; this.dimension = server.size();
this.chardID = new AtomicLong(0); this.shardID = new AtomicLong(0);
} }
public Method getMethod() {
return this.method;
}
private int selectRoundRobin() { private int selectRoundRobin() {
return (int) (this.chardID.getAndIncrement() % this.dimension); int rr = (int) (this.shardID.getAndIncrement() % this.dimension);
if (this.shardID.get() < 0) this.shardID.set(0);
return rr;
} }
public int select(final SolrInputDocument solrdoc) throws IOException { public SolrServer server4write(final SolrInputDocument solrdoc) throws IOException {
if (this.method == Method.MODULO_HOST_MD5) { if (this.method == Method.MODULO_HOST_MD5) {
SolrInputField sif = solrdoc.getField(CollectionSchema.host_s.getSolrFieldName()); SolrInputField sif = solrdoc.getField(CollectionSchema.host_s.getSolrFieldName());
if (sif != null) { if (sif != null) {
final String host = (String) sif.getValue(); final String host = (String) sif.getValue();
return selectHost(host); if (host != null && host.length() > 0) return server4write(host);
} }
sif = solrdoc.getField(CollectionSchema.sku.getSolrFieldName()); sif = solrdoc.getField(CollectionSchema.sku.getSolrFieldName());
if (sif != null) { if (sif != null) {
final String url = (String) sif.getValue(); final String url = (String) sif.getValue();
try { if (url != null && url.length() > 0) try {
return selectURL(url); return server4write(new URL(url));
} catch (final IOException e) { } catch (final IOException e) {
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
return 0; return this.server.get(0);
} }
} }
return 0; return this.server.get(0);
} }
// finally if no method matches use ROUND_ROBIN // finally if no method matches use ROUND_ROBIN
return selectRoundRobin(); return this.server.get(selectRoundRobin());
} }
public int selectHost(final String host) throws IOException { public SolrServer server4write(final String host) throws IOException {
if (host == null) throw new IOException("sharding - host url, host empty: " + host);
if (host.indexOf("://") >= 0) return server4write(new URL(host)); // security catch for accidantly using the wrong method
if (this.method == Method.MODULO_HOST_MD5) { if (this.method == Method.MODULO_HOST_MD5) {
if (host == null) throw new IOException("sharding - host url, host empty: " + host);
try { try {
final MessageDigest digest = MessageDigest.getInstance("MD5"); final MessageDigest digest = MessageDigest.getInstance("MD5");
digest.update(ASCII.getBytes(host)); digest.update(ASCII.getBytes(host));
final byte[] md5 = digest.digest(); final byte[] md5 = digest.digest();
return (0xff & md5[0]) % this.dimension; return this.server.get((0xff & md5[0]) % this.dimension);
} catch (final NoSuchAlgorithmException e) { } catch (final NoSuchAlgorithmException e) {
throw new IOException("sharding - no md5 available: " + e.getMessage()); throw new IOException("sharding - no md5 available: " + e.getMessage());
} }
} }
// finally if no method matches use ROUND_ROBIN // finally if no method matches use ROUND_ROBIN
return (int) (this.chardID.getAndIncrement() % this.dimension); return this.server.get(selectRoundRobin());
} }
public int selectURL(final String sku) throws IOException { public SolrServer server4write(final URL url) throws IOException {
if (this.method == Method.MODULO_HOST_MD5) { return server4write(url.getHost());
try { }
final URL url = new URL(sku);
final String host = url.getHost(); public List<SolrServer> server4read() {
return selectHost(host); if (this.method == Method.MODULO_HOST_MD5 || this.method == Method.ROUND_ROBIN) return this.server; // return all
} catch (final MalformedURLException e) { // this is a SolrCloud, we select just one of the SolrCloud server(s)
throw new IOException("sharding - bad url: " + sku); ArrayList<SolrServer> a = new ArrayList<>(1);
} a.add(this.server.get(selectRoundRobin()));
} return a;
}
// finally if no method matches use ROUND_ROBIN /**
return (int) (this.chardID.getAndIncrement() % this.dimension); * return all solr server
*/
@Override
public Iterator<SolrServer> iterator() {
return this.server.iterator();
} }
} }

70
source/net/yacy/cora/federate/solr/instance/ServerShard.java
Unescape View File

@ -51,13 +51,11 @@ public class ServerShard extends SolrServer {
_dummyOKResponse.setResponse(new NamedList<Object>()); _dummyOKResponse.setResponse(new NamedList<Object>());
} }
private final ArrayList<SolrServer> server; private final ShardSelection shards;
private final ShardSelection sharding;
private final boolean writeEnabled; private final boolean writeEnabled;
public ServerShard(ArrayList<SolrServer> server, final ShardSelection.Method method, final boolean writeEnabled) { public ServerShard(ArrayList<SolrServer> server, final ShardSelection.Method method, final boolean writeEnabled) {
this.server = server; this.shards = new ShardSelection(server, method);
this.sharding = new ShardSelection(method, this.server.size());
this.writeEnabled = writeEnabled; this.writeEnabled = writeEnabled;
} }
@ -70,7 +68,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse add(Collection<SolrInputDocument> docs) throws SolrServerException, IOException { public UpdateResponse add(Collection<SolrInputDocument> docs) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse; if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null; UpdateResponse ur = null;
for (SolrInputDocument doc: docs) ur = server.get(this.sharding.select(doc)).add(doc); for (SolrInputDocument doc: docs) ur = this.shards.server4write(doc).add(doc);
return ur; // TODO: this accumlation of update responses is wrong, but sufficient (because we do not evaluate it) return ur; // TODO: this accumlation of update responses is wrong, but sufficient (because we do not evaluate it)
} }
@ -85,7 +83,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse add(Collection<SolrInputDocument> docs, int commitWithinMs) throws SolrServerException, IOException { public UpdateResponse add(Collection<SolrInputDocument> docs, int commitWithinMs) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse; if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null; UpdateResponse ur = null;
for (SolrInputDocument doc: docs) ur = server.get(this.sharding.select(doc)).add(doc, commitWithinMs); for (SolrInputDocument doc: docs) ur = this.shards.server4write(doc).add(doc, commitWithinMs);
return ur; return ur;
} }
@ -98,7 +96,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse addBeans(Collection<?> beans ) throws SolrServerException, IOException { public UpdateResponse addBeans(Collection<?> beans ) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse; if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null; UpdateResponse ur = null;
for (SolrServer s: server) ur = s.addBeans(beans); for (SolrServer s: this.shards) ur = s.addBeans(beans);
return ur; return ur;
} }
@ -113,7 +111,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse addBeans(Collection<?> beans, int commitWithinMs) throws SolrServerException, IOException { public UpdateResponse addBeans(Collection<?> beans, int commitWithinMs) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse; if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null; UpdateResponse ur = null;
for (SolrServer s: server) ur = s.addBeans(beans, commitWithinMs); for (SolrServer s: this.shards) ur = s.addBeans(beans, commitWithinMs);
return ur; return ur;
} }
@ -125,7 +123,7 @@ public class ServerShard extends SolrServer {
@Override @Override
public UpdateResponse add(SolrInputDocument doc) throws SolrServerException, IOException { public UpdateResponse add(SolrInputDocument doc) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse; if (!this.writeEnabled) return _dummyOKResponse;
return server.get(this.sharding.select(doc)).add(doc); return this.shards.server4write(doc).add(doc);
} }
/** /**
@ -138,7 +136,7 @@ public class ServerShard extends SolrServer {
@Override @Override
public UpdateResponse add(SolrInputDocument doc, int commitWithinMs) throws SolrServerException, IOException { public UpdateResponse add(SolrInputDocument doc, int commitWithinMs) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse; if (!this.writeEnabled) return _dummyOKResponse;
return server.get(this.sharding.select(doc)).add(doc, commitWithinMs); return this.shards.server4write(doc).add(doc, commitWithinMs);
} }
/** /**
@ -150,7 +148,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse addBean(Object obj) throws IOException, SolrServerException { public UpdateResponse addBean(Object obj) throws IOException, SolrServerException {
if (!this.writeEnabled) return _dummyOKResponse; if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null; UpdateResponse ur = null;
for (SolrServer s: server) ur = s.addBean(obj); for (SolrServer s: this.shards) ur = s.addBean(obj);
return ur; return ur;
} }
@ -165,7 +163,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse addBean(Object obj, int commitWithinMs) throws IOException, SolrServerException { public UpdateResponse addBean(Object obj, int commitWithinMs) throws IOException, SolrServerException {
if (!this.writeEnabled) return _dummyOKResponse; if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null; UpdateResponse ur = null;
for (SolrServer s: server) ur = s.addBean(obj, commitWithinMs); for (SolrServer s: this.shards) ur = s.addBean(obj, commitWithinMs);
return ur; return ur;
} }
@ -179,7 +177,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse commit() throws SolrServerException, IOException { public UpdateResponse commit() throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse; if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null; UpdateResponse ur = null;
for (SolrServer s: server) ur = s.commit(); for (SolrServer s: this.shards) ur = s.commit();
return ur; return ur;
} }
@ -195,7 +193,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse optimize() throws SolrServerException, IOException { public UpdateResponse optimize() throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse; if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null; UpdateResponse ur = null;
for (SolrServer s: server) ur = s.optimize(); for (SolrServer s: this.shards) ur = s.optimize();
return ur; return ur;
} }
@ -209,7 +207,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse commit(boolean waitFlush, boolean waitSearcher) throws SolrServerException, IOException { public UpdateResponse commit(boolean waitFlush, boolean waitSearcher) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse; if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null; UpdateResponse ur = null;
for (SolrServer s: server) ur = s.commit(waitFlush, waitSearcher); for (SolrServer s: this.shards) ur = s.commit(waitFlush, waitSearcher);
return ur; return ur;
} }
@ -224,7 +222,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse commit(boolean waitFlush, boolean waitSearcher, boolean softCommit) throws SolrServerException, IOException { public UpdateResponse commit(boolean waitFlush, boolean waitSearcher, boolean softCommit) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse; if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null; UpdateResponse ur = null;
for (SolrServer s: server) ur = s.commit(waitFlush, waitSearcher, softCommit); for (SolrServer s: this.shards) ur = s.commit(waitFlush, waitSearcher, softCommit);
return ur; return ur;
} }
@ -240,7 +238,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse optimize(boolean waitFlush, boolean waitSearcher) throws SolrServerException, IOException { public UpdateResponse optimize(boolean waitFlush, boolean waitSearcher) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse; if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null; UpdateResponse ur = null;
for (SolrServer s: server) ur = s.optimize(waitFlush, waitSearcher); for (SolrServer s: this.shards) ur = s.optimize(waitFlush, waitSearcher);
return ur; return ur;
} }
@ -257,7 +255,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse optimize(boolean waitFlush, boolean waitSearcher, int maxSegments) throws SolrServerException, IOException { public UpdateResponse optimize(boolean waitFlush, boolean waitSearcher, int maxSegments) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse; if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null; UpdateResponse ur = null;
for (SolrServer s: server) ur = s.optimize(waitFlush, waitSearcher, maxSegments); for (SolrServer s: this.shards) ur = s.optimize(waitFlush, waitSearcher, maxSegments);
return ur; return ur;
} }
@ -273,7 +271,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse rollback() throws SolrServerException, IOException { public UpdateResponse rollback() throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse; if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null; UpdateResponse ur = null;
for (SolrServer s: server) ur = s.rollback(); for (SolrServer s: this.shards) ur = s.rollback();
return ur; return ur;
} }
@ -286,7 +284,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse deleteById(String id) throws SolrServerException, IOException { public UpdateResponse deleteById(String id) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse; if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null; UpdateResponse ur = null;
for (SolrServer s: server) ur = s.deleteById(id); for (SolrServer s: this.shards.server4read()) ur = s.deleteById(id);
return ur; return ur;
} }
@ -301,7 +299,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse deleteById(String id, int commitWithinMs) throws SolrServerException, IOException { public UpdateResponse deleteById(String id, int commitWithinMs) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse; if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null; UpdateResponse ur = null;
for (SolrServer s: server) ur = s.deleteById(id, commitWithinMs); for (SolrServer s: this.shards.server4read()) ur = s.deleteById(id, commitWithinMs);
return ur; return ur;
} }
@ -314,7 +312,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse deleteById(List<String> ids) throws SolrServerException, IOException { public UpdateResponse deleteById(List<String> ids) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse; if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null; UpdateResponse ur = null;
for (SolrServer s: server) ur = s.deleteById(ids); for (SolrServer s: this.shards.server4read()) ur = s.deleteById(ids);
return ur; return ur;
} }
@ -329,7 +327,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse deleteById(List<String> ids, int commitWithinMs) throws SolrServerException, IOException { public UpdateResponse deleteById(List<String> ids, int commitWithinMs) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse; if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null; UpdateResponse ur = null;
for (SolrServer s: server) ur = s.deleteById(ids, commitWithinMs); for (SolrServer s: this.shards.server4read()) ur = s.deleteById(ids, commitWithinMs);
return ur; return ur;
} }
@ -342,7 +340,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse deleteByQuery(String query) throws SolrServerException, IOException { public UpdateResponse deleteByQuery(String query) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse; if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null; UpdateResponse ur = null;
for (SolrServer s: server) ur = s.deleteByQuery(query); for (SolrServer s: this.shards.server4read()) ur = s.deleteByQuery(query);
return ur; return ur;
} }
@ -357,7 +355,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse deleteByQuery(String query, int commitWithinMs) throws SolrServerException, IOException { public UpdateResponse deleteByQuery(String query, int commitWithinMs) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse; if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null; UpdateResponse ur = null;
for (SolrServer s: server) ur = s.deleteByQuery(query, commitWithinMs); for (SolrServer s: this.shards.server4read()) ur = s.deleteByQuery(query, commitWithinMs);
return ur; return ur;
} }
@ -367,7 +365,7 @@ public class ServerShard extends SolrServer {
*/ */
@Override @Override
public SolrPingResponse ping() throws SolrServerException, IOException { public SolrPingResponse ping() throws SolrServerException, IOException {
for (SolrServer s: server) { for (SolrServer s: this.shards) {
SolrPingResponse spr = s.ping(); SolrPingResponse spr = s.ping();
if (spr != null) return spr; if (spr != null) return spr;
} }
@ -380,11 +378,15 @@ public class ServerShard extends SolrServer {
*/ */
@Override @Override
public QueryResponse query(final SolrParams params) throws SolrServerException { public QueryResponse query(final SolrParams params) throws SolrServerException {
List<SolrServer> qs = this.shards.server4read();
if (qs.size() == 1) {
return qs.get(0).query(params);
}
final Collection<QueryResponse> qrl = new ConcurrentLinkedQueue<QueryResponse>();
// concurrently call all shards // concurrently call all shards
final Collection<QueryResponse> qrl = new ConcurrentLinkedQueue<QueryResponse>();
List<Thread> t = new ArrayList<Thread>(); List<Thread> t = new ArrayList<Thread>();
for (final SolrServer s: server) { for (final SolrServer s: qs) {
Thread t0 = new Thread() { Thread t0 = new Thread() {
@Override @Override
public void run() { public void run() {
@ -414,11 +416,15 @@ public class ServerShard extends SolrServer {
*/ */
@Override @Override
public QueryResponse query(final SolrParams params, final METHOD method) throws SolrServerException { public QueryResponse query(final SolrParams params, final METHOD method) throws SolrServerException {
List<SolrServer> qs = this.shards.server4read();
if (qs.size() == 1) {
return qs.get(0).query(params, method);
}
final Collection<QueryResponse> qrl = new ConcurrentLinkedQueue<QueryResponse>(); final Collection<QueryResponse> qrl = new ConcurrentLinkedQueue<QueryResponse>();
// concurrently call all shards // concurrently call all shards
List<Thread> t = new ArrayList<Thread>(); List<Thread> t = new ArrayList<Thread>();
for (final SolrServer s: server) { for (final SolrServer s: qs) {
Thread t0 = new Thread() { Thread t0 = new Thread() {
@Override @Override
public void run() { public void run() {
@ -464,14 +470,14 @@ public class ServerShard extends SolrServer {
@Override @Override
public NamedList<Object> request(final SolrRequest request) throws SolrServerException, IOException { public NamedList<Object> request(final SolrRequest request) throws SolrServerException, IOException {
ResponseAccumulator acc = new ResponseAccumulator(); ResponseAccumulator acc = new ResponseAccumulator();
for (SolrServer s: server) acc.addResponse(s.request(request)); for (SolrServer s: this.shards.server4read()) acc.addResponse(s.request(request));
return acc.getAccumulatedResponse(); return acc.getAccumulatedResponse();
} }
@Override @Override
public DocumentObjectBinder getBinder() { public DocumentObjectBinder getBinder() {
DocumentObjectBinder db; DocumentObjectBinder db;
for (SolrServer s: server) { for (SolrServer s: this.shards) {
db = s.getBinder(); db = s.getBinder();
if (db != null) return db; if (db != null) return db;
} }
@ -480,7 +486,7 @@ public class ServerShard extends SolrServer {
@Override @Override
public void shutdown() { public void shutdown() {
for (SolrServer s: server) { for (SolrServer s: this.shards) {
s.shutdown(); s.shutdown();
} }
} }

9
source/net/yacy/search/Switchboard.java
Unescape View File

@ -98,6 +98,7 @@ import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.federate.solr.FailCategory; import net.yacy.cora.federate.solr.FailCategory;
import net.yacy.cora.federate.solr.Ranking; import net.yacy.cora.federate.solr.Ranking;
import net.yacy.cora.federate.solr.SchemaConfiguration; import net.yacy.cora.federate.solr.SchemaConfiguration;
import net.yacy.cora.federate.solr.connector.ShardSelection;
import net.yacy.cora.federate.solr.connector.SolrConnector.Metadata; import net.yacy.cora.federate.solr.connector.SolrConnector.Metadata;
import net.yacy.cora.federate.solr.instance.RemoteInstance; import net.yacy.cora.federate.solr.instance.RemoteInstance;
import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.federate.yacy.CacheStrategy;
@ -527,7 +528,9 @@ public final class Switchboard extends serverSwitch {
if (usesolr && solrurls != null && solrurls.length() > 0) { if (usesolr && solrurls != null && solrurls.length() > 0) {
try { try {
ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout); ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout);
this.index.fulltext().connectRemoteSolr(instances, writeEnabled); String shardMethodName = getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name());
ShardSelection.Method shardMethod = ShardSelection.Method.valueOf(shardMethodName);
this.index.fulltext().connectRemoteSolr(instances, shardMethod, writeEnabled);
} catch (final IOException e ) { } catch (final IOException e ) {
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
} }
@ -1371,7 +1374,9 @@ public final class Switchboard extends serverSwitch {
if (usesolr && solrurls != null && solrurls.length() > 0) { if (usesolr && solrurls != null && solrurls.length() > 0) {
try { try {
ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout); ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout);
this.index.fulltext().connectRemoteSolr(instances, writeEnabled); String shardMethodName = getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name());
ShardSelection.Method shardMethod = ShardSelection.Method.valueOf(shardMethodName);
this.index.fulltext().connectRemoteSolr(instances, shardMethod, writeEnabled);
} catch (final IOException e ) { } catch (final IOException e ) {
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
} }

4
source/net/yacy/search/index/Fulltext.java
Unescape View File

@ -148,8 +148,8 @@ public final class Fulltext {
return this.solrInstances.isConnectedRemote(); return this.solrInstances.isConnectedRemote();
} }
public void connectRemoteSolr(final ArrayList<RemoteInstance> instances, final boolean writeEnabled) { public void connectRemoteSolr(final ArrayList<RemoteInstance> instances, final ShardSelection.Method shardMethod, final boolean writeEnabled) {
this.solrInstances.connectRemote(new ShardInstance(instances, ShardSelection.Method.MODULO_HOST_MD5, writeEnabled)); this.solrInstances.connectRemote(new ShardInstance(instances, shardMethod, writeEnabled));
} }
public void disconnectRemoteSolr() { public void disconnectRemoteSolr() {

Write Preview
Loading…
Cancel
Save