From ee17bd0b69cb651c3b147aca5d56dca03ee1f4f6 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Fri, 27 Dec 2013 02:55:21 +0100 Subject: [PATCH] added option to attach remote solr servers in read-only mode --- defaults/yacy.init | 1 + htroot/IndexFederated_p.java | 4 +-- .../solr/connector/RemoteSolrConnector.java | 2 +- .../federate/solr/instance/ServerShard.java | 34 +++++++++++++++++-- .../federate/solr/instance/ShardInstance.java | 8 +++-- source/net/yacy/search/Switchboard.java | 6 ++-- .../net/yacy/search/SwitchboardConstants.java | 23 +++++++------ source/net/yacy/search/index/Fulltext.java | 4 +-- 8 files changed, 58 insertions(+), 24 deletions(-) diff --git a/defaults/yacy.init b/defaults/yacy.init index 88ca29529..5708df76b 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -1090,6 +1090,7 @@ federated.service.solr.indexing.sharding = MODULO_HOST_MD5 # the lazy attribute causes that fields containing "" or 0 are not added and not written federated.service.solr.indexing.lazy = true federated.service.solr.indexing.timeout = 60000 +federated.service.solr.indexing.writeEnabled = true # temporary definition of backend services to use. # After the migration a rwi+solr combination is used, the solr contains the content of the previously used metadata-db. diff --git a/htroot/IndexFederated_p.java b/htroot/IndexFederated_p.java index 2dec9cf7e..4947b4c5c 100644 --- a/htroot/IndexFederated_p.java +++ b/htroot/IndexFederated_p.java @@ -133,11 +133,12 @@ public class IndexFederated_p { // switch on final boolean usesolr = sb.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) & solrurls.length() > 0; final int solrtimeout = sb.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 10000); + final boolean writeEnabled = sb.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_WRITEENABLED, true); try { if (usesolr) { ArrayList instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout); - sb.index.fulltext().connectRemoteSolr(instances); + sb.index.fulltext().connectRemoteSolr(instances, writeEnabled); } else { sb.index.fulltext().disconnectRemoteSolr(); } @@ -182,7 +183,6 @@ public class IndexFederated_p { prop.put("solr.indexing.solrremote.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) ? 1 : 0); prop.put("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr").replace(",", "\n")); prop.put("solr.indexing.sharding", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, "modulo-host-md5")); - prop.put("solr.indexing.lazy.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, true) ? 1 : 0); // return rewrite properties diff --git a/source/net/yacy/cora/federate/solr/connector/RemoteSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/RemoteSolrConnector.java index e01e83e49..f2ad6c6dc 100644 --- a/source/net/yacy/cora/federate/solr/connector/RemoteSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/RemoteSolrConnector.java @@ -121,7 +121,7 @@ public class RemoteSolrConnector extends SolrServerConnector implements SolrConn RemoteInstance instance = new RemoteInstance("http://127.0.0.1:8983/solr/", null, "collection1", 10000); ArrayList instances = new ArrayList(); instances.add(instance); - solr = new RemoteSolrConnector(new ShardInstance(instances, ShardSelection.Method.MODULO_HOST_MD5), true, "solr"); + solr = new RemoteSolrConnector(new ShardInstance(instances, ShardSelection.Method.MODULO_HOST_MD5, true), true, "solr"); solr.clear(); final File exampleDir = new File("test/parsertest/"); long t, t0, a = 0; diff --git a/source/net/yacy/cora/federate/solr/instance/ServerShard.java b/source/net/yacy/cora/federate/solr/instance/ServerShard.java index 80c174a1d..3123752d7 100644 --- a/source/net/yacy/cora/federate/solr/instance/ServerShard.java +++ b/source/net/yacy/cora/federate/solr/instance/ServerShard.java @@ -45,13 +45,20 @@ import net.yacy.cora.federate.solr.connector.ShardSelection; public class ServerShard extends SolrServer { private static final long serialVersionUID = -3524175189049786312L; + private static final UpdateResponse _dummyOKResponse = new UpdateResponse(); + static { + _dummyOKResponse.setElapsedTime(0); + _dummyOKResponse.setResponse(new NamedList()); + } - ArrayList server; - ShardSelection sharding; + private final ArrayList server; + private final ShardSelection sharding; + private final boolean writeEnabled; - public ServerShard(ArrayList server, final ShardSelection.Method method) { + public ServerShard(ArrayList server, final ShardSelection.Method method, final boolean writeEnabled) { this.server = server; this.sharding = new ShardSelection(method, this.server.size()); + this.writeEnabled = writeEnabled; } /** @@ -60,6 +67,7 @@ public class ServerShard extends SolrServer { * @throws IOException If there is a low-level I/O error. */ public UpdateResponse add(Collection docs) throws SolrServerException, IOException { + if (!this.writeEnabled) return _dummyOKResponse; UpdateResponse ur = null; for (SolrInputDocument doc: docs) ur = server.get(this.sharding.select(doc)).add(doc); return ur; // TODO: this accumlation of update responses is wrong, but sufficient (because we do not evaluate it) @@ -73,6 +81,7 @@ public class ServerShard extends SolrServer { * @since solr 3.5 */ public UpdateResponse add(Collection docs, int commitWithinMs) throws SolrServerException, IOException { + if (!this.writeEnabled) return _dummyOKResponse; UpdateResponse ur = null; for (SolrInputDocument doc: docs) ur = server.get(this.sharding.select(doc)).add(doc, commitWithinMs); return ur; @@ -84,6 +93,7 @@ public class ServerShard extends SolrServer { * @throws IOException If there is a low-level I/O error. */ public UpdateResponse addBeans(Collection beans ) throws SolrServerException, IOException { + if (!this.writeEnabled) return _dummyOKResponse; UpdateResponse ur = null; for (SolrServer s: server) ur = s.addBeans(beans); return ur; @@ -97,6 +107,7 @@ public class ServerShard extends SolrServer { * @since solr 3.5 */ public UpdateResponse addBeans(Collection beans, int commitWithinMs) throws SolrServerException, IOException { + if (!this.writeEnabled) return _dummyOKResponse; UpdateResponse ur = null; for (SolrServer s: server) ur = s.addBeans(beans, commitWithinMs); return ur; @@ -108,6 +119,7 @@ public class ServerShard extends SolrServer { * @throws IOException If there is a low-level I/O error. */ public UpdateResponse add(SolrInputDocument doc) throws SolrServerException, IOException { + if (!this.writeEnabled) return _dummyOKResponse; return server.get(this.sharding.select(doc)).add(doc); } @@ -119,6 +131,7 @@ public class ServerShard extends SolrServer { * @since solr 3.5 */ public UpdateResponse add(SolrInputDocument doc, int commitWithinMs) throws SolrServerException, IOException { + if (!this.writeEnabled) return _dummyOKResponse; return server.get(this.sharding.select(doc)).add(doc, commitWithinMs); } @@ -128,6 +141,7 @@ public class ServerShard extends SolrServer { * @throws IOException If there is a low-level I/O error. */ public UpdateResponse addBean(Object obj) throws IOException, SolrServerException { + if (!this.writeEnabled) return _dummyOKResponse; UpdateResponse ur = null; for (SolrServer s: server) ur = s.addBean(obj); return ur; @@ -141,6 +155,7 @@ public class ServerShard extends SolrServer { * @since solr 3.5 */ public UpdateResponse addBean(Object obj, int commitWithinMs) throws IOException, SolrServerException { + if (!this.writeEnabled) return _dummyOKResponse; UpdateResponse ur = null; for (SolrServer s: server) ur = s.addBean(obj, commitWithinMs); return ur; @@ -153,6 +168,7 @@ public class ServerShard extends SolrServer { * @throws IOException If there is a low-level I/O error. */ public UpdateResponse commit() throws SolrServerException, IOException { + if (!this.writeEnabled) return _dummyOKResponse; UpdateResponse ur = null; for (SolrServer s: server) ur = s.commit(); return ur; @@ -167,6 +183,7 @@ public class ServerShard extends SolrServer { * @throws IOException If there is a low-level I/O error. */ public UpdateResponse optimize() throws SolrServerException, IOException { + if (!this.writeEnabled) return _dummyOKResponse; UpdateResponse ur = null; for (SolrServer s: server) ur = s.optimize(); return ur; @@ -179,6 +196,7 @@ public class ServerShard extends SolrServer { * @throws IOException If there is a low-level I/O error. */ public UpdateResponse commit(boolean waitFlush, boolean waitSearcher) throws SolrServerException, IOException { + if (!this.writeEnabled) return _dummyOKResponse; UpdateResponse ur = null; for (SolrServer s: server) ur = s.commit(waitFlush, waitSearcher); return ur; @@ -192,6 +210,7 @@ public class ServerShard extends SolrServer { * @throws IOException If there is a low-level I/O error. */ public UpdateResponse commit(boolean waitFlush, boolean waitSearcher, boolean softCommit) throws SolrServerException, IOException { + if (!this.writeEnabled) return _dummyOKResponse; UpdateResponse ur = null; for (SolrServer s: server) ur = s.commit(waitFlush, waitSearcher, softCommit); return ur; @@ -206,6 +225,7 @@ public class ServerShard extends SolrServer { * @throws IOException If there is a low-level I/O error. */ public UpdateResponse optimize(boolean waitFlush, boolean waitSearcher) throws SolrServerException, IOException { + if (!this.writeEnabled) return _dummyOKResponse; UpdateResponse ur = null; for (SolrServer s: server) ur = s.optimize(waitFlush, waitSearcher); return ur; @@ -221,6 +241,7 @@ public class ServerShard extends SolrServer { * @throws IOException If there is a low-level I/O error. */ public UpdateResponse optimize(boolean waitFlush, boolean waitSearcher, int maxSegments) throws SolrServerException, IOException { + if (!this.writeEnabled) return _dummyOKResponse; UpdateResponse ur = null; for (SolrServer s: server) ur = s.optimize(waitFlush, waitSearcher, maxSegments); return ur; @@ -235,6 +256,7 @@ public class ServerShard extends SolrServer { * @throws IOException If there is a low-level I/O error. */ public UpdateResponse rollback() throws SolrServerException, IOException { + if (!this.writeEnabled) return _dummyOKResponse; UpdateResponse ur = null; for (SolrServer s: server) ur = s.rollback(); return ur; @@ -246,6 +268,7 @@ public class ServerShard extends SolrServer { * @throws IOException If there is a low-level I/O error. */ public UpdateResponse deleteById(String id) throws SolrServerException, IOException { + if (!this.writeEnabled) return _dummyOKResponse; UpdateResponse ur = null; for (SolrServer s: server) ur = s.deleteById(id); return ur; @@ -259,6 +282,7 @@ public class ServerShard extends SolrServer { * @since 3.6 */ public UpdateResponse deleteById(String id, int commitWithinMs) throws SolrServerException, IOException { + if (!this.writeEnabled) return _dummyOKResponse; UpdateResponse ur = null; for (SolrServer s: server) ur = s.deleteById(id, commitWithinMs); return ur; @@ -270,6 +294,7 @@ public class ServerShard extends SolrServer { * @throws IOException If there is a low-level I/O error. */ public UpdateResponse deleteById(List ids) throws SolrServerException, IOException { + if (!this.writeEnabled) return _dummyOKResponse; UpdateResponse ur = null; for (SolrServer s: server) ur = s.deleteById(ids); return ur; @@ -283,6 +308,7 @@ public class ServerShard extends SolrServer { * @since 3.6 */ public UpdateResponse deleteById(List ids, int commitWithinMs) throws SolrServerException, IOException { + if (!this.writeEnabled) return _dummyOKResponse; UpdateResponse ur = null; for (SolrServer s: server) ur = s.deleteById(ids, commitWithinMs); return ur; @@ -294,6 +320,7 @@ public class ServerShard extends SolrServer { * @throws IOException If there is a low-level I/O error. */ public UpdateResponse deleteByQuery(String query) throws SolrServerException, IOException { + if (!this.writeEnabled) return _dummyOKResponse; UpdateResponse ur = null; for (SolrServer s: server) ur = s.deleteByQuery(query); return ur; @@ -307,6 +334,7 @@ public class ServerShard extends SolrServer { * @since 3.6 */ public UpdateResponse deleteByQuery(String query, int commitWithinMs) throws SolrServerException, IOException { + if (!this.writeEnabled) return _dummyOKResponse; UpdateResponse ur = null; for (SolrServer s: server) ur = s.deleteByQuery(query, commitWithinMs); return ur; diff --git a/source/net/yacy/cora/federate/solr/instance/ShardInstance.java b/source/net/yacy/cora/federate/solr/instance/ShardInstance.java index 1400b0042..112c12873 100644 --- a/source/net/yacy/cora/federate/solr/instance/ShardInstance.java +++ b/source/net/yacy/cora/federate/solr/instance/ShardInstance.java @@ -36,10 +36,12 @@ public class ShardInstance implements SolrInstance { private final ShardSelection.Method method; private SolrServer defaultServer; private Map serverCache; + private final boolean writeEnabled; - public ShardInstance(final ArrayList instances, final ShardSelection.Method method) { + public ShardInstance(final ArrayList instances, final ShardSelection.Method method, final boolean writeEnabled) { this.instances = instances; this.method = method; + this.writeEnabled = writeEnabled; this.defaultServer = null; this.serverCache = new ConcurrentHashMap(); } @@ -59,7 +61,7 @@ public class ShardInstance implements SolrInstance { if (this.defaultServer != null) return this.defaultServer; ArrayList server = new ArrayList(instances.size()); for (int i = 0; i < instances.size(); i++) server.set(i, instances.get(i).getDefaultServer()); - this.defaultServer = new ServerShard(server, method); + this.defaultServer = new ServerShard(server, method, this.writeEnabled); return this.defaultServer; } @@ -69,7 +71,7 @@ public class ShardInstance implements SolrInstance { if (s != null) return s; ArrayList server = new ArrayList(instances.size()); for (int i = 0; i < instances.size(); i++) server.add(i, instances.get(i).getServer(name)); - s = new ServerShard(server, method); + s = new ServerShard(server, method, this.writeEnabled); this.serverCache.put(name, s); return s; } diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 0d600f8ce..1c164816d 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -511,11 +511,12 @@ public final class Switchboard extends serverSwitch { final String solrurls = getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr"); final boolean usesolr = getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) & solrurls.length() > 0; final int solrtimeout = getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 60000); + final boolean writeEnabled = getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_WRITEENABLED, true); if (usesolr && solrurls != null && solrurls.length() > 0) { try { ArrayList instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout); - this.index.fulltext().connectRemoteSolr(instances); + this.index.fulltext().connectRemoteSolr(instances, writeEnabled); } catch (final IOException e ) { ConcurrentLog.logException(e); } @@ -1334,11 +1335,12 @@ public final class Switchboard extends serverSwitch { final String solrurls = getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr"); final boolean usesolr = getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) & solrurls.length() > 0; final int solrtimeout = getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 60000); + final boolean writeEnabled = getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_WRITEENABLED, true); if (usesolr && solrurls != null && solrurls.length() > 0) { try { ArrayList instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout); - this.index.fulltext().connectRemoteSolr(instances); + this.index.fulltext().connectRemoteSolr(instances, writeEnabled); } catch (final IOException e ) { ConcurrentLog.logException(e); } diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java index 9660cab17..10d203109 100644 --- a/source/net/yacy/search/SwitchboardConstants.java +++ b/source/net/yacy/search/SwitchboardConstants.java @@ -308,19 +308,20 @@ public final class SwitchboardConstants { public static final String REMOTESEARCH_MAXCOUNT_USER = "remotesearch.maxcount"; public static final String REMOTESEARCH_MAXTIME_USER = "remotesearch.maxtime"; - public static final String FEDERATED_SERVICE_SOLR_INDEXING_ENABLED = "federated.service.solr.indexing.enabled"; - public static final String FEDERATED_SERVICE_SOLR_INDEXING_URL = "federated.service.solr.indexing.url"; - public static final String FEDERATED_SERVICE_SOLR_INDEXING_SHARDING = "federated.service.solr.indexing.sharding"; - public static final String FEDERATED_SERVICE_SOLR_INDEXING_LAZY = "federated.service.solr.indexing.lazy"; - public static final String FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT = "federated.service.solr.indexing.timeout"; + public static final String FEDERATED_SERVICE_SOLR_INDEXING_ENABLED = "federated.service.solr.indexing.enabled"; + public static final String FEDERATED_SERVICE_SOLR_INDEXING_URL = "federated.service.solr.indexing.url"; + public static final String FEDERATED_SERVICE_SOLR_INDEXING_SHARDING = "federated.service.solr.indexing.sharding"; + public static final String FEDERATED_SERVICE_SOLR_INDEXING_LAZY = "federated.service.solr.indexing.lazy"; + public static final String FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT = "federated.service.solr.indexing.timeout"; + public static final String FEDERATED_SERVICE_SOLR_INDEXING_WRITEENABLED = "federated.service.solr.indexing.writeEnabled"; //public static final String CORE_SERVICE_URLDB = "core.service.urldb.tmp"; //public static final String CORE_SERVICE_SOLR = "core.service.solr.tmp"; - public static final String CORE_SERVICE_FULLTEXT = "core.service.fulltext"; - public static final String CORE_SERVICE_RWI = "core.service.rwi.tmp"; - public static final String CORE_SERVICE_CITATION = "core.service.citation.tmp"; - public static final String CORE_SERVICE_WEBGRAPH = "core.service.webgraph.tmp"; - public static final String CORE_SERVICE_JENA = "core.service.jena.tmp"; + public static final String CORE_SERVICE_FULLTEXT = "core.service.fulltext"; + public static final String CORE_SERVICE_RWI = "core.service.rwi.tmp"; + public static final String CORE_SERVICE_CITATION = "core.service.citation.tmp"; + public static final String CORE_SERVICE_WEBGRAPH = "core.service.webgraph.tmp"; + public static final String CORE_SERVICE_JENA = "core.service.jena.tmp"; /** *

public static final String CRAWLER_THREADS_ACTIVE_MAX = "crawler.MaxActiveThreads"

@@ -333,7 +334,7 @@ public final class SwitchboardConstants { /** * debug flags */ - public static final String DEBUG_SEARCH_LOCAL_DHT_OFF = "debug.search.local.dht.off"; // =true: do not use the local dht/rwi index (which is not done if we do remote searches) + public static final String DEBUG_SEARCH_LOCAL_DHT_OFF = "debug.search.local.dht.off"; // =true: do not use the local dht/rwi index (which is not done if we do remote searches) public static final String DEBUG_SEARCH_LOCAL_SOLR_OFF = "debug.search.local.solr.off"; // =true: do not use solr public static final String DEBUG_SEARCH_REMOTE_DHT_OFF = "debug.search.remote.dht.off"; // =true: do not use dht/rwi public static final String DEBUG_SEARCH_REMOTE_DHT_TESTLOCAL= "debug.search.remote.dht.testlocal"; // =true: do not use dht, search local peer in a shortcut to the own server diff --git a/source/net/yacy/search/index/Fulltext.java b/source/net/yacy/search/index/Fulltext.java index 0e2079feb..e8c5c5b58 100644 --- a/source/net/yacy/search/index/Fulltext.java +++ b/source/net/yacy/search/index/Fulltext.java @@ -187,8 +187,8 @@ public final class Fulltext { return this.solrInstances.isConnected1(); } - public void connectRemoteSolr(final ArrayList instances) { - this.solrInstances.connect1(new ShardInstance(instances, ShardSelection.Method.MODULO_HOST_MD5)); + public void connectRemoteSolr(final ArrayList instances, final boolean writeEnabled) { + this.solrInstances.connect1(new ShardInstance(instances, ShardSelection.Method.MODULO_HOST_MD5, writeEnabled)); } public void disconnectRemoteSolr() {