From 8dd469b9ddc0cc6d78d8890e901531f70142d376 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Mon, 25 Jun 2012 14:59:46 +0200 Subject: [PATCH] added option to configure the autocommit delay time of solr on-the-fly --- defaults/yacy.init | 1 + htroot/IndexFederated_p.html | 2 ++ htroot/IndexFederated_p.java | 11 ++++++++- .../federated/solr/AbstractSolrConnector.java | 24 +++++++++++++++++-- .../federated/solr/MultipleSolrConnector.java | 21 +++++++++++++++- .../federated/solr/RetrySolrConnector.java | 14 +++++++++++ .../federated/solr/ShardSolrConnector.java | 14 +++++++++++ .../federated/solr/SolrConnector.java | 15 ++++++++++++ source/net/yacy/search/Switchboard.java | 13 ++++++---- 9 files changed, 106 insertions(+), 9 deletions(-) diff --git a/defaults/yacy.init b/defaults/yacy.init index 38de012bb..e9d67aaf3 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -1049,6 +1049,7 @@ color_searchurlhover = #008000 # - to check whats in solr after indexing, open http://localhost:8983/solr/admin/ federated.service.solr.indexing.enabled = false federated.service.solr.indexing.url = http://127.0.0.1:8983/solr +federated.service.solr.indexing.commitWithinMs = 180000 federated.service.solr.indexing.sharding = MODULO_HOST_MD5 federated.service.solr.indexing.schemefile = solr.keys.default.list diff --git a/htroot/IndexFederated_p.html b/htroot/IndexFederated_p.html index dbf4cad51..4ae4a2e7b 100644 --- a/htroot/IndexFederated_p.html +++ b/htroot/IndexFederated_p.html @@ -61,6 +61,8 @@ #(/table)#
Solr URL(s)
+
Commit-Within (milliseconds)
+
Sharding Method
Scheme
diff --git a/htroot/IndexFederated_p.java b/htroot/IndexFederated_p.java index 79ce3dbc4..aca303562 100644 --- a/htroot/IndexFederated_p.java +++ b/htroot/IndexFederated_p.java @@ -59,6 +59,7 @@ public class IndexFederated_p { final boolean solrIsOnAfterwards = post.getBoolean("solr.indexing.solrremote", false); env.setConfig("federated.service.solr.indexing.enabled", solrIsOnAfterwards); String solrurls = post.get("solr.indexing.url", env.getConfig("federated.service.solr.indexing.url", "http://127.0.0.1:8983/solr")); + int commitWithinMs = post.getInt("solr.indexing.commitWithinMs", env.getConfigInt("federated.service.solr.indexing.commitWithinMs", 180000)); final BufferedReader r = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(UTF8.getBytes(solrurls)))); final StringBuilder s = new StringBuilder(); String s0; @@ -76,6 +77,7 @@ public class IndexFederated_p { } solrurls = s.toString().trim(); env.setConfig("federated.service.solr.indexing.url", solrurls); + env.setConfig("federated.service.solr.indexing.commitWithinMs", commitWithinMs); env.setConfig("federated.service.solr.indexing.sharding", post.get("solr.indexing.sharding", env.getConfig("federated.service.solr.indexing.sharding", "modulo-host-md5"))); final String schemename = post.get("solr.indexing.schemefile", env.getConfig("federated.service.solr.indexing.schemefile", "solr.keys.default.list")); env.setConfig("federated.service.solr.indexing.schemefile", schemename); @@ -90,7 +92,13 @@ public class IndexFederated_p { // switch on final boolean usesolr = sb.getConfigBool("federated.service.solr.indexing.enabled", false) & solrurls.length() > 0; try { - sb.indexSegments.segment(Segments.Process.LOCALCRAWLING).connectRemoteSolr((usesolr) ? new ShardSolrConnector(solrurls, ShardSelection.Method.MODULO_HOST_MD5, 10000, true) : null); + if (usesolr) { + SolrConnector solr = new ShardSolrConnector(solrurls, ShardSelection.Method.MODULO_HOST_MD5, 10000, true); + solr.setCommitWithinMs(commitWithinMs); + sb.indexSegments.segment(Segments.Process.LOCALCRAWLING).connectRemoteSolr(solr); + } else { + sb.indexSegments.segment(Segments.Process.LOCALCRAWLING).connectRemoteSolr(null); + } } catch (final IOException e) { Log.logException(e); sb.indexSegments.segment(Segments.Process.LOCALCRAWLING).connectRemoteSolr(null); @@ -179,6 +187,7 @@ public class IndexFederated_p { prop.put("yacy.indexing.engine.off.checked", env.getConfig("federated.service.yacy.indexing.engine", "classic").equals("off") ? 1 : 0); prop.put("solr.indexing.solrremote.checked", env.getConfigBool("federated.service.solr.indexing.enabled", false) ? 1 : 0); prop.put("solr.indexing.url", env.getConfig("federated.service.solr.indexing.url", "http://127.0.0.1:8983/solr").replace(",", "\n")); + prop.put("solr.indexing.commitWithinMs", env.getConfigInt("federated.service.solr.indexing.commitWithinMs", 180000)); prop.put("solr.indexing.sharding", env.getConfig("federated.service.solr.indexing.sharding", "modulo-host-md5")); prop.put("solr.indexing.schemefile", schemename); diff --git a/source/net/yacy/cora/services/federated/solr/AbstractSolrConnector.java b/source/net/yacy/cora/services/federated/solr/AbstractSolrConnector.java index 4962acdf6..2c9e86317 100644 --- a/source/net/yacy/cora/services/federated/solr/AbstractSolrConnector.java +++ b/source/net/yacy/cora/services/federated/solr/AbstractSolrConnector.java @@ -41,9 +41,11 @@ import org.apache.solr.common.SolrInputDocument; public class AbstractSolrConnector implements SolrConnector { protected SolrServer server; + protected int commitWithinMs; // max time (in ms) before a commit will happen protected AbstractSolrConnector() { this.server = null; + this.commitWithinMs = 180000; } protected void init(SolrServer server) { @@ -54,6 +56,24 @@ public class AbstractSolrConnector implements SolrConnector { return this.server; } + /** + * get the solr autocommit delay + * @return the maximum waiting time after a solr command until it is transported to the server + */ + @Override + public int getCommitWithinMs() { + return this.commitWithinMs; + } + + /** + * set the solr autocommit delay + * @param c the maximum waiting time after a solr command until it is transported to the server + */ + @Override + public void setCommitWithinMs(int c) { + this.commitWithinMs = c; + } + @Override public synchronized void close() { try { @@ -137,7 +157,7 @@ public class AbstractSolrConnector implements SolrConnector { @Override public void add(final SolrDoc solrdoc) throws IOException, SolrException { try { - this.server.add(solrdoc,180000); // commitWithIn 180s + this.server.add(solrdoc, this.commitWithinMs); //this.server.commit(); } catch (SolrServerException e) { Log.logWarning("SolrConnector", e.getMessage() + " DOC=" + solrdoc.toString()); @@ -150,7 +170,7 @@ public class AbstractSolrConnector implements SolrConnector { ArrayList l = new ArrayList(); for (SolrDoc d: solrdocs) l.add(d); try { - this.server.add(l,180000); // commitWithIn 120s + this.server.add(l, this.commitWithinMs); //this.server.commit(); } catch (SolrServerException e) { Log.logWarning("SolrConnector", e.getMessage() + " DOC=" + solrdocs.toString()); diff --git a/source/net/yacy/cora/services/federated/solr/MultipleSolrConnector.java b/source/net/yacy/cora/services/federated/solr/MultipleSolrConnector.java index 92f7963a8..73483ce9b 100644 --- a/source/net/yacy/cora/services/federated/solr/MultipleSolrConnector.java +++ b/source/net/yacy/cora/services/federated/solr/MultipleSolrConnector.java @@ -15,11 +15,13 @@ public class MultipleSolrConnector implements SolrConnector { private final ArrayBlockingQueue queue; private final AddWorker[] worker; private final SolrConnector solr; + private int commitWithinMs; public MultipleSolrConnector(final String url, int connections) throws IOException { this.solr = new SingleSolrConnector(url); this.queue = new ArrayBlockingQueue(1000); this.worker = new AddWorker[connections]; + this.commitWithinMs = 180000; for (int i = 0; i < connections; i++) { this.worker[i] = new AddWorker(url); this.worker[i].start(); @@ -30,6 +32,7 @@ public class MultipleSolrConnector implements SolrConnector { private final SolrConnector solr; public AddWorker(final String url) throws IOException { this.solr = new SingleSolrConnector(url); + this.solr.setCommitWithinMs(MultipleSolrConnector.this.commitWithinMs); } @Override public void run() { @@ -51,6 +54,22 @@ public class MultipleSolrConnector implements SolrConnector { } } + @Override + public int getCommitWithinMs() { + return this.commitWithinMs; + } + + /** + * set the solr autocommit delay + * @param c the maximum waiting time after a solr command until it is transported to the server + */ + @Override + public void setCommitWithinMs(int c) { + this.commitWithinMs = c; + this.solr.setCommitWithinMs(c); + for (AddWorker w: this.worker) w.solr.setCommitWithinMs(c); + } + @Override public void close() { for (@SuppressWarnings("unused") AddWorker element : this.worker) { @@ -59,8 +78,8 @@ public class MultipleSolrConnector implements SolrConnector { } catch (InterruptedException e) { e.printStackTrace(); } - this.solr.close(); } + this.solr.close(); } @Override diff --git a/source/net/yacy/cora/services/federated/solr/RetrySolrConnector.java b/source/net/yacy/cora/services/federated/solr/RetrySolrConnector.java index 74787b94b..f3863732a 100644 --- a/source/net/yacy/cora/services/federated/solr/RetrySolrConnector.java +++ b/source/net/yacy/cora/services/federated/solr/RetrySolrConnector.java @@ -41,6 +41,20 @@ public class RetrySolrConnector implements SolrConnector { this.retryMaxTime = retryMaxTime; } + @Override + public int getCommitWithinMs() { + return this.solrConnector.getCommitWithinMs(); + } + + /** + * set the solr autocommit delay + * @param c the maximum waiting time after a solr command until it is transported to the server + */ + @Override + public void setCommitWithinMs(int c) { + this.solrConnector.setCommitWithinMs(c); + } + @Override public synchronized void close() { this.solrConnector.close(); diff --git a/source/net/yacy/cora/services/federated/solr/ShardSolrConnector.java b/source/net/yacy/cora/services/federated/solr/ShardSolrConnector.java index ed8de4cdf..d409dd0bc 100644 --- a/source/net/yacy/cora/services/federated/solr/ShardSolrConnector.java +++ b/source/net/yacy/cora/services/federated/solr/ShardSolrConnector.java @@ -55,6 +55,20 @@ public class ShardSolrConnector implements SolrConnector { this.sharding = new ShardSelection(method, this.urls.length); } + @Override + public int getCommitWithinMs() { + return this.connectors.get(0).getCommitWithinMs(); + } + + /** + * set the solr autocommit delay + * @param c the maximum waiting time after a solr command until it is transported to the server + */ + @Override + public void setCommitWithinMs(int c) { + for (final SolrConnector connector: this.connectors) connector.setCommitWithinMs(c); + } + @Override public synchronized void close() { for (final SolrConnector connector: this.connectors) connector.close(); diff --git a/source/net/yacy/cora/services/federated/solr/SolrConnector.java b/source/net/yacy/cora/services/federated/solr/SolrConnector.java index 3180e2837..a05e38d35 100644 --- a/source/net/yacy/cora/services/federated/solr/SolrConnector.java +++ b/source/net/yacy/cora/services/federated/solr/SolrConnector.java @@ -33,6 +33,21 @@ import org.apache.solr.common.SolrException; public interface SolrConnector { + /** + * get the solr autocommit delay + * @return the maximum waiting time after a solr command until it is transported to the server + */ + public int getCommitWithinMs(); + + /** + * set the solr autocommit delay + * @param c the maximum waiting time after a solr command until it is transported to the server + */ + public void setCommitWithinMs(int c); + + /** + * close the server connection + */ public void close(); /** diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 7c9d80ccf..ae26b5132 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -95,6 +95,7 @@ import net.yacy.cora.protocol.http.HTTPClient; import net.yacy.cora.protocol.http.ProxySettings; import net.yacy.cora.services.federated.solr.ShardSelection; import net.yacy.cora.services.federated.solr.ShardSolrConnector; +import net.yacy.cora.services.federated.solr.SolrConnector; import net.yacy.cora.services.federated.solr.SolrDoc; import net.yacy.cora.services.federated.yacy.CacheStrategy; import net.yacy.document.Condenser; @@ -407,14 +408,16 @@ public final class Switchboard extends serverSwitch // set up the solr interface final String solrurls = getConfig("federated.service.solr.indexing.url", "http://127.0.0.1:8983/solr"); final boolean usesolr = getConfigBool("federated.service.solr.indexing.enabled", false) & solrurls.length() > 0; + int commitWithinMs = getConfigInt("federated.service.solr.indexing.commitWithinMs", 180000); if (usesolr && solrurls != null && solrurls.length() > 0) { try { - this.indexSegments.segment(Segments.Process.LOCALCRAWLING).connectRemoteSolr( - new ShardSolrConnector( - solrurls, - ShardSelection.Method.MODULO_HOST_MD5, - 10000, true)); + SolrConnector solr = new ShardSolrConnector( + solrurls, + ShardSelection.Method.MODULO_HOST_MD5, + 10000, true); + solr.setCommitWithinMs(commitWithinMs); + this.indexSegments.segment(Segments.Process.LOCALCRAWLING).connectRemoteSolr(solr); } catch ( final IOException e ) { Log.logException(e); }