From 5a3c8298726fd27306840f1cfbbed1a82e1a0196 Mon Sep 17 00:00:00 2001 From: orbiter Date: Fri, 20 Jul 2012 11:40:33 +0200 Subject: [PATCH] embedded solr is only initiated if it is activated with IndexFederated_p.html --- htroot/IndexFederated_p.java | 30 ++++++++++++++----- htroot/solr/select.java | 2 +- source/net/yacy/search/Switchboard.java | 8 +++-- .../net/yacy/search/index/DocumentIndex.java | 2 +- .../yacy/search/index/MetadataRepository.java | 22 ++++++++++++-- source/net/yacy/search/index/Segment.java | 13 ++++++-- source/net/yacy/yacy.java | 4 +-- 7 files changed, 63 insertions(+), 18 deletions(-) diff --git a/htroot/IndexFederated_p.java b/htroot/IndexFederated_p.java index 906448d7c..d0d738ac8 100644 --- a/htroot/IndexFederated_p.java +++ b/htroot/IndexFederated_p.java @@ -51,13 +51,28 @@ public class IndexFederated_p { if (post != null && post.containsKey("set")) { // yacy - String localindex = post.get("yacy.indexing", "off"); + String localindex = post.get("yacy.indexing", "off"); // possible values: classic, solr, off + final boolean solrLocalWasOn = sb.index.getLocalSolr() != null && env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_YACY_INDEXING_ENGINE, "off").equals("solr"); + final boolean solrLocalIsOnAfterwards = localindex.equals("solr"); env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_YACY_INDEXING_ENGINE, localindex); + + if (solrLocalWasOn && !solrLocalIsOnAfterwards) { + sb.index.disconnectLocalSolr(); + } + if (!solrLocalWasOn && solrLocalIsOnAfterwards) { + // switch on + try { + sb.index.connectLocalSolr(); + } catch (IOException e) { + Log.logException(e); + } + } + // solr - final boolean solrWasOn = sb.index.getRemoteSolr() != null && env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, true); - final boolean solrIsOnAfterwards = post.getBoolean("solr.indexing.solrremote"); - env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, solrIsOnAfterwards); + final boolean solrRemoteWasOn = sb.index.getRemoteSolr() != null && env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, true); + final boolean solrRemoteIsOnAfterwards = post.getBoolean("solr.indexing.solrremote"); + env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, solrRemoteIsOnAfterwards); String solrurls = post.get("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr")); int commitWithinMs = post.getInt("solr.indexing.commitWithinMs", env.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, 180000)); boolean lazy = post.getBoolean("solr.indexing.lazy"); @@ -84,13 +99,12 @@ public class IndexFederated_p { final String schemename = post.get("solr.indexing.schemefile", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SCHEMEFILE, "solr.keys.default.list")); env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SCHEMEFILE, schemename); - if (solrWasOn) { + if (solrRemoteWasOn && !solrRemoteIsOnAfterwards) { // switch off - sb.index.getRemoteSolr().close(); - sb.index.connectRemoteSolr(null); + sb.index.disconnectRemoteSolr(); } - if (solrIsOnAfterwards) { + if (!solrRemoteWasOn && solrRemoteIsOnAfterwards) { // switch on final boolean usesolr = sb.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) & solrurls.length() > 0; try { diff --git a/htroot/solr/select.java b/htroot/solr/select.java index 754377078..772a038d7 100644 --- a/htroot/solr/select.java +++ b/htroot/solr/select.java @@ -41,7 +41,7 @@ public class select { * @param out * @return */ - public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env, final OutputStream out) { + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env, final OutputStream out) { // this uses the methods in the jetty servlet environment and can be removed if jetty in implemented Switchboard sb = (Switchboard) env; diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 8fca688f6..f381602a1 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -382,6 +382,7 @@ public final class Switchboard extends serverSwitch // initialize index ReferenceContainer.maxReferences = getConfigInt("index.maxReferences", 0); final File segmentsPath = new File(new File(indexPath, networkName), "SEGMENTS"); + final boolean solrLocal = this.getConfig(SwitchboardConstants.FEDERATED_SERVICE_YACY_INDEXING_ENGINE, "off").equals("solr"); this.index = new Segment( this.log, @@ -389,7 +390,8 @@ public final class Switchboard extends serverSwitch wordCacheMaxCount, fileSizeMax, this.useTailCache, - this.exceed134217727); + this.exceed134217727, + solrLocal); // prepare a solr index profile switch list final File solrBackupProfile = new File("defaults/solr.keys.list"); @@ -1179,6 +1181,7 @@ public final class Switchboard extends serverSwitch setConfig("heuristic.site", false); setConfig("heuristic.blekko", false); + final boolean solrLocal = this.getConfig(SwitchboardConstants.FEDERATED_SERVICE_YACY_INDEXING_ENGINE, "off").equals("solr"); // relocate this.peers.relocate( this.networkRoot, @@ -1193,7 +1196,8 @@ public final class Switchboard extends serverSwitch wordCacheMaxCount, fileSizeMax, this.useTailCache, - this.exceed134217727); + this.exceed134217727, + solrLocal); this.crawlQueues.relocate(this.queuesRoot); // cannot be closed because the busy threads are working with that object // create a crawler diff --git a/source/net/yacy/search/index/DocumentIndex.java b/source/net/yacy/search/index/DocumentIndex.java index 30af3a83b..2be41ccb9 100644 --- a/source/net/yacy/search/index/DocumentIndex.java +++ b/source/net/yacy/search/index/DocumentIndex.java @@ -74,7 +74,7 @@ public class DocumentIndex extends Segment public DocumentIndex(final File segmentPath, final CallbackListener callback, final int cachesize) throws IOException { - super(new Log("DocumentIndex"), segmentPath, cachesize, targetFileSize * 4 - 1, false, false); + super(new Log("DocumentIndex"), segmentPath, cachesize, targetFileSize * 4 - 1, false, false, true); final int cores = Runtime.getRuntime().availableProcessors() + 1; this.callback = callback; this.queue = new LinkedBlockingQueue(cores * 300); diff --git a/source/net/yacy/search/index/MetadataRepository.java b/source/net/yacy/search/index/MetadataRepository.java index 240c26489..f9faa57ad 100644 --- a/source/net/yacy/search/index/MetadataRepository.java +++ b/source/net/yacy/search/index/MetadataRepository.java @@ -98,6 +98,12 @@ public final class MetadataRepository implements /*Metadata,*/ Iterable this.remoteSolr = solr; } + public void disconnectRemoteSolr() { + if (this.remoteSolr == null) return; + this.remoteSolr.close(); + this.remoteSolr = null; + } + public void connectLocalSolr() throws IOException { File solrLocation = this.location; if (solrLocation.getName().equals("default")) solrLocation = solrLocation.getParentFile(); @@ -112,6 +118,12 @@ public final class MetadataRepository implements /*Metadata,*/ Iterable this.localSolr = solr; } + public void disconnectLocalSolr() { + if (this.localSolr == null) return; + this.localSolr.close(); + this.localSolr = null; + } + public SolrConnector getLocalSolr() { return this.localSolr; } @@ -147,8 +159,14 @@ public final class MetadataRepository implements /*Metadata,*/ Iterable this.urlIndexFile.close(); this.urlIndexFile = null; } - if (this.localSolr != null) this.localSolr.close(); - if (this.remoteSolr != null) this.remoteSolr.close(); + if (this.localSolr != null) { + this.localSolr.close(); + this.localSolr = null; + } + if (this.remoteSolr != null) { + this.remoteSolr.close(); + this.remoteSolr = null; + } } public int writeCacheSize() { diff --git a/source/net/yacy/search/index/Segment.java b/source/net/yacy/search/index/Segment.java index e95277eed..437f18d33 100644 --- a/source/net/yacy/search/index/Segment.java +++ b/source/net/yacy/search/index/Segment.java @@ -108,7 +108,8 @@ public class Segment { final int entityCacheMaxSize, final long maxFileSize, final boolean useTailCache, - final boolean exceed134217727) throws IOException { + final boolean exceed134217727, + final boolean connectLocalSolr) throws IOException { log.logInfo("Initializing Segment '" + segmentPath + "."); @@ -139,7 +140,7 @@ public class Segment { // create LURL-db this.urlMetadata = new MetadataRepository(segmentPath, "text.urlmd", useTailCache, exceed134217727); - this.connectLocalSolr(); + if (connectLocalSolr) this.connectLocalSolr(); } public long URLCount() { @@ -157,11 +158,19 @@ public class Segment { public void connectRemoteSolr(final SolrConnector solr) { this.urlMetadata.connectRemoteSolr(solr); } + + public void disconnectRemoteSolr() { + this.urlMetadata.disconnectRemoteSolr(); + } public void connectLocalSolr() throws IOException { this.urlMetadata.connectLocalSolr(); } + public void disconnectLocalSolr() { + this.urlMetadata.disconnectLocalSolr(); + } + public SolrConnector getRemoteSolr() { return this.urlMetadata.getRemoteSolr(); } diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java index aa0fa5119..6255833fb 100644 --- a/source/net/yacy/yacy.java +++ b/source/net/yacy/yacy.java @@ -669,7 +669,7 @@ public final class yacy { log, new File(new File(indexPrimaryRoot, "freeworld"), "TEXT"), 10000, - Integer.MAX_VALUE, false, false); + Integer.MAX_VALUE, false, false, false); final Iterator> indexContainerIterator = wordIndex.termIndex().referenceContainerIterator("AAAAAAAAAAAA".getBytes(), false, false); long urlCounter = 0, wordCounter = 0; @@ -849,7 +849,7 @@ public final class yacy { log, new File(new File(indexPrimaryRoot, "freeworld"), "TEXT"), 10000, - Integer.MAX_VALUE, false, false); + Integer.MAX_VALUE, false, false, false); indexContainerIterator = WordIndex.termIndex().referenceContainerIterator(wordChunkStartHash.getBytes(), false, false); } int counter = 0;