diff --git a/htroot/IndexControlRWIs_p.java b/htroot/IndexControlRWIs_p.java index 0ea62aaa7..458bfdb00 100644 --- a/htroot/IndexControlRWIs_p.java +++ b/htroot/IndexControlRWIs_p.java @@ -92,7 +92,7 @@ public class IndexControlRWIs_p { prop.put("keyhash", ""); prop.put("result", ""); prop.put("cleanup", post == null || post.containsKey("maxReferencesLimit") ? 1 : 0); - prop.put("cleanup_solr", sb.index.getRemoteSolr() == null ? 0 : 1); + prop.put("cleanup_solr", sb.index.connectedRemoteSolr() ? 1 : 0); // switch off all optional forms/lists prop.put("searchresult", 0); @@ -157,7 +157,7 @@ public class IndexControlRWIs_p { if ( post.get("deleteIndex", "").equals("on") ) { segment.clear(); } - if ( post.get("deleteRemoteSolr", "").equals("on") && sb.index.getRemoteSolr() != null) { + if ( post.get("deleteRemoteSolr", "").equals("on") && sb.index.connectedRemoteSolr()) { try { sb.index.getRemoteSolr().clear(); } catch ( final Exception e ) { diff --git a/htroot/IndexFederated_p.java b/htroot/IndexFederated_p.java index 31a1e3dc4..2d43aff42 100644 --- a/htroot/IndexFederated_p.java +++ b/htroot/IndexFederated_p.java @@ -72,11 +72,12 @@ public class IndexFederated_p { sb.index.connectCitation(wordCacheMaxCount, fileSizeMax); } catch (IOException e) { Log.logException(e); } // switch on + final int commitWithinMs = post.getInt("solr.indexing.commitWithinMs", env.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, 180000)); boolean post_core_solr = post.getBoolean(SwitchboardConstants.CORE_SERVICE_SOLR); final boolean previous_core_solr = sb.index.connectedLocalSolr() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_SOLR, false); env.setConfig(SwitchboardConstants.CORE_SERVICE_SOLR, post_core_solr); if (previous_core_solr && !post_core_solr) sb.index.disconnectLocalSolr(); // switch off - if (!previous_core_solr && post_core_solr) try { sb.index.connectLocalSolr(); } catch (IOException e) { Log.logException(e); } // switch on + if (!previous_core_solr && post_core_solr) try { sb.index.connectLocalSolr(commitWithinMs); } catch (IOException e) { Log.logException(e); } // switch on boolean post_core_urldb = post.getBoolean(SwitchboardConstants.CORE_SERVICE_URLDB); final boolean previous_core_urldb = sb.index.connectedUrlDb() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_URLDB, false); @@ -89,7 +90,6 @@ public class IndexFederated_p { final boolean solrRemoteIsOnAfterwards = post.getBoolean("solr.indexing.solrremote"); env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, solrRemoteIsOnAfterwards); String solrurls = post.get("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr")); - int commitWithinMs = post.getInt("solr.indexing.commitWithinMs", env.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, 180000)); boolean lazy = post.getBoolean("solr.indexing.lazy"); final BufferedReader r = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(UTF8.getBytes(solrurls)))); final StringBuilder s = new StringBuilder(); @@ -167,7 +167,7 @@ public class IndexFederated_p { } // show solr host table - if (sb.index.getRemoteSolr() == null) { + if (!sb.index.connectedRemoteSolr()) { prop.put("table", 0); } else { prop.put("table", 1); diff --git a/source/de/anomic/crawler/CrawlQueues.java b/source/de/anomic/crawler/CrawlQueues.java index fa2bfe8e5..25b6ee77a 100644 --- a/source/de/anomic/crawler/CrawlQueues.java +++ b/source/de/anomic/crawler/CrawlQueues.java @@ -81,8 +81,8 @@ public class CrawlQueues { this.log.logConfig("Starting Crawling Management"); this.noticeURL = new NoticedURL(queuePath, sb.peers.myBotIDs(), sb.useTailCache, sb.exceed134217727); FileUtils.deletedelete(new File(queuePath, ERROR_DB_FILENAME)); - this.errorURL = new ZURL(sb.index.getRemoteSolr(), sb.solrScheme, queuePath, ERROR_DB_FILENAME, false, sb.useTailCache, sb.exceed134217727); - this.delegatedURL = new ZURL(sb.index.getRemoteSolr(), sb.solrScheme, queuePath, DELEGATED_DB_FILENAME, true, sb.useTailCache, sb.exceed134217727); + this.errorURL = new ZURL(sb.index.getSolr(), sb.solrScheme, queuePath, ERROR_DB_FILENAME, false, sb.useTailCache, sb.exceed134217727); + this.delegatedURL = new ZURL(sb.index.getSolr(), sb.solrScheme, queuePath, DELEGATED_DB_FILENAME, true, sb.useTailCache, sb.exceed134217727); } public void relocate(final File newQueuePath) { @@ -93,8 +93,8 @@ public class CrawlQueues { this.noticeURL = new NoticedURL(newQueuePath, this.sb.peers.myBotIDs(), this.sb.useTailCache, this.sb.exceed134217727); FileUtils.deletedelete(new File(newQueuePath, ERROR_DB_FILENAME)); - this.errorURL = new ZURL(this.sb.index.getRemoteSolr(), this.sb.solrScheme, newQueuePath, ERROR_DB_FILENAME, false, this.sb.useTailCache, this.sb.exceed134217727); - this.delegatedURL = new ZURL(this.sb.index.getRemoteSolr(), this.sb.solrScheme, newQueuePath, DELEGATED_DB_FILENAME, true, this.sb.useTailCache, this.sb.exceed134217727); + this.errorURL = new ZURL(this.sb.index.getSolr(), this.sb.solrScheme, newQueuePath, ERROR_DB_FILENAME, false, this.sb.useTailCache, this.sb.exceed134217727); + this.delegatedURL = new ZURL(this.sb.index.getSolr(), this.sb.solrScheme, newQueuePath, DELEGATED_DB_FILENAME, true, this.sb.useTailCache, this.sb.exceed134217727); } public synchronized void close() { diff --git a/source/net/yacy/cora/services/federated/solr/AbstractSolrConnector.java b/source/net/yacy/cora/services/federated/solr/AbstractSolrConnector.java index 7bafdb5f6..ef634298b 100644 --- a/source/net/yacy/cora/services/federated/solr/AbstractSolrConnector.java +++ b/source/net/yacy/cora/services/federated/solr/AbstractSolrConnector.java @@ -115,7 +115,7 @@ public class AbstractSolrConnector implements SolrConnector { @Override public void delete(final String id) throws IOException { try { - this.server.deleteById(id); + this.server.deleteById(id, this.commitWithinMs); } catch (final Throwable e) { throw new IOException(e); } @@ -124,7 +124,7 @@ public class AbstractSolrConnector implements SolrConnector { @Override public void delete(final List ids) throws IOException { try { - this.server.deleteById(ids); + this.server.deleteById(ids, this.commitWithinMs); } catch (final Throwable e) { throw new IOException(e); } @@ -147,6 +147,7 @@ public class AbstractSolrConnector implements SolrConnector { up.setParam("literal.id", solrId); up.setParam("uprefix", "attr_"); up.setParam("fmap.content", "attr_content"); + up.setCommitWithin(this.commitWithinMs); //up.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true); try { this.server.request(up); diff --git a/source/net/yacy/cora/services/federated/solr/DoubleSolrConnector.java b/source/net/yacy/cora/services/federated/solr/DoubleSolrConnector.java new file mode 100644 index 000000000..9fe918daa --- /dev/null +++ b/source/net/yacy/cora/services/federated/solr/DoubleSolrConnector.java @@ -0,0 +1,229 @@ +/** + * DoubleChardingConnector + * Copyright 2012 by Michael Peter Christen + * First released 23.07.2012 at http://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see . + */ + +package net.yacy.cora.services.federated.solr; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; + +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrException; + + +public class DoubleSolrConnector implements SolrConnector { + + private SolrConnector solr0; + private SolrConnector solr1; + + public DoubleSolrConnector() { + this.solr0 = null; + this.solr1 = null; + } + + public boolean isConnected0() { + return this.solr0 != null; + } + + public void connect0(SolrConnector c) { + this.solr0 = c; + } + + public SolrConnector getSolr0() { + return this.solr0; + } + + public void disconnect0() { + if (this.solr0 == null) return; + this.solr0.close(); + this.solr0 = null; + } + + public boolean isConnected1() { + return this.solr1 != null; + } + + public void connect1(SolrConnector c) { + this.solr1 = c; + } + + public SolrConnector getSolr1() { + return this.solr1; + } + + public void disconnect1() { + if (this.solr1 == null) return; + this.solr1.close(); + this.solr1 = null; + } + + @Override + public int getCommitWithinMs() { + if (this.solr0 != null) this.solr0.getCommitWithinMs(); + if (this.solr1 != null) this.solr1.getCommitWithinMs(); + return -1; + } + + /** + * set the solr autocommit delay + * @param c the maximum waiting time after a solr command until it is transported to the server + */ + @Override + public void setCommitWithinMs(int c) { + if (this.solr0 != null) this.solr0.setCommitWithinMs(c); + if (this.solr1 != null) this.solr1.setCommitWithinMs(c); + } + + @Override + public synchronized void close() { + if (this.solr0 != null) this.solr0.close(); + if (this.solr1 != null) this.solr1.close(); + } + + /** + * delete everything in the solr index + * @throws IOException + */ + @Override + public void clear() throws IOException { + if (this.solr0 != null) this.solr0.clear(); + if (this.solr1 != null) this.solr1.clear(); + } + + /** + * delete an entry from solr + * @param id the url hash of the entry + * @throws IOException + */ + @Override + public void delete(final String id) throws IOException { + if (this.solr0 != null) this.solr0.delete(id); + if (this.solr1 != null) this.solr1.delete(id); + } + + /** + * delete a set of entries from solr; entries are identified by their url hash + * @param ids a list of url hashes + * @throws IOException + */ + @Override + public void delete(final List ids) throws IOException { + if (this.solr0 != null) this.solr0.delete(ids); + if (this.solr1 != null) this.solr1.delete(ids); + } + + /** + * check if a given id exists in solr + * @param id + * @return true if any entry in solr exists + * @throws IOException + */ + @Override + public boolean exists(final String id) throws IOException { + if (this.solr0 != null) { + if (this.solr0.exists(id)) return true; + } + if (this.solr1 != null) { + if (this.solr1.exists(id)) return true; + } + return false; + } + + @Override + public SolrDocument get(String id) throws IOException { + if (this.solr0 != null) { + SolrDocument doc = this.solr0.get(id); + if (doc != null) return doc; + } + if (this.solr1 != null) { + SolrDocument doc = this.solr1.get(id); + if (doc != null) return doc; + } + return null; + } + + /** + * add a Solr document + * @param solrdoc + * @throws IOException + */ + @Override + public void add(final SolrDoc solrdoc) throws IOException { + if (this.solr0 != null) { + this.solr0.add(solrdoc); + } + if (this.solr1 != null) { + this.solr1.add(solrdoc); + } + } + + @Override + public void add(final Collection solrdocs) throws IOException, SolrException { + if (this.solr0 != null) { + for (SolrDoc d: solrdocs) this.solr0.add(d); + } + if (this.solr1 != null) { + for (SolrDoc d: solrdocs) this.solr1.add(d); + } + } + + /** + * get a query result from solr + * to get all results set the query String to "*:*" + * @param querystring + * @throws IOException + */ + @Override + public SolrDocumentList query(final String querystring, final int offset, final int count) throws IOException { + if (this.solr0 == null && this.solr1 == null) return new SolrDocumentList(); + if (this.solr0 != null && this.solr1 == null) return this.solr0.query(querystring, offset, count); + if (this.solr1 != null && this.solr0 == null) return this.solr1.query(querystring, offset, count); + + // combine both lists + SolrDocumentList l; + l = this.solr0.query(querystring, offset, count); + if (l.size() >= count) return l; + + // at this point we need to know how many results are in solr0 + // compute this with a very bad hack; replace with better method later + int size0 = 0; + { //bad hack - TODO: replace + SolrDocumentList lHack = this.solr0.query(querystring, 0, Integer.MAX_VALUE); + size0 = lHack.size(); + } + + // now use the size of the first query to do a second query + final SolrDocumentList list = new SolrDocumentList(); + for (final SolrDocument d: l) list.add(d); + l = this.solr1.query(querystring, offset + l.size() - size0, count - l.size()); + for (final SolrDocument d: l) list.add(d); + return list; + } + + @Override + public long getSize() { + long s = 0; + if (this.solr0 != null) s += this.solr0.getSize(); + if (this.solr1 != null) s += this.solr1.getSize(); + return s; + } + +} diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index ac19075ff..9ff51e63e 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -381,15 +381,15 @@ public final class Switchboard extends serverSwitch ReferenceContainer.maxReferences = getConfigInt("index.maxReferences", 0); final File segmentsPath = new File(new File(indexPath, networkName), "SEGMENTS"); this.index = new Segment(this.log, new File(segmentsPath, "default")); + final int connectWithinMs = this.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, 180000); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, true)) this.index.connectRWI(wordCacheMaxCount, fileSizeMax); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_URLDB, true)) this.index.connectUrlDb(this.useTailCache, this.exceed134217727); - if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_SOLR, true)) this.index.connectLocalSolr(); + if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_SOLR, true)) this.index.connectLocalSolr(connectWithinMs); // prepare a solr index profile switch list final File solrBackupProfile = new File("defaults/solr.keys.list"); - final String schemename = - getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SCHEMEFILE, "solr.keys.default.list"); + final String schemename = getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SCHEMEFILE, "solr.keys.default.list"); final File solrWorkProfile = new File(getDataPath(), "DATA/SETTINGS/" + schemename); if ( !solrWorkProfile.exists() ) { Files.copy(solrBackupProfile, solrWorkProfile); @@ -412,7 +412,7 @@ public final class Switchboard extends serverSwitch solrurls, ShardSelection.Method.MODULO_HOST_MD5, 10000, true); - solr.setCommitWithinMs(getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, 180000)); + solr.setCommitWithinMs(connectWithinMs); this.index.connectRemoteSolr(solr); } catch ( final IOException e ) { Log.logException(e); @@ -1181,9 +1181,10 @@ public final class Switchboard extends serverSwitch this.useTailCache, this.exceed134217727); this.index = new Segment(this.log, new File(new File(new File(indexPrimaryPath, networkName), "SEGMENTS"), "default")); + final int connectWithinMs = this.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, 180000); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, true)) this.index.connectRWI(wordCacheMaxCount, fileSizeMax); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax); - if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_SOLR, true)) this.index.connectLocalSolr(); + if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_SOLR, true)) this.index.connectLocalSolr(connectWithinMs); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_URLDB, true)) this.index.connectUrlDb(this.useTailCache, this.exceed134217727); // set up the solr interface @@ -1196,7 +1197,7 @@ public final class Switchboard extends serverSwitch solrurls, ShardSelection.Method.MODULO_HOST_MD5, 10000, true); - solr.setCommitWithinMs(getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, 180000)); + solr.setCommitWithinMs(connectWithinMs); this.index.connectRemoteSolr(solr); } catch ( final IOException e ) { Log.logException(e); @@ -2412,8 +2413,8 @@ public final class Switchboard extends serverSwitch return new indexingQueueEntry(in.queueEntry, in.documents, null); } - boolean localSolr = this.index.getLocalSolr() != null; - boolean remoteSolr = this.index.getRemoteSolr() != null; + boolean localSolr = this.index.connectedLocalSolr(); + boolean remoteSolr = this.index.connectedRemoteSolr(); if (localSolr || remoteSolr) { // send the documents to solr for ( final Document doc : in.documents ) { @@ -2433,8 +2434,7 @@ public final class Switchboard extends serverSwitch } try { SolrDoc solrDoc = this.solrScheme.yacy2solr(id, in.queueEntry.getResponseHeader(), doc); - if (localSolr) this.index.getLocalSolr().add(solrDoc); - if (remoteSolr) this.index.getRemoteSolr().add(solrDoc); + this.index.getSolr().add(solrDoc); } catch ( final IOException e ) { Log.logWarning( "SOLR", diff --git a/source/net/yacy/search/index/DocumentIndex.java b/source/net/yacy/search/index/DocumentIndex.java index b52a6f6c0..21ebf5878 100644 --- a/source/net/yacy/search/index/DocumentIndex.java +++ b/source/net/yacy/search/index/DocumentIndex.java @@ -82,7 +82,7 @@ public class DocumentIndex extends Segment false, // useTailCache false // exceed134217727 ); - super.connectLocalSolr(); + super.connectLocalSolr(1000); final int cores = Runtime.getRuntime().availableProcessors() + 1; this.callback = callback; this.queue = new LinkedBlockingQueue(cores * 300); diff --git a/source/net/yacy/search/index/MetadataRepository.java b/source/net/yacy/search/index/MetadataRepository.java index 043c8128a..d872885a4 100644 --- a/source/net/yacy/search/index/MetadataRepository.java +++ b/source/net/yacy/search/index/MetadataRepository.java @@ -43,6 +43,7 @@ import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.document.UTF8; import net.yacy.cora.order.CloneableIterator; import net.yacy.cora.protocol.http.HTTPClient; +import net.yacy.cora.services.federated.solr.DoubleSolrConnector; import net.yacy.cora.services.federated.solr.SolrConnector; import net.yacy.cora.sorting.ConcurrentScoreMap; import net.yacy.cora.sorting.ScoreMap; @@ -77,7 +78,7 @@ public final class MetadataRepository implements /*Metadata,*/ Iterable private Export exportthread; // will have a export thread assigned if exporter is running private String tablename; private ArrayList statsDump; - private SolrConnector localSolr, remoteSolr; + private final DoubleSolrConnector solr; public MetadataRepository(final File path) { this.location = path; @@ -85,8 +86,7 @@ public final class MetadataRepository implements /*Metadata,*/ Iterable this.urlIndexFile = null; this.exportthread = null; // will have a export thread assigned if exporter is running this.statsDump = null; - this.remoteSolr = null; - this.localSolr = null; + this.solr = new DoubleSolrConnector(); } public boolean connectedUrlDb() { @@ -106,49 +106,50 @@ public final class MetadataRepository implements /*Metadata,*/ Iterable } public boolean connectedLocalSolr() { - return this.localSolr != null; + return this.solr.isConnected0(); } - public void connectLocalSolr() throws IOException { + public void connectLocalSolr(final int commitWithin) throws IOException { File solrLocation = this.location; if (solrLocation.getName().equals("default")) solrLocation = solrLocation.getParentFile(); String solrPath = "solr_36"; solrLocation = new File(solrLocation, solrPath); // the number should be identical to the number in the property luceneMatchVersion in solrconfig.xml - EmbeddedSolrConnector solr = new EmbeddedSolrConnector(solrLocation, new File(new File(Switchboard.getSwitchboard().appPath, "defaults"), "solr")); - Version luceneVersion = solr.getConfig().getLuceneVersion("luceneMatchVersion"); + EmbeddedSolrConnector esc = new EmbeddedSolrConnector(solrLocation, new File(new File(Switchboard.getSwitchboard().appPath, "defaults"), "solr")); + esc.setCommitWithinMs(commitWithin); + Version luceneVersion = esc.getConfig().getLuceneVersion("luceneMatchVersion"); String lvn = luceneVersion.name(); int p = lvn.indexOf('_'); assert solrPath.endsWith(lvn.substring(p)) : "luceneVersion = " + lvn + ", solrPath = " + solrPath + ", p = " + p; Log.logInfo("MetadataRepository", "connected solr in " + solrLocation.toString() + ", lucene version " + lvn); - this.localSolr = solr; + this.solr.connect0(esc); } public void disconnectLocalSolr() { - if (this.localSolr == null) return; - this.localSolr.close(); - this.localSolr = null; + this.solr.disconnect0(); } public boolean connectedRemoteSolr() { - return this.remoteSolr != null; + return this.solr.isConnected1(); } - public void connectRemoteSolr(final SolrConnector solr) { - this.remoteSolr = solr; + public void connectRemoteSolr(final SolrConnector rs) { + this.solr.connect1(rs); } public void disconnectRemoteSolr() { - if (this.remoteSolr == null) return; - this.remoteSolr.close(); - this.remoteSolr = null; + this.solr.disconnect1(); } public SolrConnector getLocalSolr() { - return this.localSolr; + return this.solr.getSolr0(); } public SolrConnector getRemoteSolr() { - return this.remoteSolr; + return this.solr.getSolr1(); + } + + public SolrConnector getSolr() { + return this.solr; } public void clearCache() { @@ -164,9 +165,7 @@ public final class MetadataRepository implements /*Metadata,*/ Iterable } else { this.urlIndexFile.clear(); } - if (this.localSolr != null) { - this.localSolr.clear(); - } + this.solr.clear(); // the remote solr is not cleared here because that shall be done separately this.statsDump = null; } @@ -174,8 +173,7 @@ public final class MetadataRepository implements /*Metadata,*/ Iterable public int size() { int size = 0; size += this.urlIndexFile == null ? 0 : this.urlIndexFile.size(); - size += this.localSolr == null ? 0 : this.localSolr.getSize(); - size += this.remoteSolr == null ? 0 : this.remoteSolr.getSize(); + size += this.solr.getSize(); return size; } @@ -185,14 +183,7 @@ public final class MetadataRepository implements /*Metadata,*/ Iterable this.urlIndexFile.close(); this.urlIndexFile = null; } - if (this.localSolr != null) { - this.localSolr.close(); - this.localSolr = null; - } - if (this.remoteSolr != null) { - this.remoteSolr.close(); - this.remoteSolr = null; - } + this.solr.close(); } public int writeCacheSize() { @@ -219,13 +210,11 @@ public final class MetadataRepository implements /*Metadata,*/ Iterable Log.logException(e); } /* - if (this.localSolr != null) { try { - SolrDocument doc = this.localSolr.get(ASCII.String(urlHash)); + SolrDocument doc = this.solr.get(ASCII.String(urlHash)); } catch (IOException e) { Log.logException(e); } - } */ return null; } @@ -274,18 +263,10 @@ public final class MetadataRepository implements /*Metadata,*/ Iterable public boolean remove(final byte[] urlHash) { if (urlHash == null) return false; - if (this.localSolr != null || this.remoteSolr != null) { - String urls = ASCII.String(urlHash); - if (this.localSolr != null) try { - this.localSolr.delete(urls); - } catch (final Throwable e) { - Log.logException(e); - } - if (this.remoteSolr != null) try { - this.remoteSolr.delete(urls); - } catch (final Throwable e) { - Log.logException(e); - } + try { + this.solr.delete(ASCII.String(urlHash)); + } catch (final Throwable e) { + Log.logException(e); } if (this.urlIndexFile != null) try { final Row.Entry r = this.urlIndexFile.remove(urlHash); @@ -299,22 +280,10 @@ public final class MetadataRepository implements /*Metadata,*/ Iterable public boolean exists(final byte[] urlHash) { if (urlHash == null) return false; - if (this.localSolr != null || this.remoteSolr != null) { - String urls = ASCII.String(urlHash); - try { - if (this.localSolr != null && this.localSolr.exists(urls)) { - return true; - } - } catch (final Throwable e) { - Log.logException(e); - } - try { - if (this.remoteSolr != null && this.remoteSolr.exists(urls)) { - return true; - } - } catch (final Throwable e) { - Log.logException(e); - } + try { + if (this.solr.exists(ASCII.String(urlHash))) return true; + } catch (final Throwable e) { + Log.logException(e); } if (this.urlIndexFile == null) return false; // case may happen during shutdown return this.urlIndexFile.has(urlHash); diff --git a/source/net/yacy/search/index/Segment.java b/source/net/yacy/search/index/Segment.java index d982db5eb..2faf13d25 100644 --- a/source/net/yacy/search/index/Segment.java +++ b/source/net/yacy/search/index/Segment.java @@ -190,18 +190,20 @@ public class Segment { return this.urlMetadata.connectedLocalSolr(); } - public void connectLocalSolr() throws IOException { - this.urlMetadata.connectLocalSolr(); + public void connectLocalSolr(final int connectWithin) throws IOException { + this.urlMetadata.connectLocalSolr(connectWithin); } public void disconnectLocalSolr() { this.urlMetadata.disconnectLocalSolr(); } + public SolrConnector getSolr() { + return this.urlMetadata.getSolr(); + } public SolrConnector getRemoteSolr() { return this.urlMetadata.getRemoteSolr(); } - public SolrConnector getLocalSolr() { return this.urlMetadata.getLocalSolr(); } diff --git a/source/net/yacy/search/query/SnippetProcess.java b/source/net/yacy/search/query/SnippetProcess.java index 21058971c..e2d4b24e9 100644 --- a/source/net/yacy/search/query/SnippetProcess.java +++ b/source/net/yacy/search/query/SnippetProcess.java @@ -67,7 +67,7 @@ import de.anomic.data.WorkTables; public class SnippetProcess { public static Log log = new Log("SEARCH"); - + private final static int SNIPPET_WORKER_THREADS = Math.max(4, Runtime.getRuntime().availableProcessors() * 2); // input values @@ -447,7 +447,7 @@ public class SnippetProcess { this.timeout = System.currentTimeMillis() + Math.max(1000, maxlifetime); this.neededResults = neededResults; this.shallrun = true; - this.solr = SnippetProcess.this.rankingProcess.getQuery().getSegment().getRemoteSolr(); + this.solr = SnippetProcess.this.rankingProcess.getQuery().getSegment().getSolr(); } @Override