diff --git a/htroot/IndexControlRWIs_p.java b/htroot/IndexControlRWIs_p.java index 3db8df815..61fb29ae7 100644 --- a/htroot/IndexControlRWIs_p.java +++ b/htroot/IndexControlRWIs_p.java @@ -93,7 +93,7 @@ public class IndexControlRWIs_p { prop.put("keyhash", ""); prop.put("result", ""); prop.put("cleanup", post == null || post.containsKey("maxReferencesLimit") ? 1 : 0); - prop.put("cleanup_solr", sb.index.connectedSolr() ? 1 : 0); + prop.put("cleanup_solr", sb.index.urlMetadata().connectedSolr() ? 1 : 0); // switch off all optional forms/lists prop.put("searchresult", 0); @@ -158,9 +158,9 @@ public class IndexControlRWIs_p { if ( post.get("deleteIndex", "").equals("on") ) { segment.clear(); } - if ( post.get("deleteRemoteSolr", "").equals("on") && sb.index.connectedSolr()) { + if ( post.get("deleteRemoteSolr", "").equals("on") && sb.index.urlMetadata().connectedSolr()) { try { - sb.index.getSolr().clear(); + sb.index.urlMetadata().getSolr().clear(); } catch ( final Exception e ) { Log.logException(e); } diff --git a/htroot/IndexFederated_p.java b/htroot/IndexFederated_p.java index 8f5a434ab..9121b341c 100644 --- a/htroot/IndexFederated_p.java +++ b/htroot/IndexFederated_p.java @@ -73,23 +73,23 @@ public class IndexFederated_p { } catch (IOException e) { Log.logException(e); } // switch on boolean post_core_fulltext = post.getBoolean(SwitchboardConstants.CORE_SERVICE_FULLTEXT); - final boolean previous_core_fulltext = sb.index.connectedLocalSolr() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, false); + final boolean previous_core_fulltext = sb.index.urlMetadata().connectedLocalSolr() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, false); env.setConfig(SwitchboardConstants.CORE_SERVICE_FULLTEXT, post_core_fulltext); final int commitWithinMs = post.getInt("solr.indexing.commitWithinMs", env.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, 180000)); if (previous_core_fulltext && !post_core_fulltext) { // switch off - sb.index.disconnectLocalSolr(); - sb.index.disconnectUrlDb(); + sb.index.urlMetadata().disconnectLocalSolr(); + sb.index.urlMetadata().disconnectUrlDb(); } if (!previous_core_fulltext && post_core_fulltext) { // switch on sb.index.connectUrlDb(sb.useTailCache, sb.exceed134217727); - try { sb.index.connectLocalSolr(commitWithinMs); } catch (IOException e) { Log.logException(e); } + try { sb.index.urlMetadata().connectLocalSolr(commitWithinMs); } catch (IOException e) { Log.logException(e); } } // solr - final boolean solrRemoteWasOn = sb.index.connectedRemoteSolr() && env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, true); + final boolean solrRemoteWasOn = sb.index.urlMetadata().connectedRemoteSolr() && env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, true); final boolean solrRemoteIsOnAfterwards = post.getBoolean("solr.indexing.solrremote"); env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, solrRemoteIsOnAfterwards); String solrurls = post.get("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr")); @@ -119,7 +119,7 @@ public class IndexFederated_p { if (solrRemoteWasOn && !solrRemoteIsOnAfterwards) { // switch off - sb.index.disconnectRemoteSolr(); + sb.index.urlMetadata().disconnectRemoteSolr(); } if (!solrRemoteWasOn && solrRemoteIsOnAfterwards) { @@ -129,18 +129,18 @@ public class IndexFederated_p { if (usesolr) { SolrConnector solr = new ShardSolrConnector(solrurls, ShardSelection.Method.MODULO_HOST_MD5, 10000, true); solr.setCommitWithinMs(commitWithinMs); - sb.index.connectRemoteSolr(solr); + sb.index.urlMetadata().connectRemoteSolr(solr); } else { - sb.index.disconnectRemoteSolr(); + sb.index.urlMetadata().disconnectRemoteSolr(); } } catch (final IOException e) { Log.logException(e); - sb.index.disconnectRemoteSolr(); + sb.index.urlMetadata().disconnectRemoteSolr(); } } // read index scheme table flags - final Iterator i = sb.index.getSolrScheme().entryIterator(); + final Iterator i = sb.index.urlMetadata().getSolrScheme().entryIterator(); ConfigurationSet.Entry entry; boolean modified = false; // flag to remember changes while (i.hasNext()) { @@ -163,18 +163,18 @@ public class IndexFederated_p { } if (modified) { // save settings to config file if modified try { - sb.index.getSolrScheme().commit(); + sb.index.urlMetadata().getSolrScheme().commit(); modified = false; } catch (IOException ex) {} } } // show solr host table - if (!sb.index.connectedRemoteSolr()) { + if (!sb.index.urlMetadata().connectedRemoteSolr()) { prop.put("table", 0); } else { prop.put("table", 1); - final SolrConnector solr = sb.index.getRemoteSolr(); + final SolrConnector solr = sb.index.urlMetadata().getRemoteSolr(); final long[] size = (solr instanceof ShardSolrConnector) ? ((ShardSolrConnector) solr).getSizeList() : new long[]{((SingleSolrConnector) solr).getSize()}; final String[] urls = (solr instanceof ShardSolrConnector) ? ((ShardSolrConnector) solr).getAdminInterfaceList() : new String[]{((SingleSolrConnector) solr).getAdminInterface()}; boolean dark = false; @@ -194,7 +194,7 @@ public class IndexFederated_p { // use enum SolrField to keep defined order for(YaCySchema field : YaCySchema.values()) { prop.put("scheme_" + c + "_dark", dark ? 1 : 0); dark = !dark; - prop.put("scheme_" + c + "_checked", sb.index.getSolrScheme().contains(field.name()) ? 1 : 0); + prop.put("scheme_" + c + "_checked", sb.index.urlMetadata().getSolrScheme().contains(field.name()) ? 1 : 0); prop.putHTML("scheme_" + c + "_key", field.name()); prop.putHTML("scheme_" + c + "_solrfieldname",field.name().equalsIgnoreCase(field.getSolrFieldName()) ? "" : field.getSolrFieldName()); if (field.getComment() != null) prop.putHTML("scheme_" + c + "_comment",field.getComment()); diff --git a/htroot/api/schema_p.java b/htroot/api/schema_p.java index a2802bfbb..d8f74e00a 100644 --- a/htroot/api/schema_p.java +++ b/htroot/api/schema_p.java @@ -38,7 +38,7 @@ public class schema_p { // write scheme int c = 0; - SolrConfiguration solrScheme = sb.index.getSolrScheme(); + SolrConfiguration solrScheme = sb.index.urlMetadata().getSolrScheme(); for (YaCySchema field : YaCySchema.values()) { if (solrScheme.contains(field.name())) { prop.put("fields_" + c + "_solrname", field.getSolrFieldName()); diff --git a/htroot/gsa/searchresult.java b/htroot/gsa/searchresult.java index 67a06b8f8..105fa78bc 100644 --- a/htroot/gsa/searchresult.java +++ b/htroot/gsa/searchresult.java @@ -96,7 +96,7 @@ public class searchresult { post.put(CommonParams.ROWS, post.remove("num")); // get the embedded connector - EmbeddedSolrConnector connector = (EmbeddedSolrConnector) sb.index.getLocalSolr(); + EmbeddedSolrConnector connector = (EmbeddedSolrConnector) sb.index.urlMetadata().getLocalSolr(); if (connector == null) return null; // do the solr request diff --git a/htroot/solr/select.java b/htroot/solr/select.java index b0e93c28a..bc8543861 100644 --- a/htroot/solr/select.java +++ b/htroot/solr/select.java @@ -146,7 +146,7 @@ public class select { } // get the embedded connector - EmbeddedSolrConnector connector = (EmbeddedSolrConnector) sb.index.getLocalSolr(); + EmbeddedSolrConnector connector = (EmbeddedSolrConnector) sb.index.urlMetadata().getLocalSolr(); if (connector == null) return null; // do the solr request diff --git a/source/de/anomic/crawler/CrawlQueues.java b/source/de/anomic/crawler/CrawlQueues.java index ec35a37fa..b9c408e8a 100644 --- a/source/de/anomic/crawler/CrawlQueues.java +++ b/source/de/anomic/crawler/CrawlQueues.java @@ -82,8 +82,8 @@ public class CrawlQueues { this.log.logConfig("Starting Crawling Management"); this.noticeURL = new NoticedURL(queuePath, sb.peers.myBotIDs(), sb.useTailCache, sb.exceed134217727); FileUtils.deletedelete(new File(queuePath, ERROR_DB_FILENAME)); - this.errorURL = new ZURL(sb.index.getSolr(), sb.index.getSolrScheme(), queuePath, ERROR_DB_FILENAME, false, sb.useTailCache, sb.exceed134217727); - this.delegatedURL = new ZURL(sb.index.getSolr(), sb.index.getSolrScheme(), queuePath, DELEGATED_DB_FILENAME, true, sb.useTailCache, sb.exceed134217727); + this.errorURL = new ZURL(sb.index.urlMetadata().getSolr(), sb.index.urlMetadata().getSolrScheme(), queuePath, ERROR_DB_FILENAME, false, sb.useTailCache, sb.exceed134217727); + this.delegatedURL = new ZURL(sb.index.urlMetadata().getSolr(), sb.index.urlMetadata().getSolrScheme(), queuePath, DELEGATED_DB_FILENAME, true, sb.useTailCache, sb.exceed134217727); } public void relocate(final File newQueuePath) { @@ -94,8 +94,8 @@ public class CrawlQueues { this.noticeURL = new NoticedURL(newQueuePath, this.sb.peers.myBotIDs(), this.sb.useTailCache, this.sb.exceed134217727); FileUtils.deletedelete(new File(newQueuePath, ERROR_DB_FILENAME)); - this.errorURL = new ZURL(this.sb.index.getSolr(), this.sb.index.getSolrScheme(), newQueuePath, ERROR_DB_FILENAME, false, this.sb.useTailCache, this.sb.exceed134217727); - this.delegatedURL = new ZURL(this.sb.index.getSolr(), this.sb.index.getSolrScheme(), newQueuePath, DELEGATED_DB_FILENAME, true, this.sb.useTailCache, this.sb.exceed134217727); + this.errorURL = new ZURL(this.sb.index.urlMetadata().getSolr(), this.sb.index.urlMetadata().getSolrScheme(), newQueuePath, ERROR_DB_FILENAME, false, this.sb.useTailCache, this.sb.exceed134217727); + this.delegatedURL = new ZURL(this.sb.index.urlMetadata().getSolr(), this.sb.index.urlMetadata().getSolrScheme(), newQueuePath, DELEGATED_DB_FILENAME, true, this.sb.useTailCache, this.sb.exceed134217727); } public synchronized void close() { diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 6e8ed4e54..40a739a60 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -397,7 +397,7 @@ public final class Switchboard extends serverSwitch if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, true)) { this.index.connectUrlDb(this.useTailCache, this.exceed134217727); - this.index.connectLocalSolr(connectWithinMs); + this.index.urlMetadata().connectLocalSolr(connectWithinMs); } // set up the solr interface @@ -411,7 +411,7 @@ public final class Switchboard extends serverSwitch ShardSelection.Method.MODULO_HOST_MD5, 10000, true); solr.setCommitWithinMs(connectWithinMs); - this.index.connectRemoteSolr(solr); + this.index.urlMetadata().connectRemoteSolr(solr); } catch ( final IOException e ) { Log.logException(e); } @@ -1133,7 +1133,7 @@ public final class Switchboard extends serverSwitch synchronized ( this ) { // remember the solr scheme - SolrConfiguration solrScheme = this.index.getSolrScheme(); + SolrConfiguration solrScheme = this.index.urlMetadata().getSolrScheme(); // shut down this.crawler.close(); @@ -1186,7 +1186,7 @@ public final class Switchboard extends serverSwitch if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, true)) this.index.connectRWI(wordCacheMaxCount, fileSizeMax); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, true)) { - this.index.connectLocalSolr(connectWithinMs); + this.index.urlMetadata().connectLocalSolr(connectWithinMs); this.index.connectUrlDb(this.useTailCache, this.exceed134217727); } @@ -1201,7 +1201,7 @@ public final class Switchboard extends serverSwitch ShardSelection.Method.MODULO_HOST_MD5, 10000, true); solr.setCommitWithinMs(connectWithinMs); - this.index.connectRemoteSolr(solr); + this.index.urlMetadata().connectRemoteSolr(solr); } catch ( final IOException e ) { Log.logException(e); } diff --git a/source/net/yacy/search/index/DocumentIndex.java b/source/net/yacy/search/index/DocumentIndex.java index 79aebaf56..7327484b6 100644 --- a/source/net/yacy/search/index/DocumentIndex.java +++ b/source/net/yacy/search/index/DocumentIndex.java @@ -80,7 +80,7 @@ public class DocumentIndex extends Segment { false, // useTailCache false // exceed134217727 ); - super.connectLocalSolr(1000); + super.urlMetadata().connectLocalSolr(1000); final int cores = Runtime.getRuntime().availableProcessors() + 1; this.callback = callback; this.queue = new LinkedBlockingQueue(cores * 300); diff --git a/source/net/yacy/search/index/Segment.java b/source/net/yacy/search/index/Segment.java index 0573a71d6..7d1fba525 100644 --- a/source/net/yacy/search/index/Segment.java +++ b/source/net/yacy/search/index/Segment.java @@ -39,7 +39,6 @@ import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.document.UTF8; import net.yacy.cora.order.ByteOrder; import net.yacy.cora.protocol.ResponseHeader; -import net.yacy.cora.services.federated.solr.SolrConnector; import net.yacy.cora.services.federated.solr.SolrDoc; import net.yacy.cora.services.federated.yacy.CacheStrategy; import net.yacy.cora.storage.HandleSet; @@ -162,61 +161,10 @@ public class Segment { this.urlCitationIndex = null; } - public boolean connectedUrlDb() { - return this.urlMetadata.connectedUrlDb(); - } - public void connectUrlDb(final boolean useTailCache, final boolean exceed134217727) { this.urlMetadata.connectUrlDb(UrlDbName, useTailCache, exceed134217727); } - public void disconnectUrlDb() { - this.urlMetadata.disconnectUrlDb(); - } - - public boolean connectedSolr() { - return this.urlMetadata.connectedSolr(); - } - - public boolean connectedRemoteSolr() { - return this.urlMetadata.connectedRemoteSolr(); - } - - public void connectRemoteSolr(final SolrConnector solr) { - this.urlMetadata.connectRemoteSolr(solr); - } - - public void disconnectRemoteSolr() { - this.urlMetadata.disconnectRemoteSolr(); - } - - public boolean connectedLocalSolr() { - return this.urlMetadata.connectedLocalSolr(); - } - - public void connectLocalSolr(final int connectWithin) throws IOException { - this.urlMetadata.connectLocalSolr(connectWithin); - } - - public void disconnectLocalSolr() { - this.urlMetadata.disconnectLocalSolr(); - } - - public SolrConnector getSolr() { - return this.urlMetadata.getSolr(); - } - - public SolrConfiguration getSolrScheme() { - return this.urlMetadata.getSolrScheme(); - } - - public SolrConnector getRemoteSolr() { - return this.urlMetadata.getRemoteSolr(); - } - public SolrConnector getLocalSolr() { - return this.urlMetadata.getLocalSolr(); - } - public MetadataRepository urlMetadata() { return this.urlMetadata; } @@ -452,13 +400,13 @@ public class Segment { document.getVideolinks().size(), // lvideo document.getApplinks().size() // lapp ); - + // STORE TO SOLR // we do not store the data in metadatadb any more if a solr is connected - if (this.connectedSolr()) { + if (this.urlMetadata.connectedSolr()) { try { SolrDoc solrDoc = this.urlMetadata.getSolrScheme().yacy2solr(id, responseHeader, document, metadata); - this.getSolr().add(solrDoc); + this.urlMetadata.getSolr().add(solrDoc); } catch ( final IOException e ) { Log.logWarning("SOLR", "failed to send " + urlNormalform + " to solr: " + e.getMessage()); } diff --git a/source/net/yacy/search/query/RWIProcess.java b/source/net/yacy/search/query/RWIProcess.java index 71e37cb71..f38892201 100644 --- a/source/net/yacy/search/query/RWIProcess.java +++ b/source/net/yacy/search/query/RWIProcess.java @@ -208,7 +208,7 @@ public final class RWIProcess extends Thread String solrQuery = RWIProcess.this.query.solrQuery(); try { ReferenceContainer wr = ReferenceContainer.emptyContainer(Segment.wordReferenceFactory, null); - SolrDocumentList sdl = RWIProcess.this.query.getSegment().getSolr().query(solrQuery, 0, 20); + SolrDocumentList sdl = RWIProcess.this.query.getSegment().urlMetadata().getSolr().query(solrQuery, 0, 20); for (SolrDocument d : sdl) { try {wr.add(new WordReferenceVars(d));} catch (SpaceExceededException e) {} }