boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

embedded solr is only initiated if it is activated with

Browse Source 
IndexFederated_p.html
pull/1/head
orbiter 13 years ago
parent 49ee31f837
commit 5a3c829872
7 changed files with 63 additions and 18 deletions
Show Stats Download Patch File Download Diff File

30
htroot/IndexFederated_p.java
Unescape View File

@ -51,13 +51,28 @@ public class IndexFederated_p {
if (post != null && post.containsKey("set")) {
// yacy
String localindex = post.get("yacy.indexing", "off");
String localindex = post.get("yacy.indexing", "off"); // possible values: classic, solr, off
final boolean solrLocalWasOn = sb.index.getLocalSolr() != null && env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_YACY_INDEXING_ENGINE, "off").equals("solr");
final boolean solrLocalIsOnAfterwards = localindex.equals("solr");
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_YACY_INDEXING_ENGINE, localindex);
if (solrLocalWasOn && !solrLocalIsOnAfterwards) {
sb.index.disconnectLocalSolr();
}
if (!solrLocalWasOn && solrLocalIsOnAfterwards) {
// switch on
try {
sb.index.connectLocalSolr();
} catch (IOException e) {
Log.logException(e);
}
}
// solr
final boolean solrWasOn = sb.index.getRemoteSolr() != null && env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, true);
final boolean solrIsOnAfterwards = post.getBoolean("solr.indexing.solrremote");
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, solrIsOnAfterwards);
final boolean solrRemoteWasOn = sb.index.getRemoteSolr() != null && env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, true);
final boolean solrRemoteIsOnAfterwards = post.getBoolean("solr.indexing.solrremote");
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, solrRemoteIsOnAfterwards);
String solrurls = post.get("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr"));
int commitWithinMs = post.getInt("solr.indexing.commitWithinMs", env.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, 180000));
boolean lazy = post.getBoolean("solr.indexing.lazy");
@ -84,13 +99,12 @@ public class IndexFederated_p {
final String schemename = post.get("solr.indexing.schemefile", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SCHEMEFILE, "solr.keys.default.list"));
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SCHEMEFILE, schemename);
if (solrWasOn) {
if (solrRemoteWasOn && !solrRemoteIsOnAfterwards) {
// switch off
sb.index.getRemoteSolr().close();
sb.index.connectRemoteSolr(null);
sb.index.disconnectRemoteSolr();
}
if (solrIsOnAfterwards) {
if (!solrRemoteWasOn && solrRemoteIsOnAfterwards) {
// switch on
final boolean usesolr = sb.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) & solrurls.length() > 0;
try {

2
htroot/solr/select.java
Unescape View File

@ -41,7 +41,7 @@ public class select {
* @param out
* @return
*/
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env, final OutputStream out) {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env, final OutputStream out) {
// this uses the methods in the jetty servlet environment and can be removed if jetty in implemented
Switchboard sb = (Switchboard) env;

8
source/net/yacy/search/Switchboard.java
Unescape View File

@ -382,6 +382,7 @@ public final class Switchboard extends serverSwitch
// initialize index
ReferenceContainer.maxReferences = getConfigInt("index.maxReferences", 0);
final File segmentsPath = new File(new File(indexPath, networkName), "SEGMENTS");
final boolean solrLocal = this.getConfig(SwitchboardConstants.FEDERATED_SERVICE_YACY_INDEXING_ENGINE, "off").equals("solr");
this.index =
new Segment(
this.log,
@ -389,7 +390,8 @@ public final class Switchboard extends serverSwitch
wordCacheMaxCount,
fileSizeMax,
this.useTailCache,
this.exceed134217727);
this.exceed134217727,
solrLocal);
// prepare a solr index profile switch list
final File solrBackupProfile = new File("defaults/solr.keys.list");
@ -1179,6 +1181,7 @@ public final class Switchboard extends serverSwitch
setConfig("heuristic.site", false);
setConfig("heuristic.blekko", false);
final boolean solrLocal = this.getConfig(SwitchboardConstants.FEDERATED_SERVICE_YACY_INDEXING_ENGINE, "off").equals("solr");
// relocate
this.peers.relocate(
this.networkRoot,
@ -1193,7 +1196,8 @@ public final class Switchboard extends serverSwitch
wordCacheMaxCount,
fileSizeMax,
this.useTailCache,
this.exceed134217727);
this.exceed134217727,
solrLocal);
this.crawlQueues.relocate(this.queuesRoot); // cannot be closed because the busy threads are working with that object
// create a crawler

2
source/net/yacy/search/index/DocumentIndex.java
Unescape View File

@ -74,7 +74,7 @@ public class DocumentIndex extends Segment
public DocumentIndex(final File segmentPath, final CallbackListener callback, final int cachesize)
throws IOException {
super(new Log("DocumentIndex"), segmentPath, cachesize, targetFileSize * 4 - 1, false, false);
super(new Log("DocumentIndex"), segmentPath, cachesize, targetFileSize * 4 - 1, false, false, true);
final int cores = Runtime.getRuntime().availableProcessors() + 1;
this.callback = callback;
this.queue = new LinkedBlockingQueue<DigestURI>(cores * 300);

22
source/net/yacy/search/index/MetadataRepository.java
Unescape View File

@ -98,6 +98,12 @@ public final class MetadataRepository implements /*Metadata,*/ Iterable<byte[]>
this.remoteSolr = solr;
}
public void disconnectRemoteSolr() {
if (this.remoteSolr == null) return;
this.remoteSolr.close();
this.remoteSolr = null;
}
public void connectLocalSolr() throws IOException {
File solrLocation = this.location;
if (solrLocation.getName().equals("default")) solrLocation = solrLocation.getParentFile();
@ -112,6 +118,12 @@ public final class MetadataRepository implements /*Metadata,*/ Iterable<byte[]>
this.localSolr = solr;
}
public void disconnectLocalSolr() {
if (this.localSolr == null) return;
this.localSolr.close();
this.localSolr = null;
}
public SolrConnector getLocalSolr() {
return this.localSolr;
}
@ -147,8 +159,14 @@ public final class MetadataRepository implements /*Metadata,*/ Iterable<byte[]>
this.urlIndexFile.close();
this.urlIndexFile = null;
}
if (this.localSolr != null) this.localSolr.close();
if (this.remoteSolr != null) this.remoteSolr.close();
if (this.localSolr != null) {
this.localSolr.close();
this.localSolr = null;
}
if (this.remoteSolr != null) {
this.remoteSolr.close();
this.remoteSolr = null;
}
}
public int writeCacheSize() {

13
source/net/yacy/search/index/Segment.java
Unescape View File

@ -108,7 +108,8 @@ public class Segment {
final int entityCacheMaxSize,
final long maxFileSize,
final boolean useTailCache,
final boolean exceed134217727) throws IOException {
final boolean exceed134217727,
final boolean connectLocalSolr) throws IOException {
log.logInfo("Initializing Segment '" + segmentPath + ".");
@ -139,7 +140,7 @@ public class Segment {
// create LURL-db
this.urlMetadata = new MetadataRepository(segmentPath, "text.urlmd", useTailCache, exceed134217727);
this.connectLocalSolr();
if (connectLocalSolr) this.connectLocalSolr();
}
public long URLCount() {
@ -157,11 +158,19 @@ public class Segment {
public void connectRemoteSolr(final SolrConnector solr) {
this.urlMetadata.connectRemoteSolr(solr);
}
public void disconnectRemoteSolr() {
this.urlMetadata.disconnectRemoteSolr();
}
public void connectLocalSolr() throws IOException {
this.urlMetadata.connectLocalSolr();
}
public void disconnectLocalSolr() {
this.urlMetadata.disconnectLocalSolr();
}
public SolrConnector getRemoteSolr() {
return this.urlMetadata.getRemoteSolr();
}

4
source/net/yacy/yacy.java
Unescape View File

@ -669,7 +669,7 @@ public final class yacy {
log,
new File(new File(indexPrimaryRoot, "freeworld"), "TEXT"),
10000,
Integer.MAX_VALUE, false, false);
Integer.MAX_VALUE, false, false, false);
final Iterator<ReferenceContainer<WordReference>> indexContainerIterator = wordIndex.termIndex().referenceContainerIterator("AAAAAAAAAAAA".getBytes(), false, false);
long urlCounter = 0, wordCounter = 0;
@ -849,7 +849,7 @@ public final class yacy {
log,
new File(new File(indexPrimaryRoot, "freeworld"), "TEXT"),
10000,
Integer.MAX_VALUE, false, false);
Integer.MAX_VALUE, false, false, false);
indexContainerIterator = WordIndex.termIndex().referenceContainerIterator(wordChunkStartHash.getBytes(), false, false);
}
int counter = 0;

Write Preview
Loading…
Cancel
Save