boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

FOR THE BRAVE.. this is a forced migration to solr which is now ready

Browse Source 
for production as a replacement of the metadata-db.
This intermediate release 1.041 will switch on the previously optional
solr index and the old metadata-db will still work as it did before.
Solr+metadata are accessed in mixed mode, no migration is done yet.
If this causes not a catastrophe until the end of the weekend, we will
do a YaCy 1.1 main release containing this as default.
pull/1/head
Michael Peter Christen 13 years ago
parent a1b2c9a67d
commit 23226676c6
6 changed files with 32 additions and 27 deletions
Show Stats Download Patch File Download Diff File

2
build.properties
Unescape View File

@ -3,7 +3,7 @@ javacSource=1.6
javacTarget=1.6 javacTarget=1.6
# Release Configuration # Release Configuration
releaseVersion=1.04 releaseVersion=1.041
stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
sourceReleaseFile=yacy_src_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz sourceReleaseFile=yacy_src_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
releaseFileParentDir=yacy releaseFileParentDir=yacy

8
defaults/yacy.init
Unescape View File

@ -1048,14 +1048,14 @@ federated.service.solr.indexing.schemefile = solr.keys.default.list
# the lazy attribute causes that fields containing "" or 0 are not added and not written # the lazy attribute causes that fields containing "" or 0 are not added and not written
federated.service.solr.indexing.lazy = true federated.service.solr.indexing.lazy = true
# temporary definition of backend services to use. The standard is urldb+rwi, but in the future it should be rwi+solr # temporary definition of backend services to use.
# to get a handle for a migration, these values are defined as temporary, if the migration starts the values are renamed # After the migration a rwi+solr combination is used, the solr contains the content of the previously used metadata-db.
# To get a handle for a migration, these values are defined as temporary, if the migration starts the values are renamed
# and defined with different default values. # and defined with different default values.
# The citation service is used for ranking; this is a reverse linking index. It should be on before and after the migration. # The citation service is used for ranking; this is a reverse linking index. It should be on before and after the migration.
# It can be switched off if only a remote solr index is used. # It can be switched off if only a remote solr index is used.
core.service.urldb.tmp = true core.service.fulltext = true
core.service.rwi.tmp = true core.service.rwi.tmp = true
core.service.solr.tmp = false
core.service.citation.tmp = true core.service.citation.tmp = true
# RDF triplestore settings # RDF triplestore settings

3
htroot/IndexFederated_p.html
Unescape View File

@ -26,9 +26,8 @@
This is an experimental switchboard to test an index migration from embedded metadata to embedded solr. The 'classic' configuration is rwi + metadata switched on. The future configuration is rwi + solr switched on. This is an experimental switchboard to test an index migration from embedded metadata to embedded solr. The 'classic' configuration is rwi + metadata switched on. The future configuration is rwi + solr switched on.
The rwi index is necessary for index transmission and shall be switched off in future portalmode configurations. The rwi index is necessary for index transmission and shall be switched off in future portalmode configurations.
<dl> <dl>
<dt><input type="checkbox" name="core.service.fulltext" id="core.service.fulltext" #(core.service.fulltext.checked)#:: checked="checked"#(/core.service.fulltext.checked)# /></dt><dd>url metadata and embedded solr fulltext search index, interface at: <a href="/solr/select?q=*:*&start=0&rows=10" target="_blank">/solr/select?q=*:*&amp;start=0&amp;rows=10</a></dd>
<dt><input type="checkbox" name="core.service.rwi.tmp" id="core.service.rwi" #(core.service.rwi.tmp.checked)#:: checked="checked"#(/core.service.rwi.tmp.checked)# /></dt><dd>embedded 'classic' rwi index</dd> <dt><input type="checkbox" name="core.service.rwi.tmp" id="core.service.rwi" #(core.service.rwi.tmp.checked)#:: checked="checked"#(/core.service.rwi.tmp.checked)# /></dt><dd>embedded 'classic' rwi index</dd>
<dt><input type="checkbox" name="core.service.urldb.tmp" id="core.service.urldb" #(core.service.urldb.tmp.checked)#:: checked="checked"#(/core.service.urldb.tmp.checked)# /></dt><dd>embedded 'classic' metadata index</dd>
<dt><input type="checkbox" name="core.service.solr.tmp" id="core.service.solr" #(core.service.solr.tmp.checked)#:: checked="checked"#(/core.service.solr.tmp.checked)# /></dt><dd>embedded solr search index, interface at: <a href="/solr/select?q=*:*&start=0&rows=10" target="_blank">/solr/select?q=*:*&amp;start=0&amp;rows=10</a></dd>
<dt><input type="checkbox" name="core.service.citation.tmp" id="core.service.citation" #(core.service.citation.tmp.checked)#:: checked="checked"#(/core.service.citation.tmp.checked)# /></dt><dd>embedded citation reference index (link structure, used for ranking)</dd> <dt><input type="checkbox" name="core.service.citation.tmp" id="core.service.citation" #(core.service.citation.tmp.checked)#:: checked="checked"#(/core.service.citation.tmp.checked)# /></dt><dd>embedded citation reference index (link structure, used for ranking)</dd>
<dt></dt><dd><input type="submit" name="set" value="Set" /></dd> <dt></dt><dd><input type="submit" name="set" value="Set" /></dd>
</dl> </dl>

28
htroot/IndexFederated_p.java
Unescape View File

@ -72,18 +72,21 @@ public class IndexFederated_p {
sb.index.connectCitation(wordCacheMaxCount, fileSizeMax); sb.index.connectCitation(wordCacheMaxCount, fileSizeMax);
} catch (IOException e) { Log.logException(e); } // switch on } catch (IOException e) { Log.logException(e); } // switch on
boolean post_core_fulltext = post.getBoolean(SwitchboardConstants.CORE_SERVICE_FULLTEXT);
final boolean previous_core_fulltext = sb.index.connectedLocalSolr() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, false);
env.setConfig(SwitchboardConstants.CORE_SERVICE_FULLTEXT, post_core_fulltext);
final int commitWithinMs = post.getInt("solr.indexing.commitWithinMs", env.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, 180000)); final int commitWithinMs = post.getInt("solr.indexing.commitWithinMs", env.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, 180000));
boolean post_core_solr = post.getBoolean(SwitchboardConstants.CORE_SERVICE_SOLR); if (previous_core_fulltext && !post_core_fulltext) {
final boolean previous_core_solr = sb.index.connectedLocalSolr() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_SOLR, false); // switch off
env.setConfig(SwitchboardConstants.CORE_SERVICE_SOLR, post_core_solr); sb.index.disconnectLocalSolr();
if (previous_core_solr && !post_core_solr) sb.index.disconnectLocalSolr(); // switch off sb.index.disconnectUrlDb();
if (!previous_core_solr && post_core_solr) try { sb.index.connectLocalSolr(commitWithinMs); } catch (IOException e) { Log.logException(e); } // switch on }
if (!previous_core_fulltext && post_core_fulltext) {
boolean post_core_urldb = post.getBoolean(SwitchboardConstants.CORE_SERVICE_URLDB); // switch on
final boolean previous_core_urldb = sb.index.connectedUrlDb() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_URLDB, false); sb.index.connectUrlDb(sb.useTailCache, sb.exceed134217727);
env.setConfig(SwitchboardConstants.CORE_SERVICE_URLDB, post_core_urldb); try { sb.index.connectLocalSolr(commitWithinMs); } catch (IOException e) { Log.logException(e); }
if (previous_core_urldb && !post_core_urldb) sb.index.disconnectUrlDb(); // switch off }
if (!previous_core_urldb && post_core_urldb) sb.index.connectUrlDb(sb.useTailCache, sb.exceed134217727);
// solr // solr
final boolean solrRemoteWasOn = sb.index.connectedRemoteSolr() && env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, true); final boolean solrRemoteWasOn = sb.index.connectedRemoteSolr() && env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, true);
@ -203,9 +206,8 @@ public class IndexFederated_p {
// allowed values are: classic, solr, off // allowed values are: classic, solr, off
// federated.service.yacy.indexing.engine = classic // federated.service.yacy.indexing.engine = classic
prop.put(SwitchboardConstants.CORE_SERVICE_URLDB + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_URLDB, false) ? 1 : 0); prop.put(SwitchboardConstants.CORE_SERVICE_FULLTEXT + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, false) ? 1 : 0);
prop.put(SwitchboardConstants.CORE_SERVICE_RWI + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, false) ? 1 : 0); prop.put(SwitchboardConstants.CORE_SERVICE_RWI + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, false) ? 1 : 0);
prop.put(SwitchboardConstants.CORE_SERVICE_SOLR + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_SOLR, false) ? 1 : 0);
prop.put(SwitchboardConstants.CORE_SERVICE_CITATION + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, false) ? 1 : 0); prop.put(SwitchboardConstants.CORE_SERVICE_CITATION + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, false) ? 1 : 0);
prop.put("solr.indexing.solrremote.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) ? 1 : 0); prop.put("solr.indexing.solrremote.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) ? 1 : 0);
prop.put("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr").replace(",", "\n")); prop.put("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr").replace(",", "\n"));

13
source/net/yacy/search/Switchboard.java
Unescape View File

@ -395,9 +395,10 @@ public final class Switchboard extends serverSwitch
final int connectWithinMs = this.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, 180000); final int connectWithinMs = this.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, 180000);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, true)) this.index.connectRWI(wordCacheMaxCount, fileSizeMax); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, true)) this.index.connectRWI(wordCacheMaxCount, fileSizeMax);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_URLDB, true)) this.index.connectUrlDb(this.useTailCache, this.exceed134217727); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, true)) {
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_SOLR, true)) this.index.connectLocalSolr(connectWithinMs); this.index.connectUrlDb(this.useTailCache, this.exceed134217727);
this.index.connectLocalSolr(connectWithinMs);
}
// set up the solr interface // set up the solr interface
final String solrurls = getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr"); final String solrurls = getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr");
@ -1184,8 +1185,10 @@ public final class Switchboard extends serverSwitch
final int connectWithinMs = this.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, 180000); final int connectWithinMs = this.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, 180000);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, true)) this.index.connectRWI(wordCacheMaxCount, fileSizeMax); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, true)) this.index.connectRWI(wordCacheMaxCount, fileSizeMax);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_SOLR, true)) this.index.connectLocalSolr(connectWithinMs); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, true)) {
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_URLDB, true)) this.index.connectUrlDb(this.useTailCache, this.exceed134217727); this.index.connectLocalSolr(connectWithinMs);
this.index.connectUrlDb(this.useTailCache, this.exceed134217727);
}
// set up the solr interface // set up the solr interface
final String solrurls = getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr"); final String solrurls = getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr");

5
source/net/yacy/search/SwitchboardConstants.java
Unescape View File

@ -297,9 +297,10 @@ public final class SwitchboardConstants {
public static final String FEDERATED_SERVICE_SOLR_INDEXING_SCHEMEFILE = "federated.service.solr.indexing.schemefile"; public static final String FEDERATED_SERVICE_SOLR_INDEXING_SCHEMEFILE = "federated.service.solr.indexing.schemefile";
public static final String FEDERATED_SERVICE_SOLR_INDEXING_LAZY = "federated.service.solr.indexing.lazy"; public static final String FEDERATED_SERVICE_SOLR_INDEXING_LAZY = "federated.service.solr.indexing.lazy";
public static final String CORE_SERVICE_URLDB = "core.service.urldb.tmp"; //public static final String CORE_SERVICE_URLDB = "core.service.urldb.tmp";
//public static final String CORE_SERVICE_SOLR = "core.service.solr.tmp";
public static final String CORE_SERVICE_FULLTEXT = "core.service.fulltext";
public static final String CORE_SERVICE_RWI = "core.service.rwi.tmp"; public static final String CORE_SERVICE_RWI = "core.service.rwi.tmp";
public static final String CORE_SERVICE_SOLR = "core.service.solr.tmp";
public static final String CORE_SERVICE_CITATION = "core.service.citation.tmp"; public static final String CORE_SERVICE_CITATION = "core.service.citation.tmp";
/** /**

Write Preview
Loading…
Cancel
Save