boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

migration to Solr 5.2: huge benefits - this is a lot faster!

Browse Source 
This is a very complex migration: many classes had been renamed or
removed, dependencies changed and the solr index type is now aligned to
be a solr cloud repository.
Together with the Solr 5.2 library update, one other dependent library
had been updated as well: httpclient 4.4->4.4.1

Older indexes are migrated from 4_10 to 5_2. However, the new index
structure is more efficient and we recommend to re-index everything.
Please use the index export before you do the update to a large
surrogate xml file. After the update, start with an empty index and then
initialize this with your dump.
pull/8/head
Michael Peter Christen 10 years ago
parent e427efbe54
commit 694b22f165
59 changed files with 327 additions and 589 deletions
Show Stats Download Patch File Download Diff File

191
.classpath
Unescape View File

@ -1,94 +1,97 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="source"/>
<classpathentry excluding="api/|env/|processing/domaingraph/applet/|yacy/|api/bookmarks/|api/ymarks/|api/bookmarks/posts/|api/bookmarks/tags/|api/bookmarks/xbel/|solr/|gsa/|solr/collection1/|api/blacklists/" kind="src" path="htroot"/>
<classpathentry excluding="bookmarks/|ymarks/|bookmarks/posts/|bookmarks/tags/|bookmarks/xbel/|blacklists/" kind="src" path="htroot/api"/>
<classpathentry kind="src" path="htroot/env"/>
<classpathentry kind="src" path="htroot/yacy"/>
<classpathentry excluding="posts/|tags/|xbel/" kind="src" path="htroot/api/bookmarks"/>
<classpathentry kind="src" path="htroot/api/ymarks"/>
<classpathentry kind="src" path="htroot/api/bookmarks/posts"/>
<classpathentry kind="src" path="htroot/api/bookmarks/tags"/>
<classpathentry kind="src" path="htroot/api/bookmarks/xbel"/>
<classpathentry kind="src" path="htroot/api/blacklists"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="lib" path="lib/commons-logging-1.2.jar"/>
<classpathentry kind="lib" path="lib/J7Zip-modified.jar"/>
<classpathentry kind="lib" path="lib/webcat-0.1-swf.jar"/>
<classpathentry kind="lib" path="lib/commons-jxpath-1.3.jar"/>
<classpathentry kind="lib" path="lib/jsch-0.1.52.jar"/>
<classpathentry kind="lib" path="lib/jakarta-oro-2.0.8.jar"/>
<classpathentry kind="lib" path="lib/apache-mime4j-0.6.jar"/>
<classpathentry kind="lib" path="lib/commons-fileupload-1.3.1.jar"/>
<classpathentry kind="lib" path="lib/json-simple-1.1.1.jar"/>
<classpathentry kind="lib" path="lib/xercesImpl.jar"/>
<classpathentry kind="lib" path="lib/xml-apis.jar"/>
<classpathentry kind="lib" path="lib/geronimo-stax-api_1.0_spec-1.0.1.jar"/>
<classpathentry kind="lib" path="lib/commons-compress-1.9.jar"/>
<classpathentry kind="lib" path="lib/commons-lang-2.6.jar"/>
<classpathentry kind="lib" path="lib/jaudiotagger-2.0.4-20111207.115108-15.jar"/>
<classpathentry kind="lib" path="lib/commons-codec-1.10.jar"/>
<classpathentry kind="lib" path="lib/jcl-over-slf4j-1.7.9.jar"/>
<classpathentry kind="lib" path="lib/wstx-asl-3.2.9.jar"/>
<classpathentry kind="lib" path="lib/slf4j-jdk14-1.7.9.jar"/>
<classpathentry kind="lib" path="lib/log4j-over-slf4j-1.7.9.jar"/>
<classpathentry kind="lib" path="lib/httpclient-4.4.jar"/>
<classpathentry kind="lib" path="lib/httpcore-4.4.1.jar"/>
<classpathentry kind="lib" path="lib/httpmime-4.4.jar"/>
<classpathentry kind="lib" path="lib/noggit-0.5.jar"/>
<classpathentry kind="lib" path="lib/metadata-extractor-2.8.1.jar"/>
<classpathentry kind="lib" path="lib/xmpcore-5.1.2.jar"/>
<classpathentry kind="lib" path="lib/jcifs-1.3.17.jar"/>
<classpathentry kind="lib" path="lib/guava-18.0.jar"/>
<classpathentry kind="lib" path="lib/spatial4j-0.4.1.jar"/>
<classpathentry kind="lib" path="lib/zookeeper-3.4.6.jar"/>
<classpathentry kind="lib" path="lib/org.restlet.jar"/>
<classpathentry kind="lib" path="lib/fontbox-1.8.9.jar"/>
<classpathentry kind="lib" path="lib/jempbox-1.8.9.jar"/>
<classpathentry kind="lib" path="lib/pdfbox-1.8.9.jar"/>
<classpathentry kind="lib" path="lib/bcmail-jdk15-1.46.jar"/>
<classpathentry kind="lib" path="lib/bcprov-jdk15-1.46.jar"/>
<classpathentry kind="lib" path="lib/poi-3.12-20150511.jar"/>
<classpathentry kind="lib" path="lib/poi-scratchpad-3.12-20150511.jar"/>
<classpathentry kind="lib" path="lib/commons-io-2.4.jar"/>
<classpathentry kind="lib" path="lib/slf4j-api-1.7.9.jar"/>
<classpathentry kind="lib" path="lib/lucene-analyzers-common-4.10.3.jar"/>
<classpathentry kind="lib" path="lib/lucene-analyzers-phonetic-4.10.3.jar"/>
<classpathentry kind="lib" path="lib/lucene-classification-4.10.3.jar"/>
<classpathentry kind="lib" path="lib/lucene-codecs-4.10.3.jar"/>
<classpathentry kind="lib" path="lib/lucene-core-4.10.3.jar"/>
<classpathentry kind="lib" path="lib/lucene-facet-4.10.3.jar"/>
<classpathentry kind="lib" path="lib/lucene-grouping-4.10.3.jar"/>
<classpathentry kind="lib" path="lib/lucene-highlighter-4.10.3.jar"/>
<classpathentry kind="lib" path="lib/lucene-join-4.10.3.jar"/>
<classpathentry kind="lib" path="lib/lucene-memory-4.10.3.jar"/>
<classpathentry kind="lib" path="lib/lucene-misc-4.10.3.jar"/>
<classpathentry kind="lib" path="lib/lucene-queries-4.10.3.jar"/>
<classpathentry kind="lib" path="lib/lucene-queryparser-4.10.3.jar"/>
<classpathentry kind="lib" path="lib/lucene-spatial-4.10.3.jar"/>
<classpathentry kind="lib" path="lib/lucene-suggest-4.10.3.jar"/>
<classpathentry kind="lib" path="lib/solr-core-4.10.3.jar"/>
<classpathentry kind="lib" path="lib/solr-solrj-4.10.3.jar"/>
<classpathentry kind="lib" path="lib/chardet.jar"/>
<classpathentry kind="lib" path="lib/jetty-client-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-continuation-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-deploy-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-http-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-io-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-jmx-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-proxy-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-security-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-server-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-servlet-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-servlets-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-util-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-webapp-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-xml-9.2.11.v20150529.jar"/>
<classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/>
<classpathentry kind="lib" path="lib/icu4j-core.jar"/>
<classpathentry kind="lib" path="lib/htmllexer.jar"/>
<classpathentry kind="lib" path="lib/jsoup-1.8.2.jar"/>
<classpathentry kind="lib" path="lib/javax.servlet-api-3.1.0.jar"/>
<classpathentry kind="lib" path="lib/weupnp-0.1.2.jar"/>
<classpathentry kind="output" path="gen"/>
</classpath>
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="source"/>
<classpathentry excluding="api/|env/|processing/domaingraph/applet/|yacy/|api/bookmarks/|api/ymarks/|api/bookmarks/posts/|api/bookmarks/tags/|api/bookmarks/xbel/|solr/|gsa/|solr/collection1/|api/blacklists/" kind="src" path="htroot"/>
<classpathentry excluding="bookmarks/|ymarks/|bookmarks/posts/|bookmarks/tags/|bookmarks/xbel/|blacklists/" kind="src" path="htroot/api"/>
<classpathentry kind="src" path="htroot/env"/>
<classpathentry kind="src" path="htroot/yacy"/>
<classpathentry excluding="posts/|tags/|xbel/" kind="src" path="htroot/api/bookmarks"/>
<classpathentry kind="src" path="htroot/api/ymarks"/>
<classpathentry kind="src" path="htroot/api/bookmarks/posts"/>
<classpathentry kind="src" path="htroot/api/bookmarks/tags"/>
<classpathentry kind="src" path="htroot/api/bookmarks/xbel"/>
<classpathentry kind="src" path="htroot/api/blacklists"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="lib" path="lib/commons-logging-1.2.jar"/>
<classpathentry kind="lib" path="lib/J7Zip-modified.jar"/>
<classpathentry kind="lib" path="lib/webcat-0.1-swf.jar"/>
<classpathentry kind="lib" path="lib/commons-jxpath-1.3.jar"/>
<classpathentry kind="lib" path="lib/jsch-0.1.52.jar"/>
<classpathentry kind="lib" path="lib/jakarta-oro-2.0.8.jar"/>
<classpathentry kind="lib" path="lib/apache-mime4j-0.6.jar"/>
<classpathentry kind="lib" path="lib/commons-fileupload-1.3.1.jar"/>
<classpathentry kind="lib" path="lib/json-simple-1.1.1.jar"/>
<classpathentry kind="lib" path="lib/xercesImpl.jar"/>
<classpathentry kind="lib" path="lib/xml-apis.jar"/>
<classpathentry kind="lib" path="lib/geronimo-stax-api_1.0_spec-1.0.1.jar"/>
<classpathentry kind="lib" path="lib/commons-compress-1.9.jar"/>
<classpathentry kind="lib" path="lib/commons-lang-2.6.jar"/>
<classpathentry kind="lib" path="lib/jaudiotagger-2.0.4-20111207.115108-15.jar"/>
<classpathentry kind="lib" path="lib/commons-codec-1.10.jar"/>
<classpathentry kind="lib" path="lib/jcl-over-slf4j-1.7.9.jar"/>
<classpathentry kind="lib" path="lib/wstx-asl-3.2.9.jar"/>
<classpathentry kind="lib" path="lib/slf4j-jdk14-1.7.9.jar"/>
<classpathentry kind="lib" path="lib/log4j-over-slf4j-1.7.9.jar"/>
<classpathentry kind="lib" path="lib/httpcore-4.4.1.jar"/>
<classpathentry kind="lib" path="lib/metadata-extractor-2.8.1.jar"/>
<classpathentry kind="lib" path="lib/xmpcore-5.1.2.jar"/>
<classpathentry kind="lib" path="lib/jcifs-1.3.17.jar"/>
<classpathentry kind="lib" path="lib/guava-18.0.jar"/>
<classpathentry kind="lib" path="lib/spatial4j-0.4.1.jar"/>
<classpathentry kind="lib" path="lib/zookeeper-3.4.6.jar"/>
<classpathentry kind="lib" path="lib/org.restlet.jar"/>
<classpathentry kind="lib" path="lib/fontbox-1.8.9.jar"/>
<classpathentry kind="lib" path="lib/jempbox-1.8.9.jar"/>
<classpathentry kind="lib" path="lib/pdfbox-1.8.9.jar"/>
<classpathentry kind="lib" path="lib/bcmail-jdk15-1.46.jar"/>
<classpathentry kind="lib" path="lib/bcprov-jdk15-1.46.jar"/>
<classpathentry kind="lib" path="lib/poi-3.12-20150511.jar"/>
<classpathentry kind="lib" path="lib/poi-scratchpad-3.12-20150511.jar"/>
<classpathentry kind="lib" path="lib/commons-io-2.4.jar"/>
<classpathentry kind="lib" path="lib/slf4j-api-1.7.9.jar"/>
<classpathentry kind="lib" path="lib/chardet.jar"/>
<classpathentry kind="lib" path="lib/jetty-client-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-continuation-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-deploy-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-http-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-io-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-jmx-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-proxy-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-security-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-server-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-servlet-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-servlets-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-util-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-webapp-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-xml-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/httpclient-4.4.1.jar"/>
<classpathentry kind="lib" path="lib/httpmime-4.4.1.jar"/>
<classpathentry kind="lib" path="lib/noggit-0.6.jar"/>
<classpathentry kind="lib" path="lib/solr-core-5.2.0.jar" sourcepath="/Volumes/Uno/noBackup/Downloads/solr-5.2.0/solr/core/src/java"/>
<classpathentry kind="lib" path="lib/solr-solrj-5.2.0.jar" sourcepath="/Volumes/Uno/noBackup/Downloads/solr-5.2.0/solr/solrj/src/java"/>
<classpathentry kind="lib" path="lib/stax2-api-3.1.4.jar"/>
<classpathentry kind="lib" path="lib/woodstox-core-asl-4.4.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-analyzers-common-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-analyzers-phonetic-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-classification-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-codecs-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-core-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-facet-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-grouping-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-highlighter-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-join-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-memory-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-misc-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-queries-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-queryparser-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-spatial-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-suggest-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-backward-codecs-5.2.0.jar"/>
<classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/>
<classpathentry kind="lib" path="lib/icu4j-core.jar"/>
<classpathentry kind="lib" path="lib/htmllexer.jar"/>
<classpathentry kind="lib" path="lib/jsoup-1.8.2.jar"/>
<classpathentry kind="lib" path="lib/javax.servlet-api-3.1.0.jar"/>
<classpathentry kind="lib" path="lib/weupnp-0.1.2.jar"/>
<classpathentry kind="output" path="gen"/>
</classpath>

41
build.xml
Unescape View File

@ -176,17 +176,17 @@
<pathelement location="${lib}/geronimo-stax-api_1.0_spec-1.0.1.jar" />
<pathelement location="${lib}/guava-18.0.jar" />
<pathelement location="${lib}/htmllexer.jar" />
<pathelement location="${lib}/httpclient-4.4.jar" />
<pathelement location="${lib}/httpclient-4.4.1.jar" />
<pathelement location="${lib}/httpcore-4.4.1.jar" />
<pathelement location="${lib}/httpmime-4.4.jar" />
<pathelement location="${lib}/httpmime-4.4.1.jar" />
<pathelement location="${lib}/icu4j-core.jar" />
<pathelement location="${lib}/J7Zip-modified.jar" />
<pathelement location="${lib}/jakarta-oro-2.0.8.jar" />
<pathelement location="${lib}/jaudiotagger-2.0.4-20111207.115108-15.jar" />
<pathelement location="${lib}/javax.servlet-api-3.1.0.jar" />
<pathelement location="${lib}/jcifs-1.3.17.jar" />
<pathelement location="${lib}/jcl-over-slf4j-1.7.9.jar" />
<pathelement location="${lib}/jempbox-1.8.9.jar" />
<pathelement location="${lib}/javax.servlet-api-3.1.0.jar" />
<pathelement location="${lib}/jetty-client-9.2.11.v20150529.jar" />
<pathelement location="${lib}/jetty-continuation-9.2.11.v20150529.jar" />
<pathelement location="${lib}/jetty-deploy-9.2.11.v20150529.jar" />
@ -205,21 +205,22 @@
<pathelement location="${lib}/json-simple-1.1.1.jar" />
<pathelement location="${lib}/jsoup-1.8.2.jar" />
<pathelement location="${lib}/log4j-over-slf4j-1.7.9.jar" />
<pathelement location="${lib}/lucene-analyzers-common-4.10.3.jar" />
<pathelement location="${lib}/lucene-analyzers-phonetic-4.10.3.jar" />
<pathelement location="${lib}/lucene-classification-4.10.3.jar" />
<pathelement location="${lib}/lucene-codecs-4.10.3.jar" />
<pathelement location="${lib}/lucene-core-4.10.3.jar" />
<pathelement location="${lib}/lucene-facet-4.10.3.jar" />
<pathelement location="${lib}/lucene-grouping-4.10.3.jar" />
<pathelement location="${lib}/lucene-highlighter-4.10.3.jar" />
<pathelement location="${lib}/lucene-join-4.10.3.jar" />
<pathelement location="${lib}/lucene-memory-4.10.3.jar" />
<pathelement location="${lib}/lucene-misc-4.10.3.jar" />
<pathelement location="${lib}/lucene-queries-4.10.3.jar" />
<pathelement location="${lib}/lucene-queryparser-4.10.3.jar" />
<pathelement location="${lib}/lucene-spatial-4.10.3.jar" />
<pathelement location="${lib}/lucene-suggest-4.10.3.jar" />
<pathelement location="${lib}/lucene-analyzers-common-5.2.0.jar" />
<pathelement location="${lib}/lucene-analyzers-phonetic-5.2.0.jar" />
<pathelement location="${lib}/lucene-backward-codecs-5.2.0.jar" />
<pathelement location="${lib}/lucene-classification-5.2.0.jar" />
<pathelement location="${lib}/lucene-codecs-5.2.0.jar" />
<pathelement location="${lib}/lucene-core-5.2.0.jar" />
<pathelement location="${lib}/lucene-facet-5.2.0.jar" />
<pathelement location="${lib}/lucene-grouping-5.2.0.jar" />
<pathelement location="${lib}/lucene-highlighter-5.2.0.jar" />
<pathelement location="${lib}/lucene-join-5.2.0.jar" />
<pathelement location="${lib}/lucene-memory-5.2.0.jar" />
<pathelement location="${lib}/lucene-misc-5.2.0.jar" />
<pathelement location="${lib}/lucene-queries-5.2.0.jar" />
<pathelement location="${lib}/lucene-queryparser-5.2.0.jar" />
<pathelement location="${lib}/lucene-spatial-5.2.0.jar" />
<pathelement location="${lib}/lucene-suggest-5.2.0.jar" />
<pathelement location="${lib}/metadata-extractor-2.8.1.jar" />
<pathelement location="${lib}/noggit-0.5.jar" />
<pathelement location="${lib}/org.restlet.jar" />
@ -228,8 +229,8 @@
<pathelement location="${lib}/poi-scratchpad-3.12-20150511.jar" />
<pathelement location="${lib}/slf4j-api-1.7.9.jar" />
<pathelement location="${lib}/slf4j-jdk14-1.7.9.jar" />
<pathelement location="${lib}/solr-core-4.10.3.jar" />
<pathelement location="${lib}/solr-solrj-4.10.3.jar" />
<pathelement location="${lib}/solr-core-5.2.0.jar" />
<pathelement location="${lib}/solr-solrj-5.2.0.jar" />
<pathelement location="${lib}/spatial4j-0.4.1.jar" />
<pathelement location="${lib}/webcat-0.1-swf.jar" />
<pathelement location="${lib}/weupnp-0.1.2.jar" />

369
defaults/solr/schema.xml
Unescape View File

@ -185,7 +185,6 @@
<dynamicField name="*_td" type="tdouble" indexed="true" stored="true"/>
<dynamicField name="*_tdt" type="tdate" indexed="true" stored="true"/>
<dynamicField name="*_pi" type="pint" indexed="true" stored="true"/>
<dynamicField name="*_c" type="currency" indexed="true" stored="true"/>
<dynamicField name="ignored_*" type="ignored" multiValued="true"/>
@ -348,21 +347,6 @@
<!--Binary data type. The data should be sent/retrieved in as Base64 encoded Strings -->
<fieldtype name="binary" class="solr.BinaryField"/>
<!--
Note:
These should only be used for compatibility with existing indexes (created with lucene or older Solr versions).
Use Trie based fields instead. As of Solr 3.5 and 4.x, Trie based fields support sortMissingFirst/Last
Plain numeric field types that store and index the text
value verbatim (and hence don't correctly support range queries, since the
lexicographic ordering isn't equal to the numeric ordering)
-->
<fieldType name="pint" class="solr.IntField"/>
<fieldType name="plong" class="solr.LongField"/>
<fieldType name="pfloat" class="solr.FloatField"/>
<fieldType name="pdouble" class="solr.DoubleField"/>
<fieldType name="pdate" class="solr.DateField" sortMissingLast="true"/>
<!-- The "RandomSortField" is not used to store or search any
data. You can declare fields of this type it in your schema
to generate pseudo-random orderings of your docs for sorting
@ -411,7 +395,7 @@
<fieldType name="text_general" class="solr.TextField" positionIncrementGap="100">
<analyzer type="index">
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" />
<!-- in this example, we will only use synonyms at query time
<filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
-->
@ -419,7 +403,7 @@
</analyzer>
<analyzer type="query">
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" />
<filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
<filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
@ -437,13 +421,10 @@
<filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
-->
<!-- Case insensitive stop word removal.
add enablePositionIncrements=true in both the index and query
analyzers to leave a 'gap' for more accurate phrase queries.
-->
<filter class="solr.StopFilterFactory"
ignoreCase="true"
words="lang/stopwords_en.txt"
enablePositionIncrements="true"
/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.EnglishPossessiveFilterFactory"/>
@ -459,7 +440,6 @@
<filter class="solr.StopFilterFactory"
ignoreCase="true"
words="lang/stopwords_en.txt"
enablePositionIncrements="true"
/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.EnglishPossessiveFilterFactory"/>
@ -480,20 +460,17 @@
cases will work, for example query "wi fi" will match
document "WiFi" or "wi-fi".
-->
<fieldType name="text_en_splitting" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="true">
<fieldType name="text_en_splitting" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="true">
<analyzer type="index">
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
<!-- in this example, we will only use synonyms at query time
<filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
-->
<!-- Case insensitive stop word removal.
add enablePositionIncrements=true in both the index and query
analyzers to leave a 'gap' for more accurate phrase queries.
-->
<filter class="solr.StopFilterFactory"
ignoreCase="true"
words="lang/stopwords_en.txt"
enablePositionIncrements="true"
/>
<filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
<filter class="solr.LowerCaseFilterFactory"/>
@ -506,7 +483,6 @@
<filter class="solr.StopFilterFactory"
ignoreCase="true"
words="lang/stopwords_en.txt"
enablePositionIncrements="true"
/>
<filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
<filter class="solr.LowerCaseFilterFactory"/>
@ -537,7 +513,7 @@
<fieldType name="text_general_rev" class="solr.TextField" positionIncrementGap="100">
<analyzer type="index">
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" />
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.ReversedWildcardFilterFactory" withOriginal="true"
maxPosAsterisk="3" maxPosQuestion="2" maxFractionAsterisk="0.33"/>
@ -545,7 +521,7 @@
<analyzer type="query">
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" />
<filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
</fieldType>
@ -672,341 +648,6 @@
-->
<fieldType name="currency" class="solr.CurrencyField" precisionStep="8" defaultCurrency="USD" currencyConfig="currency.xml" />
<!-- some examples for different languages (generally ordered by ISO code) -->
<!-- Arabic -->
<fieldType name="text_ar" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<!-- for any non-arabic -->
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ar.txt" enablePositionIncrements="true"/>
<!-- normalizes ﻯ to ﻱ, etc -->
<filter class="solr.ArabicNormalizationFilterFactory"/>
<filter class="solr.ArabicStemFilterFactory"/>
</analyzer>
</fieldType>
<!-- Bulgarian -->
<fieldType name="text_bg" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_bg.txt" enablePositionIncrements="true"/>
<filter class="solr.BulgarianStemFilterFactory"/>
</analyzer>
</fieldType>
<!-- Catalan -->
<fieldType name="text_ca" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<!-- removes l', etc -->
<filter class="solr.ElisionFilterFactory" ignoreCase="true" articles="lang/contractions_ca.txt"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ca.txt" enablePositionIncrements="true"/>
<filter class="solr.SnowballPorterFilterFactory" language="Catalan"/>
</analyzer>
</fieldType>
<!-- CJK bigram (see text_ja for a Japanese configuration using morphological analysis) -->
<fieldType name="text_cjk" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<!-- normalize width before bigram, as e.g. half-width dakuten combine -->
<filter class="solr.CJKWidthFilterFactory"/>
<!-- for any non-CJK -->
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.CJKBigramFilterFactory"/>
</analyzer>
</fieldType>
<!-- Czech -->
<fieldType name="text_cz" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_cz.txt" enablePositionIncrements="true"/>
<filter class="solr.CzechStemFilterFactory"/>
</analyzer>
</fieldType>
<!-- Danish -->
<fieldType name="text_da" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_da.txt" format="snowball" enablePositionIncrements="true"/>
<filter class="solr.SnowballPorterFilterFactory" language="Danish"/>
</analyzer>
</fieldType>
<!-- German -->
<fieldType name="text_de" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_de.txt" format="snowball" enablePositionIncrements="true"/>
<filter class="solr.GermanNormalizationFilterFactory"/>
<filter class="solr.GermanLightStemFilterFactory"/>
<!-- less aggressive: <filter class="solr.GermanMinimalStemFilterFactory"/> -->
<!-- more aggressive: <filter class="solr.SnowballPorterFilterFactory" language="German2"/> -->
</analyzer>
</fieldType>
<!-- Greek -->
<fieldType name="text_el" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<!-- greek specific lowercase for sigma -->
<filter class="solr.GreekLowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="false" words="lang/stopwords_el.txt" enablePositionIncrements="true"/>
<filter class="solr.GreekStemFilterFactory"/>
</analyzer>
</fieldType>
<!-- Spanish -->
<fieldType name="text_es" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_es.txt" format="snowball" enablePositionIncrements="true"/>
<filter class="solr.SpanishLightStemFilterFactory"/>
<!-- more aggressive: <filter class="solr.SnowballPorterFilterFactory" language="Spanish"/> -->
</analyzer>
</fieldType>
<!-- Basque -->
<fieldType name="text_eu" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_eu.txt" enablePositionIncrements="true"/>
<filter class="solr.SnowballPorterFilterFactory" language="Basque"/>
</analyzer>
</fieldType>
<!-- Persian -->
<fieldType name="text_fa" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<!-- for ZWNJ -->
<charFilter class="solr.PersianCharFilterFactory"/>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.ArabicNormalizationFilterFactory"/>
<filter class="solr.PersianNormalizationFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_fa.txt" enablePositionIncrements="true"/>
</analyzer>
</fieldType>
<!-- Finnish -->
<fieldType name="text_fi" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_fi.txt" format="snowball" enablePositionIncrements="true"/>
<filter class="solr.SnowballPorterFilterFactory" language="Finnish"/>
<!-- less aggressive: <filter class="solr.FinnishLightStemFilterFactory"/> -->
</analyzer>
</fieldType>
<!-- French -->
<fieldType name="text_fr" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<!-- removes l', etc -->
<filter class="solr.ElisionFilterFactory" ignoreCase="true" articles="lang/contractions_fr.txt"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_fr.txt" format="snowball" enablePositionIncrements="true"/>
<filter class="solr.FrenchLightStemFilterFactory"/>
<!-- less aggressive: <filter class="solr.FrenchMinimalStemFilterFactory"/> -->
<!-- more aggressive: <filter class="solr.SnowballPorterFilterFactory" language="French"/> -->
</analyzer>
</fieldType>
<!-- Irish -->
<fieldType name="text_ga" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<!-- removes d', etc -->
<filter class="solr.ElisionFilterFactory" ignoreCase="true" articles="lang/contractions_ga.txt"/>
<!-- removes n-, etc. position increments is intentionally false! -->
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/hyphenations_ga.txt" enablePositionIncrements="false"/>
<filter class="solr.IrishLowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ga.txt" enablePositionIncrements="true"/>
<filter class="solr.SnowballPorterFilterFactory" language="Irish"/>
</analyzer>
</fieldType>
<!-- Galician -->
<fieldType name="text_gl" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_gl.txt" enablePositionIncrements="true"/>
<filter class="solr.GalicianStemFilterFactory"/>
<!-- less aggressive: <filter class="solr.GalicianMinimalStemFilterFactory"/> -->
</analyzer>
</fieldType>
<!-- Hindi -->
<fieldType name="text_hi" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<!-- normalizes unicode representation -->
<filter class="solr.IndicNormalizationFilterFactory"/>
<!-- normalizes variation in spelling -->
<filter class="solr.HindiNormalizationFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_hi.txt" enablePositionIncrements="true"/>
<filter class="solr.HindiStemFilterFactory"/>
</analyzer>
</fieldType>
<!-- Hungarian -->
<fieldType name="text_hu" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_hu.txt" format="snowball" enablePositionIncrements="true"/>
<filter class="solr.SnowballPorterFilterFactory" language="Hungarian"/>
<!-- less aggressive: <filter class="solr.HungarianLightStemFilterFactory"/> -->
</analyzer>
</fieldType>
<!-- Armenian -->
<fieldType name="text_hy" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_hy.txt" enablePositionIncrements="true"/>
<filter class="solr.SnowballPorterFilterFactory" language="Armenian"/>
</analyzer>
</fieldType>
<!-- Indonesian -->
<fieldType name="text_id" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_id.txt" enablePositionIncrements="true"/>
<!-- for a less aggressive approach (only inflectional suffixes), set stemDerivational to false -->
<filter class="solr.IndonesianStemFilterFactory" stemDerivational="true"/>
</analyzer>
</fieldType>
<!-- Italian -->
<fieldType name="text_it" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<!-- removes l', etc -->
<filter class="solr.ElisionFilterFactory" ignoreCase="true" articles="lang/contractions_it.txt"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_it.txt" format="snowball" enablePositionIncrements="true"/>
<filter class="solr.ItalianLightStemFilterFactory"/>
<!-- more aggressive: <filter class="solr.SnowballPorterFilterFactory" language="Italian"/> -->
</analyzer>
</fieldType>
<!-- Latvian -->
<fieldType name="text_lv" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_lv.txt" enablePositionIncrements="true"/>
<filter class="solr.LatvianStemFilterFactory"/>
</analyzer>
</fieldType>
<!-- Dutch -->
<fieldType name="text_nl" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_nl.txt" format="snowball" enablePositionIncrements="true"/>
<filter class="solr.StemmerOverrideFilterFactory" dictionary="lang/stemdict_nl.txt" ignoreCase="false"/>
<filter class="solr.SnowballPorterFilterFactory" language="Dutch"/>
</analyzer>
</fieldType>
<!-- Norwegian -->
<fieldType name="text_no" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_no.txt" format="snowball" enablePositionIncrements="true"/>
<filter class="solr.SnowballPorterFilterFactory" language="Norwegian"/>
<!-- less aggressive: <filter class="solr.NorwegianLightStemFilterFactory"/> -->
<!-- singular/plural: <filter class="solr.NorwegianMinimalStemFilterFactory"/> -->
</analyzer>
</fieldType>
<!-- Portuguese -->
<fieldType name="text_pt" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_pt.txt" format="snowball" enablePositionIncrements="true"/>
<filter class="solr.PortugueseLightStemFilterFactory"/>
<!-- less aggressive: <filter class="solr.PortugueseMinimalStemFilterFactory"/> -->
<!-- more aggressive: <filter class="solr.SnowballPorterFilterFactory" language="Portuguese"/> -->
<!-- most aggressive: <filter class="solr.PortugueseStemFilterFactory"/> -->
</analyzer>
</fieldType>
<!-- Romanian -->
<fieldType name="text_ro" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ro.txt" enablePositionIncrements="true"/>
<filter class="solr.SnowballPorterFilterFactory" language="Romanian"/>
</analyzer>
</fieldType>
<!-- Russian -->
<fieldType name="text_ru" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ru.txt" format="snowball" enablePositionIncrements="true"/>
<filter class="solr.SnowballPorterFilterFactory" language="Russian"/>
<!-- less aggressive: <filter class="solr.RussianLightStemFilterFactory"/> -->
</analyzer>
</fieldType>
<!-- Swedish -->
<fieldType name="text_sv" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_sv.txt" format="snowball" enablePositionIncrements="true"/>
<filter class="solr.SnowballPorterFilterFactory" language="Swedish"/>
<!-- less aggressive: <filter class="solr.SwedishLightStemFilterFactory"/> -->
</analyzer>
</fieldType>
<!-- Thai -->
<fieldType name="text_th" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.ThaiWordFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_th.txt" enablePositionIncrements="true"/>
</analyzer>
</fieldType>
<!-- Turkish -->
<fieldType name="text_tr" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.TurkishLowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="false" words="lang/stopwords_tr.txt" enablePositionIncrements="true"/>
<filter class="solr.SnowballPorterFilterFactory" language="Turkish"/>
</analyzer>
</fieldType>
</types>

63
defaults/solr/solr.xml
Unescape View File

@ -1,4 +1,5 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
<solr persistent="false">
<logging enabled="true">
<watcher size="100" threshold="INFO" />
@ -8,3 +9,65 @@
<core name="webgraph" instanceDir="webgraph" />
</cores>
</solr>
-->
<!--
<solr>
<solrcloud>
<str name="host">127.0.0.1</str>
<int name="hostPort">8983</int>
<str name="hostContext">solr</str>
<bool name="genericCoreNodeNames">true</bool>
<int name="zkClientTimeout">30000</int>
<int name="distribUpdateSoTimeout">600000</int>
<int name="distribUpdateConnTimeout">60000</int>
</solrcloud>
<shardHandlerFactory name="shardHandlerFactory"
class="HttpShardHandlerFactory">
<int name="socketTimeout">600000</int>
<int name="connTimeout">60000</int>
</shardHandlerFactory>
</solr>
-->
<!--old
<solr persistent="${solr.xml.persist:false}">
<cores adminPath="/admin/cores" defaultCoreName="collection1" host="127.0.0.1" hostPort="${hostPort:8983}"
hostContext="${hostContext:solr}" zkClientTimeout="${solr.zkclienttimeout:30000}" shareSchema="${shareSchema:false}"
genericCoreNodeNames="${genericCoreNodeNames:true}">
<core name="core1" instanceDir="core1" shard="${shard:}" collection="${collection:core1}" config="${solrconfig:solrconfig.xml}" schema="${schema:schema.xml}" coreNodeName="${coreNodeName:}"/>
<core name="core2" instanceDir="core2" />
<shardHandlerFactory name="shardHandlerFactory" class="HttpShardHandlerFactory">
<int name="socketTimeout">${socketTimeout:120000}</int>
<int name="connTimeout">${connTimeout:15000}</int>
</shardHandlerFactory>
</cores>
</solr>
-->
<!--new:-->
<solr>
<solrcloud>
<str name="host">127.0.0.1</str>
<int name="hostPort">${hostPort:8983}</int>
<str name="hostContext">${hostContext:solr}</str>
<int name="zkClientTimeout">${solr.zkclienttimeout:30000}</int>
<!--<str name="shareSchema">${shareSchema:false}</str>-->
<bool name="genericCoreNodeNames">${genericCoreNodeNames:true}</bool>
</solrcloud>
<shardHandlerFactory name="shardHandlerFactory" class="HttpShardHandlerFactory">
<int name="socketTimeout">${socketTimeout:120000}</int>
<int name="connTimeout">${connTimeout:15000}</int>
</shardHandlerFactory>
</solr>
<!--
In each of "core1" and "core2" directories, there would be a core.properties file that might look like these.
name=core1
shard=${shard:}
collection=${collection:core1}
config=${solrconfig:solrconfig.xml}
schema=${schema:schema.xml}
coreNodeName=${coreNodeName:}
-->

2
defaults/solr/solrconfig.xml
Unescape View File

@ -35,7 +35,7 @@
that you fully re-index after changing this setting as it can
affect both how text is indexed and queried.
-->
<luceneMatchVersion>4.10.3</luceneMatchVersion>
<luceneMatchVersion>5.2.0</luceneMatchVersion>
<!-- <lib/> directives can be used to instruct Solr to load any Jars
identified and use them to resolve any "plugins" specified in

3
htroot/IndexControlURLs_p.java
Unescape View File

@ -30,8 +30,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.lucene.search.FieldCache;
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.federate.yacy.CacheStrategy;
@ -214,7 +212,6 @@ public class IndexControlURLs_p {
if (post.containsKey("rebootsolr")) {
segment.fulltext().rebootSolr();
FieldCache.DEFAULT.purgeAllCaches();
sb.tables.recordAPICall(post, "IndexControlURLs_p.html", WorkTables.TABLE_API_TYPE_STEERING, "solr reboot");
}

0
lib/httpclient-4.4.License → lib/httpclient-4.4.1.License
Unescape View File

BIN
lib/httpclient-4.4.jar → lib/httpclient-4.4.1.jar
View File

Binary file not shown.

0
lib/httpcore-4.4.License → lib/httpcore-4.4.1.License
Unescape View File

0
lib/httpmime-4.4.License → lib/httpmime-4.4.1.License
Unescape View File

BIN
lib/httpmime-4.4.jar → lib/httpmime-4.4.1.jar
View File

Binary file not shown.

BIN
lib/lucene-analyzers-common-4.10.3.jar → lib/lucene-analyzers-common-5.2.0.jar
View File

Binary file not shown.

BIN
lib/lucene-analyzers-phonetic-4.10.3.jar → lib/lucene-analyzers-phonetic-5.2.0.jar
View File

Binary file not shown.

BIN
lib/lucene-backward-codecs-5.2.0.jar
View File

Binary file not shown.

BIN
lib/lucene-classification-4.10.3.jar
View File

Binary file not shown.

BIN
lib/lucene-classification-5.2.0.jar
View File

Binary file not shown.

BIN
lib/lucene-codecs-4.10.3.jar
View File

Binary file not shown.

BIN
lib/lucene-codecs-5.2.0.jar
View File

Binary file not shown.

BIN
lib/lucene-core-4.10.3.jar
View File

Binary file not shown.

BIN
lib/lucene-core-5.2.0.jar
View File

Binary file not shown.

BIN
lib/lucene-facet-4.10.3.jar
View File

Binary file not shown.

BIN
lib/lucene-facet-5.2.0.jar
View File

Binary file not shown.

BIN
lib/lucene-grouping-4.10.3.jar
View File

Binary file not shown.

BIN
lib/lucene-grouping-5.2.0.jar
View File

Binary file not shown.

BIN
lib/lucene-highlighter-4.10.3.jar
View File

Binary file not shown.

BIN
lib/lucene-highlighter-5.2.0.jar
View File

Binary file not shown.

BIN
lib/lucene-join-4.10.3.jar
View File

Binary file not shown.

BIN
lib/lucene-join-5.2.0.jar
View File

Binary file not shown.

BIN
lib/lucene-memory-4.10.3.jar
View File

Binary file not shown.

BIN
lib/lucene-memory-5.2.0.jar
View File

Binary file not shown.

BIN
lib/lucene-misc-4.10.3.jar
View File

Binary file not shown.

BIN
lib/lucene-misc-5.2.0.jar
View File

Binary file not shown.

BIN
lib/lucene-queries-4.10.3.jar
View File

Binary file not shown.

BIN
lib/lucene-queries-5.2.0.jar
View File

Binary file not shown.

BIN
lib/lucene-queryparser-4.10.3.jar → lib/lucene-queryparser-5.2.0.jar
View File

Binary file not shown.

BIN
lib/lucene-spatial-4.10.3.jar
View File

Binary file not shown.

BIN
lib/lucene-spatial-5.2.0.jar
View File

Binary file not shown.

BIN
lib/lucene-suggest-4.10.3.jar
View File

Binary file not shown.

BIN
lib/lucene-suggest-5.2.0.jar
View File

Binary file not shown.

BIN
lib/noggit-0.5.jar
View File

Binary file not shown.

BIN
lib/noggit-0.6.jar
View File

Binary file not shown.

BIN
lib/solr-core-4.10.3.jar
View File

Binary file not shown.

BIN
lib/solr-core-5.2.0.jar
View File

Binary file not shown.

BIN
lib/solr-solrj-4.10.3.jar
View File

Binary file not shown.

BIN
lib/solr-solrj-5.2.0.jar
View File

Binary file not shown.

BIN
lib/stax2-api-3.1.4.jar
View File

Binary file not shown.

BIN
lib/woodstox-core-asl-4.4.1.jar
View File

Binary file not shown.

5
source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java
Unescape View File

@ -35,7 +35,6 @@ import net.yacy.search.schema.CollectionSchema;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.client.solrj.SolrQuery;
@ -56,7 +55,6 @@ import org.apache.solr.handler.component.SearchHandler;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryRequestBase;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.request.UnInvertedField;
import org.apache.solr.response.ResultContext;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.FieldType;
@ -129,7 +127,7 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
public void clearCaches() {
SolrConfig solrConfig = this.core.getSolrConfig();
@SuppressWarnings("unchecked")
SolrCache<String, UnInvertedField> fieldValueCache = solrConfig.fieldValueCacheConfig == null ? null : solrConfig.fieldValueCacheConfig.newInstance();
SolrCache<String, ?> fieldValueCache = solrConfig.fieldValueCacheConfig == null ? null : solrConfig.fieldValueCacheConfig.newInstance();
if (fieldValueCache != null) fieldValueCache.clear();
@SuppressWarnings("unchecked")
SolrCache<Query, DocSet> filterCache= solrConfig.filterCacheConfig == null ? null : solrConfig.filterCacheConfig.newInstance();
@ -144,7 +142,6 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
// clear 'lost' caches
if (ib instanceof SolrCache) ((SolrCache<?,?>) ib).clear();
}
FieldCache.DEFAULT.purgeAllCaches(); // purge the lucene field cache
}
public SolrInstance getInstance() {

6
source/net/yacy/cora/federate/solr/connector/RemoteSolrConnector.java
Unescape View File

@ -29,7 +29,7 @@ import net.yacy.cora.federate.solr.instance.RemoteInstance;
import net.yacy.cora.federate.solr.instance.ShardInstance;
import org.apache.solr.client.solrj.ResponseParser;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.BinaryResponseParser;
import org.apache.solr.client.solrj.impl.XMLResponseParser;
import org.apache.solr.client.solrj.request.QueryRequest;
@ -54,7 +54,7 @@ public class RemoteSolrConnector extends SolrServerConnector implements SolrConn
this.instance = instance;
this.useBinaryResponseWriter = useBinaryResponseWriter;
this.corename = this.instance.getDefaultCoreName();
SolrServer s = instance.getServer(this.corename);
SolrClient s = instance.getServer(this.corename);
super.init(s);
}
@ -63,7 +63,7 @@ public class RemoteSolrConnector extends SolrServerConnector implements SolrConn
this.instance = instance;
this.useBinaryResponseWriter = useBinaryResponseWriter;
this.corename = corename == null ? this.instance.getDefaultCoreName() : corename;
SolrServer s = instance.getServer(this.corename);
SolrClient s = instance.getServer(this.corename);
super.init(s);
}

20
source/net/yacy/cora/federate/solr/connector/ShardSelection.java
Unescape View File

@ -33,16 +33,16 @@ import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.search.schema.CollectionSchema;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
public class ShardSelection implements Iterable<SolrServer> {
public class ShardSelection implements Iterable<SolrClient> {
private final Method method; // the sharding method
private final AtomicLong shardID; // the next id that shall be given away
private final int dimension; // the number of chards
private final ArrayList<SolrServer> server;
private final ArrayList<SolrClient> server;
public enum Method {
MODULO_HOST_MD5("hash-based calculation of storage targets, select all for retrieval"),
@ -54,7 +54,7 @@ public class ShardSelection implements Iterable<SolrServer> {
}
}
public ShardSelection(final ArrayList<SolrServer> server, final Method method) {
public ShardSelection(final ArrayList<SolrClient> server, final Method method) {
this.server = server;
this.method = method;
this.dimension = server.size();
@ -71,7 +71,7 @@ public class ShardSelection implements Iterable<SolrServer> {
return rr;
}
public SolrServer server4write(final SolrInputDocument solrdoc) throws IOException {
public SolrClient server4write(final SolrInputDocument solrdoc) throws IOException {
if (this.method == Method.MODULO_HOST_MD5) {
SolrInputField sif = solrdoc.getField(CollectionSchema.host_s.getSolrFieldName());
if (sif != null) {
@ -95,7 +95,7 @@ public class ShardSelection implements Iterable<SolrServer> {
return this.server.get(selectRoundRobin());
}
public SolrServer server4write(final String host) throws IOException {
public SolrClient server4write(final String host) throws IOException {
if (host == null) throw new IOException("sharding - host url, host empty: " + host);
if (host.indexOf("://") >= 0) return server4write(new URL(host)); // security catch for accidantly using the wrong method
if (this.method == Method.MODULO_HOST_MD5) {
@ -113,14 +113,14 @@ public class ShardSelection implements Iterable<SolrServer> {
return this.server.get(selectRoundRobin());
}
public SolrServer server4write(final URL url) throws IOException {
public SolrClient server4write(final URL url) throws IOException {
return server4write(url.getHost());
}
public List<SolrServer> server4read() {
public List<SolrClient> server4read() {
if (this.method == Method.MODULO_HOST_MD5 || this.method == Method.ROUND_ROBIN) return this.server; // return all
// this is a SolrCloud, we select just one of the SolrCloud server(s)
ArrayList<SolrServer> a = new ArrayList<>(1);
ArrayList<SolrClient> a = new ArrayList<>(1);
a.add(this.server.get(selectRoundRobin()));
return a;
}
@ -129,7 +129,7 @@ public class ShardSelection implements Iterable<SolrServer> {
* return all solr server
*/
@Override
public Iterator<SolrServer> iterator() {
public Iterator<SolrClient> iterator() {
return this.server.iterator();
}
}

10
source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java
Unescape View File

@ -38,7 +38,7 @@ import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
import org.apache.solr.client.solrj.impl.XMLResponseParser;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
import org.apache.solr.client.solrj.request.LukeRequest;
@ -57,17 +57,17 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
static {
assert classLoaderSynchro != null;
}
protected SolrServer server;
protected SolrClient server;
protected SolrServerConnector() {
this.server = null;
}
protected void init(SolrServer server) {
protected void init(SolrClient server) {
this.server = server;
}
public SolrServer getServer() {
public SolrClient getServer() {
return this.server;
}
@ -117,7 +117,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
}
}
synchronized (this.server) {
this.server.shutdown(); // if the server is embedded, resources are freed, if it is a HttpSolrServer, only the httpclient is shut down, not the remote server
this.server.close(); // if the server is embedded, resources are freed, if it is a HttpSolrServer, only the httpclient is shut down, not the remote server
}
this.server = null;
} catch (final Throwable e) {

42
source/net/yacy/cora/federate/solr/instance/EmbeddedInstance.java
Unescape View File

@ -21,15 +21,18 @@
package net.yacy.cora.federate.solr.instance;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Collection;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.kelondro.util.MemoryControl;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
@ -43,10 +46,10 @@ public class EmbeddedInstance implements SolrInstance {
private CoreContainer coreContainer;
private String defaultCoreName;
private SolrCore defaultCore;
private SolrServer defaultCoreServer;
private SolrClient defaultCoreServer;
private File containerPath;
private Map<String, SolrCore> cores;
private Map<String, SolrServer> server;
private Map<String, SolrClient> server;
public EmbeddedInstance(final File solr_config, final File containerPath, String givenDefaultCoreName, String[] initializeCoreNames) throws IOException {
super();
@ -77,14 +80,13 @@ public class EmbeddedInstance implements SolrInstance {
}
// initialize the coreContainer
String containerDir = this.containerPath.getAbsolutePath(); // the home directory of all resources.
String containerDir = this.containerPath.getAbsolutePath(); // the home directory of all cores
File configFile = new File(solr_config, "solr.xml"); // the configuration file for all cores
this.coreContainer = CoreContainer.createAndLoad(containerDir, configFile); // this may take indefinitely long if solr files are broken
if (this.coreContainer == null) throw new IOException("cannot create core container dir = " + containerDir + ", configFile = " + configFile);
// get the default core from the coreContainer
this.defaultCoreName = this.coreContainer.getDefaultCoreName();
assert(this.defaultCoreName.equals(givenDefaultCoreName));
this.defaultCoreName = givenDefaultCoreName;
ConcurrentLog.info("SolrEmbeddedInstance", "detected default solr core: " + this.defaultCoreName);
this.defaultCore = this.coreContainer.getCore(this.defaultCoreName);
assert givenDefaultCoreName.equals(this.defaultCore.getName()) : "givenDefaultCoreName = " + givenDefaultCoreName + ", this.defaultCore.getName() = " + this.defaultCore.getName();
@ -96,7 +98,7 @@ public class EmbeddedInstance implements SolrInstance {
// initialize core cache
this.cores = new ConcurrentHashMap<String, SolrCore>();
this.cores.put(this.defaultCoreName, this.defaultCore);
this.server = new ConcurrentHashMap<String, SolrServer>();
this.server = new ConcurrentHashMap<String, SolrClient>();
this.server.put(this.defaultCoreName, this.defaultCoreServer);
}
@ -110,12 +112,30 @@ public class EmbeddedInstance implements SolrInstance {
return o instanceof EmbeddedInstance && this.containerPath.equals(((EmbeddedInstance) o).containerPath);
}
private void initializeCoreConf(final File solr_config, final File containerPath, String coreName) {
private static void initializeCoreConf(final File solr_config, final File containerPath, String coreName) {
// ensure that default core path exists
File corePath = new File(containerPath, coreName);
if (!corePath.exists()) corePath.mkdirs();
// check if core.properties exists in each path (thats new in Solr 5.0)
File core_properties = new File(corePath, "core.properties");
if (!core_properties.exists()) {
// create the file
try {
FileOutputStream fos = new FileOutputStream(core_properties);
fos.write(ASCII.getBytes("name=" + coreName + "\n"));
fos.write(ASCII.getBytes("shard=${shard:}\n"));
fos.write(ASCII.getBytes("collection=${collection:" + coreName + "}\n"));
fos.write(ASCII.getBytes("config=${solrconfig:solrconfig.xml}\n"));
fos.write(ASCII.getBytes("schema=${schema:schema.xml}\n"));
fos.write(ASCII.getBytes("coreNodeName=${coreNodeName:}\n"));
fos.close();
} catch (IOException e) {
ConcurrentLog.logException(e);
}
}
// ensure necessary subpaths exist
File conf = new File(corePath, "conf");
conf.mkdirs();
@ -191,13 +211,13 @@ public class EmbeddedInstance implements SolrInstance {
}
@Override
public SolrServer getDefaultServer() {
public SolrClient getDefaultServer() {
return this.defaultCoreServer;
}
@Override
public SolrServer getServer(String coreName) {
SolrServer s = this.server.get(coreName);
public SolrClient getServer(String coreName) {
SolrClient s = this.server.get(coreName);
if (s != null) return s;
s = new EmbeddedSolrServer(this.coreContainer, coreName);
this.server.put(coreName, s);

22
source/net/yacy/cora/federate/solr/instance/RemoteInstance.java
Unescape View File

@ -50,8 +50,8 @@ import org.apache.http.client.AuthCache;
import org.apache.http.client.entity.GzipDecompressingEntity;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.protocol.HttpContext;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrServer;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient;
@SuppressWarnings("deprecation")
public class RemoteInstance implements SolrInstance {
@ -59,9 +59,9 @@ public class RemoteInstance implements SolrInstance {
private String solrurl;
private final Object client; // not declared as org.apache.http.impl.client.DefaultHttpClient to avoid warnings during compilation. TODO: switch to org.apache.http.impl.client.HttpClientBuilder
private final String defaultCoreName;
private final ConcurrentUpdateSolrServer defaultServer;
private final ConcurrentUpdateSolrClient defaultServer;
private final Collection<String> coreNames;
private final Map<String, ConcurrentUpdateSolrServer> server;
private final Map<String, ConcurrentUpdateSolrClient> server;
private final int timeout;
public static ArrayList<RemoteInstance> getShardInstances(final String urlList, Collection<String> coreNames, String defaultCoreName, final int timeout) throws IOException {
@ -77,7 +77,7 @@ public class RemoteInstance implements SolrInstance {
public RemoteInstance(final String url, final Collection<String> coreNames, final String defaultCoreName, final int timeout) throws IOException {
this.timeout = timeout;
this.server= new HashMap<String, ConcurrentUpdateSolrServer>();
this.server= new HashMap<String, ConcurrentUpdateSolrClient>();
this.solrurl = url == null ? "http://127.0.0.1:8983/solr/" : url; // that should work for the example configuration of solr 4.x.x
this.coreNames = coreNames == null ? new ArrayList<String>() : coreNames;
if (this.coreNames.size() == 0) {
@ -180,7 +180,7 @@ public class RemoteInstance implements SolrInstance {
this.client = null;
}
this.defaultServer = (ConcurrentUpdateSolrServer) getServer(this.defaultCoreName);
this.defaultServer = (ConcurrentUpdateSolrClient) getServer(this.defaultCoreName);
if (this.defaultServer == null) throw new IOException("cannot connect to url " + url + " and connect core " + defaultCoreName);
}
@ -216,14 +216,14 @@ public class RemoteInstance implements SolrInstance {
}
@Override
public SolrServer getDefaultServer() {
public SolrClient getDefaultServer() {
return this.defaultServer;
}
@Override
public SolrServer getServer(String name) {
public SolrClient getServer(String name) {
// try to get the server from the cache
ConcurrentUpdateSolrServer s = this.server.get(name);
ConcurrentUpdateSolrClient s = this.server.get(name);
if (s != null) return s;
// create new http server
if (this.client != null) {
@ -238,10 +238,10 @@ public class RemoteInstance implements SolrInstance {
String solrpath = u.getPath();
String p = "http://" + host + ":" + port + solrpath;
ConcurrentLog.info("RemoteSolrConnector", "connecting Solr authenticated with url:" + p);
s = new ConcurrentUpdateSolrServer(p, ((org.apache.http.impl.client.DefaultHttpClient) this.client), 10, Runtime.getRuntime().availableProcessors());
s = new ConcurrentUpdateSolrClient(p, ((org.apache.http.impl.client.DefaultHttpClient) this.client), 10, Runtime.getRuntime().availableProcessors());
} else {
ConcurrentLog.info("RemoteSolrConnector", "connecting Solr with url:" + this.solrurl + name);
s = new ConcurrentUpdateSolrServer(this.solrurl + name, queueSizeByMemory(), Runtime.getRuntime().availableProcessors());
s = new ConcurrentUpdateSolrClient(this.solrurl + name, queueSizeByMemory(), Runtime.getRuntime().availableProcessors());
}
//s.setAllowCompression(true);
s.setSoTimeout(this.timeout);

38
source/net/yacy/cora/federate/solr/instance/ServerMirror.java
Unescape View File

@ -25,7 +25,7 @@ import java.util.Collection;
import java.util.List;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.StreamingResponseCallback;
import org.apache.solr.client.solrj.SolrRequest.METHOD;
@ -37,21 +37,21 @@ import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
public class ServerMirror extends SolrServer {
public class ServerMirror extends SolrClient {
private static final long serialVersionUID = 4665364470785220322L;
private SolrServer solr0, solr1;
private SolrClient solr0, solr1;
public ServerMirror() {
solr0 = null;
solr1 = null;
}
public void connect0(SolrServer solr0) {
public void connect0(SolrClient solr0) {
this.solr0 = solr0;
}
public void connect1(SolrServer solr1) {
public void connect1(SolrClient solr1) {
this.solr1 = solr1;
}
@ -373,9 +373,10 @@ public class ServerMirror extends SolrServer {
/**
* Performs a query to the Solr server
* @param params an object holding all key/value parameters to send along the request
* @throws IOException
*/
@Override
public QueryResponse query(SolrParams params) throws SolrServerException {
public QueryResponse query(SolrParams params) throws SolrServerException, IOException {
if (this.solr0 != null) return this.solr0.query(params);
if (this.solr1 != null) return this.solr1.query(params);
return null;
@ -385,9 +386,10 @@ public class ServerMirror extends SolrServer {
* Performs a query to the Solr server
* @param params an object holding all key/value parameters to send along the request
* @param method specifies the HTTP method to use for the request, such as GET or POST
* @throws IOException
*/
@Override
public QueryResponse query(SolrParams params, METHOD method) throws SolrServerException {
public QueryResponse query(SolrParams params, METHOD method) throws SolrServerException, IOException {
if (this.solr0 != null) return this.solr0.query(params, method);
if (this.solr1 != null) return this.solr1.query(params, method);
return null;
@ -413,11 +415,11 @@ public class ServerMirror extends SolrServer {
/**
* SolrServer implementations need to implement how a request is actually processed
*/
*/
@Override
public NamedList<Object> request( final SolrRequest request ) throws SolrServerException, IOException {
if (this.solr0 != null) return this.solr0.request(request);
if (this.solr1 != null) return this.solr1.request(request);
public NamedList<Object> request(@SuppressWarnings("rawtypes") SolrRequest request, String collection) throws SolrServerException, IOException {
if (this.solr0 != null) return this.solr0.request(request, collection);
if (this.solr1 != null) return this.solr1.request(request, collection);
return null;
}
@ -430,8 +432,18 @@ public class ServerMirror extends SolrServer {
@Override
public void shutdown() {
if (this.solr0 != null) this.solr0.shutdown();
if (this.solr1 != null) this.solr1.shutdown();
if (this.solr0 != null)
try {
this.solr0.close();
} catch (IOException e) {
e.printStackTrace();
}
if (this.solr1 != null)
try {
this.solr1.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}

72
source/net/yacy/cora/federate/solr/instance/ServerShard.java
Unescape View File

@ -28,7 +28,7 @@ import java.util.List;
import java.util.concurrent.ConcurrentLinkedQueue;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.StreamingResponseCallback;
import org.apache.solr.client.solrj.SolrRequest.METHOD;
@ -42,7 +42,7 @@ import org.apache.solr.common.util.NamedList;
import net.yacy.cora.federate.solr.connector.ShardSelection;
public class ServerShard extends SolrServer {
public class ServerShard extends SolrClient {
private static final long serialVersionUID = -3524175189049786312L;
private static final UpdateResponse _dummyOKResponse = new UpdateResponse();
@ -54,7 +54,7 @@ public class ServerShard extends SolrServer {
private final ShardSelection shards;
private final boolean writeEnabled;
public ServerShard(ArrayList<SolrServer> server, final ShardSelection.Method method, final boolean writeEnabled) {
public ServerShard(ArrayList<SolrClient> server, final ShardSelection.Method method, final boolean writeEnabled) {
this.shards = new ShardSelection(server, method);
this.writeEnabled = writeEnabled;
}
@ -96,7 +96,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse addBeans(Collection<?> beans ) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrServer s: this.shards) ur = s.addBeans(beans);
for (SolrClient s: this.shards) ur = s.addBeans(beans);
return ur;
}
@ -111,7 +111,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse addBeans(Collection<?> beans, int commitWithinMs) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrServer s: this.shards) ur = s.addBeans(beans, commitWithinMs);
for (SolrClient s: this.shards) ur = s.addBeans(beans, commitWithinMs);
return ur;
}
@ -148,7 +148,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse addBean(Object obj) throws IOException, SolrServerException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrServer s: this.shards) ur = s.addBean(obj);
for (SolrClient s: this.shards) ur = s.addBean(obj);
return ur;
}
@ -163,7 +163,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse addBean(Object obj, int commitWithinMs) throws IOException, SolrServerException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrServer s: this.shards) ur = s.addBean(obj, commitWithinMs);
for (SolrClient s: this.shards) ur = s.addBean(obj, commitWithinMs);
return ur;
}
@ -177,7 +177,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse commit() throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrServer s: this.shards) ur = s.commit();
for (SolrClient s: this.shards) ur = s.commit();
return ur;
}
@ -193,7 +193,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse optimize() throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrServer s: this.shards) ur = s.optimize();
for (SolrClient s: this.shards) ur = s.optimize();
return ur;
}
@ -207,7 +207,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse commit(boolean waitFlush, boolean waitSearcher) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrServer s: this.shards) ur = s.commit(waitFlush, waitSearcher);
for (SolrClient s: this.shards) ur = s.commit(waitFlush, waitSearcher);
return ur;
}
@ -222,7 +222,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse commit(boolean waitFlush, boolean waitSearcher, boolean softCommit) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrServer s: this.shards) ur = s.commit(waitFlush, waitSearcher, softCommit);
for (SolrClient s: this.shards) ur = s.commit(waitFlush, waitSearcher, softCommit);
return ur;
}
@ -238,7 +238,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse optimize(boolean waitFlush, boolean waitSearcher) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrServer s: this.shards) ur = s.optimize(waitFlush, waitSearcher);
for (SolrClient s: this.shards) ur = s.optimize(waitFlush, waitSearcher);
return ur;
}
@ -255,7 +255,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse optimize(boolean waitFlush, boolean waitSearcher, int maxSegments) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrServer s: this.shards) ur = s.optimize(waitFlush, waitSearcher, maxSegments);
for (SolrClient s: this.shards) ur = s.optimize(waitFlush, waitSearcher, maxSegments);
return ur;
}
@ -271,7 +271,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse rollback() throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrServer s: this.shards) ur = s.rollback();
for (SolrClient s: this.shards) ur = s.rollback();
return ur;
}
@ -284,7 +284,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse deleteById(String id) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrServer s: this.shards.server4read()) ur = s.deleteById(id);
for (SolrClient s: this.shards.server4read()) ur = s.deleteById(id);
return ur;
}
@ -299,7 +299,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse deleteById(String id, int commitWithinMs) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrServer s: this.shards.server4read()) ur = s.deleteById(id, commitWithinMs);
for (SolrClient s: this.shards.server4read()) ur = s.deleteById(id, commitWithinMs);
return ur;
}
@ -312,7 +312,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse deleteById(List<String> ids) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrServer s: this.shards.server4read()) ur = s.deleteById(ids);
for (SolrClient s: this.shards.server4read()) ur = s.deleteById(ids);
return ur;
}
@ -327,7 +327,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse deleteById(List<String> ids, int commitWithinMs) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrServer s: this.shards.server4read()) ur = s.deleteById(ids, commitWithinMs);
for (SolrClient s: this.shards.server4read()) ur = s.deleteById(ids, commitWithinMs);
return ur;
}
@ -340,7 +340,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse deleteByQuery(String query) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrServer s: this.shards.server4read()) ur = s.deleteByQuery(query);
for (SolrClient s: this.shards.server4read()) ur = s.deleteByQuery(query);
return ur;
}
@ -355,7 +355,7 @@ public class ServerShard extends SolrServer {
public UpdateResponse deleteByQuery(String query, int commitWithinMs) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrServer s: this.shards.server4read()) ur = s.deleteByQuery(query, commitWithinMs);
for (SolrClient s: this.shards.server4read()) ur = s.deleteByQuery(query, commitWithinMs);
return ur;
}
@ -365,7 +365,7 @@ public class ServerShard extends SolrServer {
*/
@Override
public SolrPingResponse ping() throws SolrServerException, IOException {
for (SolrServer s: this.shards) {
for (SolrClient s: this.shards) {
SolrPingResponse spr = s.ping();
if (spr != null) return spr;
}
@ -375,10 +375,11 @@ public class ServerShard extends SolrServer {
/**
* Performs a query to the Solr server
* @param params an object holding all key/value parameters to send along the request
* @throws IOException
*/
@Override
public QueryResponse query(final SolrParams params) throws SolrServerException {
List<SolrServer> qs = this.shards.server4read();
public QueryResponse query(final SolrParams params) throws SolrServerException, IOException {
List<SolrClient> qs = this.shards.server4read();
if (qs.size() == 1) {
return qs.get(0).query(params);
}
@ -386,7 +387,7 @@ public class ServerShard extends SolrServer {
// concurrently call all shards
final Collection<QueryResponse> qrl = new ConcurrentLinkedQueue<QueryResponse>();
List<Thread> t = new ArrayList<Thread>();
for (final SolrServer s: qs) {
for (final SolrClient s: qs) {
Thread t0 = new Thread() {
@Override
public void run() {
@ -413,10 +414,11 @@ public class ServerShard extends SolrServer {
* Performs a query to the Solr server
* @param params an object holding all key/value parameters to send along the request
* @param method specifies the HTTP method to use for the request, such as GET or POST
* @throws IOException
*/
@Override
public QueryResponse query(final SolrParams params, final METHOD method) throws SolrServerException {
List<SolrServer> qs = this.shards.server4read();
public QueryResponse query(final SolrParams params, final METHOD method) throws SolrServerException, IOException {
List<SolrClient> qs = this.shards.server4read();
if (qs.size() == 1) {
return qs.get(0).query(params, method);
}
@ -424,7 +426,7 @@ public class ServerShard extends SolrServer {
final Collection<QueryResponse> qrl = new ConcurrentLinkedQueue<QueryResponse>();
// concurrently call all shards
List<Thread> t = new ArrayList<Thread>();
for (final SolrServer s: qs) {
for (final SolrClient s: qs) {
Thread t0 = new Thread() {
@Override
public void run() {
@ -468,16 +470,16 @@ public class ServerShard extends SolrServer {
* SolrServer implementations need to implement how a request is actually processed
*/
@Override
public NamedList<Object> request(final SolrRequest request) throws SolrServerException, IOException {
public NamedList<Object> request(@SuppressWarnings("rawtypes") SolrRequest request, String collection) throws SolrServerException, IOException {
ResponseAccumulator acc = new ResponseAccumulator();
for (SolrServer s: this.shards.server4read()) acc.addResponse(s.request(request));
return acc.getAccumulatedResponse();
for (SolrClient s: this.shards.server4read()) acc.addResponse(s.request(request, collection));
return acc.getAccumulatedResponse();
}
@Override
public DocumentObjectBinder getBinder() {
DocumentObjectBinder db;
for (SolrServer s: this.shards) {
for (SolrClient s: this.shards) {
db = s.getBinder();
if (db != null) return db;
}
@ -486,8 +488,12 @@ public class ServerShard extends SolrServer {
@Override
public void shutdown() {
for (SolrServer s: this.shards) {
s.shutdown();
for (SolrClient s: this.shards) {
try {
s.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}

18
source/net/yacy/cora/federate/solr/instance/ShardInstance.java
Unescape View File

@ -28,14 +28,14 @@ import java.util.concurrent.ConcurrentHashMap;
import net.yacy.cora.federate.solr.connector.ShardSelection;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrClient;
public class ShardInstance implements SolrInstance {
private final ArrayList<RemoteInstance> instances;
private final ShardSelection.Method method;
private SolrServer defaultServer;
private Map<String, SolrServer> serverCache;
private SolrClient defaultServer;
private Map<String, SolrClient> serverCache;
private final boolean writeEnabled;
public ShardInstance(final ArrayList<RemoteInstance> instances, final ShardSelection.Method method, final boolean writeEnabled) {
@ -43,7 +43,7 @@ public class ShardInstance implements SolrInstance {
this.method = method;
this.writeEnabled = writeEnabled;
this.defaultServer = null;
this.serverCache = new ConcurrentHashMap<String, SolrServer>();
this.serverCache = new ConcurrentHashMap<String, SolrClient>();
}
@Override
@ -57,19 +57,19 @@ public class ShardInstance implements SolrInstance {
}
@Override
public SolrServer getDefaultServer() {
public SolrClient getDefaultServer() {
if (this.defaultServer != null) return this.defaultServer;
ArrayList<SolrServer> server = new ArrayList<SolrServer>(instances.size());
ArrayList<SolrClient> server = new ArrayList<SolrClient>(instances.size());
for (int i = 0; i < instances.size(); i++) server.set(i, instances.get(i).getDefaultServer());
this.defaultServer = new ServerShard(server, method, this.writeEnabled);
return this.defaultServer;
}
@Override
public SolrServer getServer(String name) {
SolrServer s = serverCache.get(name);
public SolrClient getServer(String name) {
SolrClient s = serverCache.get(name);
if (s != null) return s;
ArrayList<SolrServer> server = new ArrayList<SolrServer>(instances.size());
ArrayList<SolrClient> server = new ArrayList<SolrClient>(instances.size());
for (int i = 0; i < instances.size(); i++) server.add(i, instances.get(i).getServer(name));
s = new ServerShard(server, method, this.writeEnabled);
this.serverCache.put(name, s);

6
source/net/yacy/cora/federate/solr/instance/SolrInstance.java
Unescape View File

@ -22,7 +22,7 @@ package net.yacy.cora.federate.solr.instance;
import java.util.Collection;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrClient;
public interface SolrInstance {
@ -30,9 +30,9 @@ public interface SolrInstance {
public Collection<String> getCoreNames();
public SolrServer getDefaultServer();
public SolrClient getDefaultServer();
public SolrServer getServer(String name);
public SolrClient getServer(String name);
public void close();
}

4
source/net/yacy/search/Switchboard.java
Unescape View File

@ -82,7 +82,6 @@ import java.util.zip.GZIPOutputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import org.apache.lucene.search.FieldCache;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument;
@ -2067,8 +2066,7 @@ public final class Switchboard extends serverSwitch {
// clear caches
if (WordCache.sizeCommonWords() > 1000) WordCache.clearCommonWords();
Word.clearCache();
// Domains.clear();
FieldCache.DEFAULT.purgeAllCaches();
// Domains.clear();
// clean up image stack
ResultImages.clearQueues();

4
source/net/yacy/search/index/Fulltext.java
Unescape View File

@ -84,8 +84,8 @@ import org.apache.lucene.util.Version;
public final class Fulltext {
private static final String SOLR_PATH = "solr_4_10"; // the number should be identical to the number in the property luceneMatchVersion in solrconfig.xml
private static final String SOLR_OLD_PATH[] = new String[]{"solr_36", "solr_40", "solr_44", "solr_45", "solr_46", "solr_47", "solr_4_9"};
private static final String SOLR_PATH = "solr_5_2"; // the number should be identical to the number in the property luceneMatchVersion in solrconfig.xml
private static final String SOLR_OLD_PATH[] = new String[]{"solr_36", "solr_40", "solr_44", "solr_45", "solr_46", "solr_47", "solr_4_9", "solr_4_10"};
// class objects
private final File segmentPath;

Write Preview
Loading…
Cancel
Save