boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of git@github.com:yacy/yacy_search_server.git

Browse Source
pull/9/merge
Michael Peter Christen 10 years ago
parent b94bd7f20a 5ba9924289
commit 11a848da5a
61 changed files with 289 additions and 327 deletions
Show Stats Download Patch File Download Diff File

194
.classpath
Unescape View File

@ -1,97 +1,97 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<classpath> <classpath>
<classpathentry kind="src" path="source"/> <classpathentry kind="src" path="source"/>
<classpathentry excluding="api/|env/|processing/domaingraph/applet/|yacy/|api/bookmarks/|api/ymarks/|api/bookmarks/posts/|api/bookmarks/tags/|api/bookmarks/xbel/|solr/|gsa/|solr/collection1/|api/blacklists/" kind="src" path="htroot"/> <classpathentry excluding="api/|env/|processing/domaingraph/applet/|yacy/|api/bookmarks/|api/ymarks/|api/bookmarks/posts/|api/bookmarks/tags/|api/bookmarks/xbel/|solr/|gsa/|solr/collection1/|api/blacklists/" kind="src" path="htroot"/>
<classpathentry excluding="bookmarks/|ymarks/|bookmarks/posts/|bookmarks/tags/|bookmarks/xbel/|blacklists/" kind="src" path="htroot/api"/> <classpathentry excluding="bookmarks/|ymarks/|bookmarks/posts/|bookmarks/tags/|bookmarks/xbel/|blacklists/" kind="src" path="htroot/api"/>
<classpathentry kind="src" path="htroot/env"/> <classpathentry kind="src" path="htroot/env"/>
<classpathentry kind="src" path="htroot/yacy"/> <classpathentry kind="src" path="htroot/yacy"/>
<classpathentry excluding="posts/|tags/|xbel/" kind="src" path="htroot/api/bookmarks"/> <classpathentry excluding="posts/|tags/|xbel/" kind="src" path="htroot/api/bookmarks"/>
<classpathentry kind="src" path="htroot/api/ymarks"/> <classpathentry kind="src" path="htroot/api/ymarks"/>
<classpathentry kind="src" path="htroot/api/bookmarks/posts"/> <classpathentry kind="src" path="htroot/api/bookmarks/posts"/>
<classpathentry kind="src" path="htroot/api/bookmarks/tags"/> <classpathentry kind="src" path="htroot/api/bookmarks/tags"/>
<classpathentry kind="src" path="htroot/api/bookmarks/xbel"/> <classpathentry kind="src" path="htroot/api/bookmarks/xbel"/>
<classpathentry kind="src" path="htroot/api/blacklists"/> <classpathentry kind="src" path="htroot/api/blacklists"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/> <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="lib" path="lib/commons-logging-1.2.jar"/> <classpathentry kind="lib" path="lib/commons-logging-1.2.jar"/>
<classpathentry kind="lib" path="lib/J7Zip-modified.jar"/> <classpathentry kind="lib" path="lib/J7Zip-modified.jar"/>
<classpathentry kind="lib" path="lib/webcat-0.1-swf.jar"/> <classpathentry kind="lib" path="lib/webcat-0.1-swf.jar"/>
<classpathentry kind="lib" path="lib/commons-jxpath-1.3.jar"/> <classpathentry kind="lib" path="lib/commons-jxpath-1.3.jar"/>
<classpathentry kind="lib" path="lib/jsch-0.1.52.jar"/> <classpathentry kind="lib" path="lib/jsch-0.1.53.jar"/>
<classpathentry kind="lib" path="lib/jakarta-oro-2.0.8.jar"/> <classpathentry kind="lib" path="lib/jakarta-oro-2.0.8.jar"/>
<classpathentry kind="lib" path="lib/apache-mime4j-0.6.jar"/> <classpathentry kind="lib" path="lib/apache-mime4j-0.6.jar"/>
<classpathentry kind="lib" path="lib/commons-fileupload-1.3.1.jar"/> <classpathentry kind="lib" path="lib/commons-fileupload-1.3.1.jar"/>
<classpathentry kind="lib" path="lib/json-simple-1.1.1.jar"/> <classpathentry kind="lib" path="lib/json-simple-1.1.1.jar"/>
<classpathentry kind="lib" path="lib/xercesImpl.jar"/> <classpathentry kind="lib" path="lib/xercesImpl.jar"/>
<classpathentry kind="lib" path="lib/xml-apis.jar"/> <classpathentry kind="lib" path="lib/xml-apis.jar"/>
<classpathentry kind="lib" path="lib/geronimo-stax-api_1.0_spec-1.0.1.jar"/> <classpathentry kind="lib" path="lib/geronimo-stax-api_1.0_spec-1.0.1.jar"/>
<classpathentry kind="lib" path="lib/commons-compress-1.9.jar"/> <classpathentry kind="lib" path="lib/commons-compress-1.9.jar"/>
<classpathentry kind="lib" path="lib/commons-lang-2.6.jar"/> <classpathentry kind="lib" path="lib/commons-lang-2.6.jar"/>
<classpathentry kind="lib" path="lib/jaudiotagger-2.0.4-20111207.115108-15.jar"/> <classpathentry kind="lib" path="lib/jaudiotagger-2.0.4-20111207.115108-15.jar"/>
<classpathentry kind="lib" path="lib/commons-codec-1.10.jar"/> <classpathentry kind="lib" path="lib/commons-codec-1.10.jar"/>
<classpathentry kind="lib" path="lib/jcl-over-slf4j-1.7.9.jar"/> <classpathentry kind="lib" path="lib/jcl-over-slf4j-1.7.12.jar"/>
<classpathentry kind="lib" path="lib/wstx-asl-3.2.9.jar"/> <classpathentry kind="lib" path="lib/wstx-asl-3.2.9.jar"/>
<classpathentry kind="lib" path="lib/slf4j-jdk14-1.7.9.jar"/> <classpathentry kind="lib" path="lib/slf4j-jdk14-1.7.12.jar"/>
<classpathentry kind="lib" path="lib/log4j-over-slf4j-1.7.9.jar"/> <classpathentry kind="lib" path="lib/log4j-over-slf4j-1.7.12.jar"/>
<classpathentry kind="lib" path="lib/httpcore-4.4.1.jar"/> <classpathentry kind="lib" path="lib/httpcore-4.4.1.jar"/>
<classpathentry kind="lib" path="lib/metadata-extractor-2.8.1.jar"/> <classpathentry kind="lib" path="lib/metadata-extractor-2.8.1.jar"/>
<classpathentry kind="lib" path="lib/xmpcore-5.1.2.jar"/> <classpathentry kind="lib" path="lib/xmpcore-5.1.2.jar"/>
<classpathentry kind="lib" path="lib/jcifs-1.3.17.jar"/> <classpathentry kind="lib" path="lib/jcifs-1.3.17.jar"/>
<classpathentry kind="lib" path="lib/guava-18.0.jar"/> <classpathentry kind="lib" path="lib/guava-18.0.jar"/>
<classpathentry kind="lib" path="lib/spatial4j-0.4.1.jar"/> <classpathentry kind="lib" path="lib/spatial4j-0.4.1.jar"/>
<classpathentry kind="lib" path="lib/zookeeper-3.4.6.jar"/> <classpathentry kind="lib" path="lib/zookeeper-3.4.6.jar"/>
<classpathentry kind="lib" path="lib/org.restlet.jar"/> <classpathentry kind="lib" path="lib/org.restlet.jar"/>
<classpathentry kind="lib" path="lib/fontbox-1.8.9.jar"/> <classpathentry kind="lib" path="lib/fontbox-1.8.10.jar"/>
<classpathentry kind="lib" path="lib/jempbox-1.8.9.jar"/> <classpathentry kind="lib" path="lib/jempbox-1.8.10.jar"/>
<classpathentry kind="lib" path="lib/pdfbox-1.8.9.jar"/> <classpathentry kind="lib" path="lib/pdfbox-1.8.10.jar"/>
<classpathentry kind="lib" path="lib/bcmail-jdk15-1.46.jar"/> <classpathentry kind="lib" path="lib/bcmail-jdk15-1.46.jar"/>
<classpathentry kind="lib" path="lib/bcprov-jdk15-1.46.jar"/> <classpathentry kind="lib" path="lib/bcprov-jdk15-1.46.jar"/>
<classpathentry kind="lib" path="lib/poi-3.12-20150511.jar"/> <classpathentry kind="lib" path="lib/poi-3.12-20150511.jar"/>
<classpathentry kind="lib" path="lib/poi-scratchpad-3.12-20150511.jar"/> <classpathentry kind="lib" path="lib/poi-scratchpad-3.12-20150511.jar"/>
<classpathentry kind="lib" path="lib/commons-io-2.4.jar"/> <classpathentry kind="lib" path="lib/commons-io-2.4.jar"/>
<classpathentry kind="lib" path="lib/slf4j-api-1.7.9.jar"/> <classpathentry kind="lib" path="lib/slf4j-api-1.7.12.jar"/>
<classpathentry kind="lib" path="lib/chardet.jar"/> <classpathentry kind="lib" path="lib/chardet.jar"/>
<classpathentry kind="lib" path="lib/jetty-client-9.2.11.v20150529.jar"/> <classpathentry kind="lib" path="lib/jetty-client-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-continuation-9.2.11.v20150529.jar"/> <classpathentry kind="lib" path="lib/jetty-continuation-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-deploy-9.2.11.v20150529.jar"/> <classpathentry kind="lib" path="lib/jetty-deploy-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-http-9.2.11.v20150529.jar"/> <classpathentry kind="lib" path="lib/jetty-http-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-io-9.2.11.v20150529.jar"/> <classpathentry kind="lib" path="lib/jetty-io-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-jmx-9.2.11.v20150529.jar"/> <classpathentry kind="lib" path="lib/jetty-jmx-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-proxy-9.2.11.v20150529.jar"/> <classpathentry kind="lib" path="lib/jetty-proxy-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-security-9.2.11.v20150529.jar"/> <classpathentry kind="lib" path="lib/jetty-security-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-server-9.2.11.v20150529.jar"/> <classpathentry kind="lib" path="lib/jetty-server-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-servlet-9.2.11.v20150529.jar"/> <classpathentry kind="lib" path="lib/jetty-servlet-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-servlets-9.2.11.v20150529.jar"/> <classpathentry kind="lib" path="lib/jetty-servlets-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-util-9.2.11.v20150529.jar"/> <classpathentry kind="lib" path="lib/jetty-util-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-webapp-9.2.11.v20150529.jar"/> <classpathentry kind="lib" path="lib/jetty-webapp-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-xml-9.2.11.v20150529.jar"/> <classpathentry kind="lib" path="lib/jetty-xml-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/httpclient-4.4.1.jar"/> <classpathentry kind="lib" path="lib/httpclient-4.5.jar"/>
<classpathentry kind="lib" path="lib/httpmime-4.4.1.jar"/> <classpathentry kind="lib" path="lib/httpmime-4.5.jar"/>
<classpathentry kind="lib" path="lib/noggit-0.6.jar"/> <classpathentry kind="lib" path="lib/noggit-0.6.jar"/>
<classpathentry kind="lib" path="lib/solr-core-5.2.0.jar" sourcepath="/Volumes/Uno/noBackup/Downloads/solr-5.2.0/solr/core/src/java"/> <classpathentry kind="lib" path="lib/solr-core-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/solr-solrj-5.2.0.jar" sourcepath="/Users/admin/Downloads/solr-5.2.1/solr/solrj/src/java"/> <classpathentry kind="lib" path="lib/solr-solrj-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/stax2-api-3.1.4.jar"/> <classpathentry kind="lib" path="lib/stax2-api-3.1.4.jar"/>
<classpathentry kind="lib" path="lib/woodstox-core-asl-4.4.1.jar"/> <classpathentry kind="lib" path="lib/woodstox-core-asl-4.4.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-analyzers-common-5.2.0.jar"/> <classpathentry kind="lib" path="lib/lucene-analyzers-common-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-analyzers-phonetic-5.2.0.jar"/> <classpathentry kind="lib" path="lib/lucene-analyzers-phonetic-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-classification-5.2.0.jar"/> <classpathentry kind="lib" path="lib/lucene-classification-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-codecs-5.2.0.jar"/> <classpathentry kind="lib" path="lib/lucene-codecs-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-core-5.2.0.jar"/> <classpathentry kind="lib" path="lib/lucene-core-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-facet-5.2.0.jar"/> <classpathentry kind="lib" path="lib/lucene-facet-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-grouping-5.2.0.jar"/> <classpathentry kind="lib" path="lib/lucene-grouping-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-highlighter-5.2.0.jar"/> <classpathentry kind="lib" path="lib/lucene-highlighter-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-join-5.2.0.jar"/> <classpathentry kind="lib" path="lib/lucene-join-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-memory-5.2.0.jar"/> <classpathentry kind="lib" path="lib/lucene-memory-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-misc-5.2.0.jar"/> <classpathentry kind="lib" path="lib/lucene-misc-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-queries-5.2.0.jar"/> <classpathentry kind="lib" path="lib/lucene-queries-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-queryparser-5.2.0.jar"/> <classpathentry kind="lib" path="lib/lucene-queryparser-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-spatial-5.2.0.jar"/> <classpathentry kind="lib" path="lib/lucene-spatial-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-suggest-5.2.0.jar"/> <classpathentry kind="lib" path="lib/lucene-suggest-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-backward-codecs-5.2.0.jar"/> <classpathentry kind="lib" path="lib/lucene-backward-codecs-5.2.1.jar"/>
<classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/> <classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/>
<classpathentry kind="lib" path="lib/icu4j-core.jar"/> <classpathentry kind="lib" path="lib/icu4j-55_1.jar"/>
<classpathentry kind="lib" path="lib/htmllexer.jar"/> <classpathentry kind="lib" path="lib/htmllexer.jar"/>
<classpathentry kind="lib" path="lib/jsoup-1.8.2.jar"/> <classpathentry kind="lib" path="lib/jsoup-1.8.2.jar"/>
<classpathentry kind="lib" path="lib/javax.servlet-api-3.1.0.jar"/> <classpathentry kind="lib" path="lib/javax.servlet-api-3.1.0.jar"/>
<classpathentry kind="lib" path="lib/weupnp-0.1.2.jar"/> <classpathentry kind="lib" path="lib/weupnp-0.1.3.jar"/>
<classpathentry kind="output" path="gen"/> <classpathentry kind="output" path="gen"/>
</classpath> </classpath>

60
build.xml
Unescape View File

@ -172,21 +172,21 @@
<pathelement location="${lib}/commons-jxpath-1.3.jar" /> <pathelement location="${lib}/commons-jxpath-1.3.jar" />
<pathelement location="${lib}/commons-lang-2.6.jar" /> <pathelement location="${lib}/commons-lang-2.6.jar" />
<pathelement location="${lib}/commons-logging-1.2.jar" /> <pathelement location="${lib}/commons-logging-1.2.jar" />
<pathelement location="${lib}/fontbox-1.8.9.jar" /> <pathelement location="${lib}/fontbox-1.8.10.jar" />
<pathelement location="${lib}/geronimo-stax-api_1.0_spec-1.0.1.jar" /> <pathelement location="${lib}/geronimo-stax-api_1.0_spec-1.0.1.jar" />
<pathelement location="${lib}/guava-18.0.jar" /> <pathelement location="${lib}/guava-18.0.jar" />
<pathelement location="${lib}/htmllexer.jar" /> <pathelement location="${lib}/htmllexer.jar" />
<pathelement location="${lib}/httpclient-4.4.1.jar" /> <pathelement location="${lib}/httpclient-4.5.jar" />
<pathelement location="${lib}/httpcore-4.4.1.jar" /> <pathelement location="${lib}/httpcore-4.4.1.jar" />
<pathelement location="${lib}/httpmime-4.4.1.jar" /> <pathelement location="${lib}/httpmime-4.5.jar" />
<pathelement location="${lib}/icu4j-core.jar" /> <pathelement location="${lib}/icu4j-55_1.jar" />
<pathelement location="${lib}/J7Zip-modified.jar" /> <pathelement location="${lib}/J7Zip-modified.jar" />
<pathelement location="${lib}/jakarta-oro-2.0.8.jar" /> <pathelement location="${lib}/jakarta-oro-2.0.8.jar" />
<pathelement location="${lib}/jaudiotagger-2.0.4-20111207.115108-15.jar" /> <pathelement location="${lib}/jaudiotagger-2.0.4-20111207.115108-15.jar" />
<pathelement location="${lib}/javax.servlet-api-3.1.0.jar" /> <pathelement location="${lib}/javax.servlet-api-3.1.0.jar" />
<pathelement location="${lib}/jcifs-1.3.17.jar" /> <pathelement location="${lib}/jcifs-1.3.17.jar" />
<pathelement location="${lib}/jcl-over-slf4j-1.7.9.jar" /> <pathelement location="${lib}/jcl-over-slf4j-1.7.12.jar" />
<pathelement location="${lib}/jempbox-1.8.9.jar" /> <pathelement location="${lib}/jempbox-1.8.10.jar" />
<pathelement location="${lib}/jetty-client-9.2.11.v20150529.jar" /> <pathelement location="${lib}/jetty-client-9.2.11.v20150529.jar" />
<pathelement location="${lib}/jetty-continuation-9.2.11.v20150529.jar" /> <pathelement location="${lib}/jetty-continuation-9.2.11.v20150529.jar" />
<pathelement location="${lib}/jetty-deploy-9.2.11.v20150529.jar" /> <pathelement location="${lib}/jetty-deploy-9.2.11.v20150529.jar" />
@ -201,39 +201,39 @@
<pathelement location="${lib}/jetty-util-9.2.11.v20150529.jar" /> <pathelement location="${lib}/jetty-util-9.2.11.v20150529.jar" />
<pathelement location="${lib}/jetty-webapp-9.2.11.v20150529.jar" /> <pathelement location="${lib}/jetty-webapp-9.2.11.v20150529.jar" />
<pathelement location="${lib}/jetty-xml-9.2.11.v20150529.jar" /> <pathelement location="${lib}/jetty-xml-9.2.11.v20150529.jar" />
<pathelement location="${lib}/jsch-0.1.52.jar" /> <pathelement location="${lib}/jsch-0.1.53.jar" />
<pathelement location="${lib}/json-simple-1.1.1.jar" /> <pathelement location="${lib}/json-simple-1.1.1.jar" />
<pathelement location="${lib}/jsoup-1.8.2.jar" /> <pathelement location="${lib}/jsoup-1.8.2.jar" />
<pathelement location="${lib}/log4j-over-slf4j-1.7.9.jar" /> <pathelement location="${lib}/log4j-over-slf4j-1.7.12.jar" />
<pathelement location="${lib}/lucene-analyzers-common-5.2.0.jar" /> <pathelement location="${lib}/lucene-analyzers-common-5.2.1.jar" />
<pathelement location="${lib}/lucene-analyzers-phonetic-5.2.0.jar" /> <pathelement location="${lib}/lucene-analyzers-phonetic-5.2.1.jar" />
<pathelement location="${lib}/lucene-backward-codecs-5.2.0.jar" /> <pathelement location="${lib}/lucene-backward-codecs-5.2.1.jar" />
<pathelement location="${lib}/lucene-classification-5.2.0.jar" /> <pathelement location="${lib}/lucene-classification-5.2.1.jar" />
<pathelement location="${lib}/lucene-codecs-5.2.0.jar" /> <pathelement location="${lib}/lucene-codecs-5.2.1.jar" />
<pathelement location="${lib}/lucene-core-5.2.0.jar" /> <pathelement location="${lib}/lucene-core-5.2.1.jar" />
<pathelement location="${lib}/lucene-facet-5.2.0.jar" /> <pathelement location="${lib}/lucene-facet-5.2.1.jar" />
<pathelement location="${lib}/lucene-grouping-5.2.0.jar" /> <pathelement location="${lib}/lucene-grouping-5.2.1.jar" />
<pathelement location="${lib}/lucene-highlighter-5.2.0.jar" /> <pathelement location="${lib}/lucene-highlighter-5.2.1.jar" />
<pathelement location="${lib}/lucene-join-5.2.0.jar" /> <pathelement location="${lib}/lucene-join-5.2.1.jar" />
<pathelement location="${lib}/lucene-memory-5.2.0.jar" /> <pathelement location="${lib}/lucene-memory-5.2.1.jar" />
<pathelement location="${lib}/lucene-misc-5.2.0.jar" /> <pathelement location="${lib}/lucene-misc-5.2.1.jar" />
<pathelement location="${lib}/lucene-queries-5.2.0.jar" /> <pathelement location="${lib}/lucene-queries-5.2.1.jar" />
<pathelement location="${lib}/lucene-queryparser-5.2.0.jar" /> <pathelement location="${lib}/lucene-queryparser-5.2.1.jar" />
<pathelement location="${lib}/lucene-spatial-5.2.0.jar" /> <pathelement location="${lib}/lucene-spatial-5.2.1.jar" />
<pathelement location="${lib}/lucene-suggest-5.2.0.jar" /> <pathelement location="${lib}/lucene-suggest-5.2.1.jar" />
<pathelement location="${lib}/metadata-extractor-2.8.1.jar" /> <pathelement location="${lib}/metadata-extractor-2.8.1.jar" />
<pathelement location="${lib}/noggit-0.5.jar" /> <pathelement location="${lib}/noggit-0.5.jar" />
<pathelement location="${lib}/org.restlet.jar" /> <pathelement location="${lib}/org.restlet.jar" />
<pathelement location="${lib}/pdfbox-1.8.9.jar" /> <pathelement location="${lib}/pdfbox-1.8.10.jar" />
<pathelement location="${lib}/poi-3.12-20150511.jar" /> <pathelement location="${lib}/poi-3.12-20150511.jar" />
<pathelement location="${lib}/poi-scratchpad-3.12-20150511.jar" /> <pathelement location="${lib}/poi-scratchpad-3.12-20150511.jar" />
<pathelement location="${lib}/slf4j-api-1.7.9.jar" /> <pathelement location="${lib}/slf4j-api-1.7.12.jar" />
<pathelement location="${lib}/slf4j-jdk14-1.7.9.jar" /> <pathelement location="${lib}/slf4j-jdk14-1.7.12.jar" />
<pathelement location="${lib}/solr-core-5.2.0.jar" /> <pathelement location="${lib}/solr-core-5.2.1.jar" />
<pathelement location="${lib}/solr-solrj-5.2.0.jar" /> <pathelement location="${lib}/solr-solrj-5.2.1.jar" />
<pathelement location="${lib}/spatial4j-0.4.1.jar" /> <pathelement location="${lib}/spatial4j-0.4.1.jar" />
<pathelement location="${lib}/webcat-0.1-swf.jar" /> <pathelement location="${lib}/webcat-0.1-swf.jar" />
<pathelement location="${lib}/weupnp-0.1.2.jar" /> <pathelement location="${lib}/weupnp-0.1.3.jar" />
<pathelement location="${lib}/wstx-asl-3.2.9.jar" /> <pathelement location="${lib}/wstx-asl-3.2.9.jar" />
<pathelement location="${lib}/xercesImpl.jar" /> <pathelement location="${lib}/xercesImpl.jar" />
<pathelement location="${lib}/xml-apis.jar" /> <pathelement location="${lib}/xml-apis.jar" />

1
defaults/yacy.init
Unescape View File

@ -103,6 +103,7 @@ filesize.max.other = 8589934591
# In case of privately managed networks, this configuration must be changed BEFORE it is released # In case of privately managed networks, this configuration must be changed BEFORE it is released
# to the members of the separated network peers. # to the members of the separated network peers.
network.unit.definition = defaults/yacy.network.freeworld.unit network.unit.definition = defaults/yacy.network.freeworld.unit
#network.unit.definition = defaults/yacy.network.webportal.unit
#network.unit.definition = defaults/yacy.network.intranet.unit #network.unit.definition = defaults/yacy.network.intranet.unit
# distinguish intranet/internet IPs: # distinguish intranet/internet IPs:

8
htroot/IndexExport_p.html
Unescape View File

@ -39,19 +39,21 @@
<dt>Only Domain:</dt> <dt>Only Domain:</dt>
<dd><input type="radio" name="format" value="dom-text" /> Plain Text List (domains only)<br /> <dd><input type="radio" name="format" value="dom-text" /> Plain Text List (domains only)<br />
<input type="radio" name="format" value="dom-html" /> HTML (domains as URLs, no title)</dd> <input type="radio" name="format" value="dom-html" /> HTML (domains as URLs, no title)</dd>
<dt>Only Text:</dt>
<dd><input type="radio" name="format" value="text-text" /> Fulltext of Search Index Text</dd>
</dl> </dl>
</dd> </dd>
<dt>&nbsp;</dt> <dt>&nbsp;</dt>
<dd><input type="submit" name="lurlexport" value="Export URLs" class="btn btn-primary" style="width:240px;"/> <dd><input type="submit" name="lurlexport" value="Export" class="btn btn-primary" style="width:240px;"/>
</dd> </dd>
</dl> </dl>
</fieldset> </fieldset>
</form>:: </form>::
<div class="alert alert-info" style="text-decoration:blink">Export to file #[exportfile]# is running .. #[urlcount]# URLs so far</div>:: <div class="alert alert-info" style="text-decoration:blink">Export to file #[exportfile]# is running .. #[urlcount]# Documents so far</div>::
#(/lurlexport)# #(/lurlexport)#
#(lurlexportfinished)#:: #(lurlexportfinished)#::
<div class="alert alert-success">Finished export of #[urlcount]# URLs to file <a href="file://#[exportfile]#" target="_">#[exportfile]#</a><br/> <div class="alert alert-success">Finished export of #[urlcount]# Documents to file <a href="file://#[exportfile]#" target="_">#[exportfile]#</a><br/>
<em>Import this file by moving it to DATA/SURROGATES/in</em></div>:: <em>Import this file by moving it to DATA/SURROGATES/in</em></div>::
#(/lurlexportfinished)# #(/lurlexportfinished)#

21
htroot/IndexExport_p.java
Unescape View File

@ -93,27 +93,28 @@ public class IndexExport_p {
if (post.containsKey("lurlexport")) { if (post.containsKey("lurlexport")) {
// parse format // parse format
int format = 0; Fulltext.ExportFormat format = Fulltext.ExportFormat.text;
final String fname = post.get("format", "url-text"); final String fname = post.get("format", "url-text");
final boolean dom = fname.startsWith("dom"); // if dom== false complete urls are exported, otherwise only the domain final boolean dom = fname.startsWith("dom"); // if dom== false complete urls are exported, otherwise only the domain
if (fname.endsWith("text")) format = 0; final boolean text = fname.startsWith("text");
if (fname.endsWith("html")) format = 1; if (fname.endsWith("text")) format = Fulltext.ExportFormat.text;
if (fname.endsWith("rss")) format = 2; if (fname.endsWith("html")) format = Fulltext.ExportFormat.html;
if (fname.endsWith("solr")) format = 3; if (fname.endsWith("rss")) format = Fulltext.ExportFormat.rss;
if (fname.endsWith("solr")) format = Fulltext.ExportFormat.solr;
// extend export file name // extend export file name
String s = post.get("exportfile", ""); String s = post.get("exportfile", "");
if (s.indexOf('.',0) < 0) { if (s.indexOf('.',0) < 0) {
if (format == 0) s = s + ".txt"; if (format == Fulltext.ExportFormat.text) s = s + ".txt";
if (format == 1) s = s + ".html"; if (format == Fulltext.ExportFormat.html) s = s + ".html";
if (format == 2 ) s = s + "_rss.xml"; if (format == Fulltext.ExportFormat.rss ) s = s + "_rss.xml";
if (format == 3) s = s + "_full.xml"; if (format == Fulltext.ExportFormat.solr) s = s + "_full.xml";
} }
final File f = new File(s); final File f = new File(s);
f.getParentFile().mkdirs(); f.getParentFile().mkdirs();
final String filter = post.get("exportfilter", ".*"); final String filter = post.get("exportfilter", ".*");
final String query = post.get("exportquery", "*:*"); final String query = post.get("exportquery", "*:*");
final Fulltext.Export running = segment.fulltext().export(f, filter, query, format, dom); final Fulltext.Export running = segment.fulltext().export(f, filter, query, format, dom, text);
prop.put("lurlexport_exportfile", s); prop.put("lurlexport_exportfile", s);
prop.put("lurlexport_urlcount", running.count()); prop.put("lurlexport_urlcount", running.count());

0
lib/fontbox-1.8.9.License → lib/fontbox-1.8.10.License
Unescape View File

BIN
lib/fontbox-1.8.9.jar → lib/fontbox-1.8.10.jar
View File

Binary file not shown.

0
lib/httpclient-4.4.1.License → lib/httpclient-4.5.License
Unescape View File

BIN
lib/httpclient-4.4.1.jar → lib/httpclient-4.5.jar
View File

Binary file not shown.

0
lib/httpmime-4.4.1.License → lib/httpmime-4.5.License
Unescape View File

BIN
lib/httpmime-4.4.1.jar → lib/httpmime-4.5.jar
View File

Binary file not shown.

BIN
lib/icu4j-55_1.jar
View File

Binary file not shown.

BIN
lib/icu4j-core.jar
View File

Binary file not shown.

BIN
lib/jcl-over-slf4j-1.7.12.jar
View File

Binary file not shown.

BIN
lib/jcl-over-slf4j-1.7.9.jar
View File

Binary file not shown.

0
lib/jempbox-1.8.9.License → lib/jempbox-1.8.10.License
Unescape View File

BIN
lib/jempbox-1.8.9.jar → lib/jempbox-1.8.10.jar
View File

Binary file not shown.

BIN
lib/jsch-0.1.52.jar
View File

Binary file not shown.

60
lib/jsch-0.1.52.License → lib/jsch-0.1.53.License
Unescape View File

@ -1,30 +1,30 @@
JSch 0.0.* was released under the GNU LGPL license. Later, we have switched JSch 0.0.* was released under the GNU LGPL license. Later, we have switched
over to a BSD-style license. over to a BSD-style license.
------------------------------------------------------------------------------ ------------------------------------------------------------------------------
Copyright (c) 2002,2003,2004 Atsuhiko Yamanaka, JCraft,Inc. Copyright (c) 2002-2015 Atsuhiko Yamanaka, JCraft,Inc.
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met: modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, 1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer. this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright 2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the distribution. the documentation and/or other materials provided with the distribution.
3. The names of the authors may not be used to endorse or promote products 3. The names of the authors may not be used to endorse or promote products
derived from this software without specific prior written permission. derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT, FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT,
INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

BIN
lib/jsch-0.1.53.jar
View File

Binary file not shown.

BIN
lib/log4j-over-slf4j-1.7.12.jar
View File

Binary file not shown.

BIN
lib/log4j-over-slf4j-1.7.9.jar
View File

Binary file not shown.

BIN
lib/lucene-analyzers-common-5.2.0.jar → lib/lucene-analyzers-common-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-analyzers-phonetic-5.2.0.jar → lib/lucene-analyzers-phonetic-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-backward-codecs-5.2.0.jar → lib/lucene-backward-codecs-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-classification-5.2.0.jar → lib/lucene-classification-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-codecs-5.2.0.jar → lib/lucene-codecs-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-core-5.2.0.jar → lib/lucene-core-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-facet-5.2.0.jar → lib/lucene-facet-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-grouping-5.2.0.jar → lib/lucene-grouping-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-highlighter-5.2.0.jar → lib/lucene-highlighter-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-join-5.2.0.jar → lib/lucene-join-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-memory-5.2.0.jar → lib/lucene-memory-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-misc-5.2.0.jar → lib/lucene-misc-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-queries-5.2.0.jar → lib/lucene-queries-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-queryparser-5.2.0.jar → lib/lucene-queryparser-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-spatial-5.2.0.jar → lib/lucene-spatial-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-suggest-5.2.0.jar → lib/lucene-suggest-5.2.1.jar
View File

Binary file not shown.

0
lib/pdfbox-1.8.9.License → lib/pdfbox-1.8.10.License
Unescape View File

BIN
lib/pdfbox-1.8.9.jar → lib/pdfbox-1.8.10.jar
View File

Binary file not shown.

BIN
lib/slf4j-api-1.7.12.jar
View File

Binary file not shown.

BIN
lib/slf4j-api-1.7.9.jar
View File

Binary file not shown.

BIN
lib/slf4j-jdk14-1.7.12.jar
View File

Binary file not shown.

BIN
lib/slf4j-jdk14-1.7.9.jar
View File

Binary file not shown.

BIN
lib/solr-core-5.2.0.jar → lib/solr-core-5.2.1.jar
View File

Binary file not shown.

BIN
lib/solr-solrj-5.2.0.jar → lib/solr-solrj-5.2.1.jar
View File

Binary file not shown.

BIN
lib/weupnp-0.1.2.jar
View File

Binary file not shown.

BIN
lib/weupnp-0.1.3.jar
View File

Binary file not shown.

2
nbproject/project.xml
Unescape View File

@ -83,7 +83,7 @@
<compilation-unit> <compilation-unit>
<package-root>source</package-root> <package-root>source</package-root>
<package-root>htroot</package-root> <package-root>htroot</package-root>
<classpath mode="compile">lib/J7Zip-modified.jar;lib/apache-mime4j-0.6.jar;lib/bcmail-jdk15-1.46.jar;lib/bcprov-jdk15-1.46.jar;lib/chardet.jar;lib/commons-codec-1.10.jar;lib/commons-compress-1.9.jar;lib/commons-fileupload-1.3.1.jar;lib/commons-io-2.4.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.2.jar;lib/fontbox-1.8.9.jar;lib/geronimo-stax-api_1.0_spec-1.0.1.jar;lib/guava-18.0.jar;lib/htmllexer.jar;lib/httpclient-4.4.1.jar;lib/httpcore-4.4.1.jar;lib/httpmime-4.4.1.jar;lib/icu4j-core.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/javax.servlet-api-3.1.0.jar;lib/jcifs-1.3.17.jar;lib/jcl-over-slf4j-1.7.9.jar;lib/jempbox-1.8.9.jar;lib/jetty-client-9.2.11.v20150529.jar;lib/jetty-continuation-9.2.11.v20150529.jar;lib/jetty-deploy-9.2.11.v20150529.jar;lib/jetty-http-9.2.11.v20150529.jar;lib/jetty-io-9.2.11.v20150529.jar;lib/jetty-jmx-9.2.11.v20150529.jar;lib/jetty-proxy-9.2.11.v20150529.jar;lib/jetty-security-9.2.11.v20150529.jar;lib/jetty-server-9.2.11.v20150529.jar;lib/jetty-servlet-9.2.11.v20150529.jar;lib/jetty-servlets-9.2.11.v20150529.jar;lib/jetty-util-9.2.11.v20150529.jar;lib/jetty-webapp-9.2.11.v20150529.jar;lib/jetty-xml-9.2.11.v20150529.jar;lib/jsch-0.1.52.jar;lib/json-simple-1.1.1.jar;lib/jsoup-1.8.2.jar;lib/log4j-over-slf4j-1.7.9.jar;lib/lucene-analyzers-common-5.2.0.jar;lib/lucene-analyzers-phonetic-5.2.0.jar;lib/lucene-backward-codecs-5.2.0.jar;lib/lucene-classification-5.2.0.jar;lib/lucene-codecs-5.2.0.jar;lib/lucene-core-5.2.0.jar;lib/lucene-facet-5.2.0.jar;lib/lucene-grouping-5.2.0.jar;lib/lucene-highlighter-5.2.0.jar;lib/lucene-join-5.2.0.jar;lib/lucene-memory-5.2.0.jar;lib/lucene-misc-5.2.0.jar;lib/lucene-queries-5.2.0.jar;lib/lucene-queryparser-5.2.0.jar;lib/lucene-spatial-5.2.0.jar;lib/lucene-suggest-5.2.0.jar;lib/metadata-extractor-2.8.1.jar;lib/noggit-0.6.jar;lib/org.restlet.jar;lib/pdfbox-1.8.9.jar;lib/poi-3.12-20150511.jar;lib/poi-scratchpad-3.12-20150511.jar;lib/slf4j-api-1.7.9.jar;lib/slf4j-jdk14-1.7.9.jar;lib/solr-core-5.2.0.jar;lib/solr-solrj-5.2.0.jar;lib/spatial4j-0.4.1.jar;lib/stax2-api-3.1.4.jar;lib/webcat-0.1-swf.jar;lib/weupnp-0.1.2.jar;lib/woodstox-core-asl-4.4.1.jar;lib/wstx-asl-3.2.9.jar;lib/xercesImpl.jar;lib/xml-apis.jar;lib/xmpcore-5.1.2.jar;lib/zookeeper-3.4.6.jar</classpath> <classpath mode="compile">lib/J7Zip-modified.jar;lib/apache-mime4j-0.6.jar;lib/bcmail-jdk15-1.46.jar;lib/bcprov-jdk15-1.46.jar;lib/chardet.jar;lib/commons-codec-1.10.jar;lib/commons-compress-1.9.jar;lib/commons-fileupload-1.3.1.jar;lib/commons-io-2.4.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.2.jar;lib/fontbox-1.8.10.jar;lib/geronimo-stax-api_1.0_spec-1.0.1.jar;lib/guava-18.0.jar;lib/htmllexer.jar;lib/httpclient-4.5.jar;lib/httpcore-4.4.1.jar;lib/httpmime-4.5.jar;lib/icu4j-55_1.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/javax.servlet-api-3.1.0.jar;lib/jcifs-1.3.17.jar;lib/jcl-over-slf4j-1.7.12.jar;lib/jempbox-1.8.10.jar;lib/jetty-client-9.2.11.v20150529.jar;lib/jetty-continuation-9.2.11.v20150529.jar;lib/jetty-deploy-9.2.11.v20150529.jar;lib/jetty-http-9.2.11.v20150529.jar;lib/jetty-io-9.2.11.v20150529.jar;lib/jetty-jmx-9.2.11.v20150529.jar;lib/jetty-proxy-9.2.11.v20150529.jar;lib/jetty-security-9.2.11.v20150529.jar;lib/jetty-server-9.2.11.v20150529.jar;lib/jetty-servlet-9.2.11.v20150529.jar;lib/jetty-servlets-9.2.11.v20150529.jar;lib/jetty-util-9.2.11.v20150529.jar;lib/jetty-webapp-9.2.11.v20150529.jar;lib/jetty-xml-9.2.11.v20150529.jar;lib/jsch-0.1.53.jar;lib/json-simple-1.1.1.jar;lib/jsoup-1.8.2.jar;lib/log4j-over-slf4j-1.7.12.jar;lib/lucene-analyzers-common-5.2.1.jar;lib/lucene-analyzers-phonetic-5.2.1.jar;lib/lucene-backward-codecs-5.2.1.jar;lib/lucene-classification-5.2.1.jar;lib/lucene-codecs-5.2.1.jar;lib/lucene-core-5.2.1.jar;lib/lucene-facet-5.2.1.jar;lib/lucene-grouping-5.2.1.jar;lib/lucene-highlighter-5.2.1.jar;lib/lucene-join-5.2.1.jar;lib/lucene-memory-5.2.1.jar;lib/lucene-misc-5.2.1.jar;lib/lucene-queries-5.2.1.jar;lib/lucene-queryparser-5.2.1.jar;lib/lucene-spatial-5.2.1.jar;lib/lucene-suggest-5.2.1.jar;lib/metadata-extractor-2.8.1.jar;lib/noggit-0.6.jar;lib/org.restlet.jar;lib/pdfbox-1.8.10.jar;lib/poi-3.12-20150511.jar;lib/poi-scratchpad-3.12-20150511.jar;lib/slf4j-api-1.7.12.jar;lib/slf4j-jdk14-1.7.12.jar;lib/solr-core-5.2.1.jar;lib/solr-solrj-5.2.1.jar;lib/spatial4j-0.4.1.jar;lib/stax2-api-3.1.4.jar;lib/webcat-0.1-swf.jar;lib/weupnp-0.1.3.jar;lib/woodstox-core-asl-4.4.1.jar;lib/wstx-asl-3.2.9.jar;lib/xercesImpl.jar;lib/xml-apis.jar;lib/xmpcore-5.1.2.jar;lib/zookeeper-3.4.6.jar</classpath>
<built-to>lib/yacycore.jar</built-to> <built-to>lib/yacycore.jar</built-to>
<source-level>1.7</source-level> <source-level>1.7</source-level>
</compilation-unit> </compilation-unit>

84
pom.xml
Unescape View File

@ -366,7 +366,7 @@
<dependency> <dependency>
<groupId>org.apache.pdfbox</groupId> <groupId>org.apache.pdfbox</groupId>
<artifactId>fontbox</artifactId> <artifactId>fontbox</artifactId>
<version>1.8.9</version> <version>1.8.10</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.geronimo.specs</groupId> <groupId>org.apache.geronimo.specs</groupId>
@ -386,7 +386,7 @@
<dependency> <dependency>
<groupId>org.apache.httpcomponents</groupId> <groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId> <artifactId>httpclient</artifactId>
<version>4.4</version> <version>4.5</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.httpcomponents</groupId> <groupId>org.apache.httpcomponents</groupId>
@ -396,12 +396,12 @@
<dependency> <dependency>
<groupId>org.apache.httpcomponents</groupId> <groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId> <artifactId>httpmime</artifactId>
<version>4.4</version> <version>4.5</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.ibm.icu</groupId> <groupId>com.ibm.icu</groupId>
<artifactId>icu4j</artifactId> <artifactId>icu4j</artifactId>
<version>4.8.1.1</version> <version>55.1</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org</groupId> <groupId>org</groupId>
@ -421,17 +421,17 @@
<dependency> <dependency>
<groupId>org.slf4j</groupId> <groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId> <artifactId>jcl-over-slf4j</artifactId>
<version>1.7.9</version> <version>1.7.12</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.pdfbox</groupId> <groupId>org.apache.pdfbox</groupId>
<artifactId>jempbox</artifactId> <artifactId>jempbox</artifactId>
<version>1.8.9</version> <version>1.8.10</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.jcraft</groupId> <groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId> <artifactId>jsch</artifactId>
<version>0.1.52</version> <version>0.1.53</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.googlecode.json-simple</groupId> <groupId>com.googlecode.json-simple</groupId>
@ -443,66 +443,16 @@
<artifactId>jsoup</artifactId> <artifactId>jsoup</artifactId>
<version>1.8.2</version> <version>1.8.2</version>
</dependency> </dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
</dependency>
<dependency> <dependency>
<groupId>org.slf4j</groupId> <groupId>org.slf4j</groupId>
<artifactId>log4j-over-slf4j</artifactId> <artifactId>log4j-over-slf4j</artifactId>
<version>1.7.9</version> <version>1.7.12</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
<version>${solr.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-phonetic</artifactId>
<version>${solr.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.lucene</groupId> <groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId> <artifactId>lucene-core</artifactId>
<version>${solr.version}</version> <version>${solr.version}</version>
</dependency> </dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-grouping</artifactId>
<version>${solr.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-highlighter</artifactId>
<version>${solr.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-memory</artifactId>
<version>${solr.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-misc</artifactId>
<version>${solr.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queries</artifactId>
<version>${solr.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-spatial</artifactId>
<version>${solr.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-suggest</artifactId>
<version>${solr.version}</version>
</dependency>
<dependency> <dependency>
<groupId>com.drewnoakes</groupId> <groupId>com.drewnoakes</groupId>
<artifactId>metadata-extractor</artifactId> <artifactId>metadata-extractor</artifactId>
@ -516,7 +466,7 @@
<dependency> <dependency>
<groupId>org.apache.pdfbox</groupId> <groupId>org.apache.pdfbox</groupId>
<artifactId>pdfbox</artifactId> <artifactId>pdfbox</artifactId>
<version>1.8.9</version> <version>1.8.10</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.poi</groupId> <groupId>org.apache.poi</groupId>
@ -536,12 +486,12 @@
<dependency> <dependency>
<groupId>org.slf4j</groupId> <groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId> <artifactId>slf4j-api</artifactId>
<version>1.7.9</version> <version>1.7.12</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.slf4j</groupId> <groupId>org.slf4j</groupId>
<artifactId>slf4j-jdk14</artifactId> <artifactId>slf4j-jdk14</artifactId>
<version>1.7.9</version> <version>1.7.12</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.solr</groupId> <groupId>org.apache.solr</groupId>
@ -552,16 +502,6 @@
<groupId>org.apache.solr</groupId> <groupId>org.apache.solr</groupId>
<artifactId>solr-core</artifactId> <artifactId>solr-core</artifactId>
<version>${solr.version}</version> <version>${solr.version}</version>
<exclusions>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>javax.servlet</artifactId>
<groupId>org.eclipse.jetty.orbit</groupId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.eclipse.jetty</groupId> <groupId>org.eclipse.jetty</groupId>
@ -631,7 +571,7 @@
<dependency> <dependency>
<groupId>org.bitlet</groupId> <groupId>org.bitlet</groupId>
<artifactId>weupnp</artifactId> <artifactId>weupnp</artifactId>
<version>0.1.2</version> <version>0.1.3</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.codehaus.woodstox</groupId> <groupId>org.codehaus.woodstox</groupId>

12
source/net/yacy/cora/document/id/MultiProtocolURL.java
Unescape View File

@ -843,14 +843,12 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
final int q = fileName.lastIndexOf('?'); final int q = fileName.lastIndexOf('?');
if (q < 0) { if (q < 0) {
return fileName.substring(p + 1).toLowerCase(); return fileName.substring(p + 1).toLowerCase();
} else {
// check last dot in query part
if (p > q) {
return ""; // TODO: last . after ? (file.ext?param=one.txt)
} else {
return fileName.substring(p + 1, q).toLowerCase();
}
} }
// check last dot in query part
if (p > q) {
return ""; // TODO: last . after ? (file.ext?param=one.txt)
}
return fileName.substring(p + 1, q).toLowerCase();
} }
public String getPath() { public String getPath() {

7
source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java
Unescape View File

@ -205,7 +205,8 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
String fq = req.getParams().get(CommonParams.FQ); String fq = req.getParams().get(CommonParams.FQ);
String sort = req.getParams().get(CommonParams.SORT); String sort = req.getParams().get(CommonParams.SORT);
String threadname = Thread.currentThread().getName(); String threadname = Thread.currentThread().getName();
if (q != null) Thread.currentThread().setName("solr query: q = " + q + (fq == null ? "" : ", fq = " + fq) + (sort == null ? "" : ", sort = " + sort)); // for debugging in Threaddump if (q != null) Thread.currentThread().setName("solr query: q=" + q + (fq == null ? "" : "&fq = " + fq) + (sort == null ? "" : "&sort = " + sort)); // for debugging in Threaddump
ConcurrentLog.info("EmbeddedSolrConnector.query", "QUERY: q=" + q + (fq == null ? "" : "&" + fq.toString()) + (sort == null ? "" : "&sort = " + sort) + " SolrQueryRequest=" + req.toString());
SolrQueryResponse rsp = new SolrQueryResponse(); SolrQueryResponse rsp = new SolrQueryResponse();
NamedList<Object> responseHeader = new SimpleOrderedMap<Object>(); NamedList<Object> responseHeader = new SimpleOrderedMap<Object>();
@ -334,8 +335,8 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
if (q != null) { if (q != null) {
StringBuilder fqa = new StringBuilder(); StringBuilder fqa = new StringBuilder();
if (fq != null) for (String f: fq) fqa.append("fq=").append(f).append(' '); if (fq != null) for (String f: fq) fqa.append("fq=").append(f).append(' ');
Thread.currentThread().setName("solr query: q = " + q + (fq == null ? "" : ", " + fqa.toString()) + (fl == null ? "" : ", fl=" + fl)); Thread.currentThread().setName("solr query: q=" + q + (fq == null ? "" : "&" + fqa.toString()) + (fl == null ? "" : "&fl=" + fl));
//System.out.println("solr query: q = " + q + (fq == null ? "" : ", " + fqa.toString()) + (fl == null ? "" : ", fl=" + fl)); ConcurrentLog.info("EmbeddedSolrConnector.getResponseByParams", "QUERY: q=" + q + (fq == null ? "" : "&" + fqa.toString()) + (fl == null ? "" : "&fl=" + fl) + " PARAMS: " + params.toString());
} }
QueryResponse rsp; QueryResponse rsp;
try { try {

22
source/net/yacy/crawler/data/Cache.java
Unescape View File

@ -86,6 +86,15 @@ public final class Cache {
responseHeaderDB = new MapHeap(dbfile, Word.commonHashLength, Base64Order.enhancedCoder, 2048, 100, ' '); responseHeaderDB = new MapHeap(dbfile, Word.commonHashLength, Base64Order.enhancedCoder, 2048, 100, ' ');
} catch (final IOException e) { } catch (final IOException e) {
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
// try a healing
if (dbfile.exists()) {
dbfile.delete();
try {
responseHeaderDB = new MapHeap(dbfile, Word.commonHashLength, Base64Order.enhancedCoder, 2048, 100, ' ');
} catch (final IOException ee) {
ConcurrentLog.logException(e);
}
}
} }
// open the cache file // open the cache file
try { try {
@ -94,8 +103,19 @@ public final class Cache {
fileDB = new Compressor(fileDBunbuffered, 6 * 1024 * 1024); fileDB = new Compressor(fileDBunbuffered, 6 * 1024 * 1024);
} catch (final IOException e) { } catch (final IOException e) {
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
// try a healing
if (cachePath.exists()) {
cachePath.delete();
try {
fileDBunbuffered = new ArrayStack(new File(cachePath, FILE_DB_NAME), prefix, Base64Order.enhancedCoder, 12, 1024 * 1024 * 2, false, true);
fileDBunbuffered.setMaxSize(maxCacheSize);
fileDB = new Compressor(fileDBunbuffered, 6 * 1024 * 1024);
} catch (final IOException ee) {
ConcurrentLog.logException(e);
}
}
} }
ConcurrentLog.info("Cache", "initialized cache database responseHeaderDB.size() = " + responseHeaderDB.size() + ", fileDB.size() = " + fileDB.size()); ConcurrentLog.info("Cache", "initialized cache database responseHeaderDB.size() = " + (responseHeaderDB == null ? "NULL" : responseHeaderDB.size()) + ", fileDB.size() = " + (fileDB == null ? "NULL" : fileDB.size()));
// clean up the responseHeaderDB which cannot be cleaned the same way as the cache files. // clean up the responseHeaderDB which cannot be cleaned the same way as the cache files.
// We do this as a concurrent job only once after start-up silently // We do this as a concurrent job only once after start-up silently

27
source/net/yacy/http/AbstractRemoteHandler.java
Unescape View File

@ -64,15 +64,28 @@ abstract public class AbstractRemoteHandler extends ConnectHandler implements Ha
new Thread() { new Thread() {
@Override @Override
public void run() { public void run() {
final InetAddress localInetAddress = Domains.myPublicLocalIP(); for (InetAddress localInetAddress : Domains.myPublicIPv4()) {
if (localInetAddress != null) { if (localInetAddress != null) {
if (!localVirtualHostNames.contains(localInetAddress.getHostName())) { if (!localVirtualHostNames.contains(localInetAddress.getHostName())) {
localVirtualHostNames.add(localInetAddress.getHostName()); localVirtualHostNames.add(localInetAddress.getHostName());
localVirtualHostNames.add(localInetAddress.getHostAddress()); // same as getServer().getURI().getHost() localVirtualHostNames.add(localInetAddress.getHostAddress()); // same as getServer().getURI().getHost()
}
if (!localVirtualHostNames.contains(localInetAddress.getCanonicalHostName())) {
localVirtualHostNames.add(localInetAddress.getCanonicalHostName());
}
} }
}
for (InetAddress localInetAddress : Domains.myPublicIPv6()) {
if (localInetAddress != null) {
if (!localVirtualHostNames.contains(localInetAddress.getHostName())) {
localVirtualHostNames.add(localInetAddress.getHostName());
localVirtualHostNames.add(localInetAddress.getHostAddress()); // same as getServer().getURI().getHost()
}
if (!localVirtualHostNames.contains(localInetAddress.getCanonicalHostName())) { if (!localVirtualHostNames.contains(localInetAddress.getCanonicalHostName())) {
localVirtualHostNames.add(localInetAddress.getCanonicalHostName()); localVirtualHostNames.add(localInetAddress.getCanonicalHostName());
}
} }
} }
if (sb.peers != null) { if (sb.peers != null) {

23
source/net/yacy/peers/Transmission.java
Unescape View File

@ -83,13 +83,11 @@ public class Transmission {
* to multiple peers and to ensure that all entries in the indexContainers * to multiple peers and to ensure that all entries in the indexContainers
* have a reference in the urls * have a reference in the urls
* - a set of yacy seeds which will shrink as the containers are transmitted to them * - a set of yacy seeds which will shrink as the containers are transmitted to them
* - a counter that gives the number of sucessful and unsuccessful transmissions so far
*/ */
private final Seed dhtTarget; private final Seed dhtTarget;
private final ReferenceContainerCache<WordReference> containers; private final ReferenceContainerCache<WordReference> containers;
private final HandleSet references; private final HandleSet references;
private final HandleSet badReferences; private final HandleSet badReferences;
private int hit, miss;
/** /**
* generate a new dispatcher target. such a target is defined with a primary target and * generate a new dispatcher target. such a target is defined with a primary target and
@ -103,8 +101,6 @@ public class Transmission {
this.containers = new ReferenceContainerCache<WordReference>(Segment.wordReferenceFactory, Segment.wordOrder, Word.commonHashLength); this.containers = new ReferenceContainerCache<WordReference>(Segment.wordReferenceFactory, Segment.wordOrder, Word.commonHashLength);
this.references = new RowHandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0); this.references = new RowHandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0);
this.badReferences = new RowHandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0); this.badReferences = new RowHandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0);
this.hit = 0;
this.miss = 0;
} }
/* /*
@ -211,28 +207,11 @@ public class Transmission {
return this.dhtTarget; return this.dhtTarget;
} }
/**
* return the number of successful transmissions
* @return
*/
public int hit() {
return this.hit;
}
/**
* return the number of unsuccessful transmissions
* @return
*/
public int miss() {
return this.miss;
}
public boolean transmit() { public boolean transmit() {
// transferring selected words to remote peer // transferring selected words to remote peer
if (this.dhtTarget == Transmission.this.seeds.mySeed() || this.dhtTarget.hash.equals(Transmission.this.seeds.mySeed().hash)) { if (this.dhtTarget == Transmission.this.seeds.mySeed() || this.dhtTarget.hash.equals(Transmission.this.seeds.mySeed().hash)) {
// target is my own peer. This is easy. Just restore the indexContainer // target is my own peer. This is easy. Just restore the indexContainer
restore(); restore();
this.hit++;
Transmission.this.log.info("Transfer of chunk to myself-target"); Transmission.this.log.info("Transfer of chunk to myself-target");
return true; return true;
} }
@ -256,13 +235,11 @@ public class Transmission {
// if the peer has set a pause time and we are in flush mode (index transfer) // if the peer has set a pause time and we are in flush mode (index transfer)
// then we pause for a while now // then we pause for a while now
Transmission.this.log.info("Transfer finished of chunk to target " + this.dhtTarget.hash + "/" + this.dhtTarget.getName()); Transmission.this.log.info("Transfer finished of chunk to target " + this.dhtTarget.hash + "/" + this.dhtTarget.getName());
this.hit++;
return true; return true;
} }
Transmission.this.log.info( Transmission.this.log.info(
"Index transfer to peer " + this.dhtTarget.getName() + ":" + this.dhtTarget.hash + "Index transfer to peer " + this.dhtTarget.getName() + ":" + this.dhtTarget.hash +
" failed: " + error); " failed: " + error);
this.miss++;
// write information that peer does not receive index transmissions // write information that peer does not receive index transmissions
Transmission.this.log.info("Transfer failed of chunk to target " + this.dhtTarget.hash + "/" + this.dhtTarget.getName() + ": " + error); Transmission.this.log.info("Transfer failed of chunk to target " + this.dhtTarget.hash + "/" + this.dhtTarget.getName() + ": " + error);
// get possibly newer target Info // get possibly newer target Info

12
source/net/yacy/search/Switchboard.java
Unescape View File

@ -315,7 +315,7 @@ public final class Switchboard extends serverSwitch {
// set loglevel and log // set loglevel and log
setLog(new ConcurrentLog("SWITCHBOARD")); setLog(new ConcurrentLog("SWITCHBOARD"));
AccessTracker.setDumpFile(new File("DATA/LOG/queries.log")); AccessTracker.setDumpFile(new File(dataPath, "DATA/LOG/queries.log"));
// set default peer name // set default peer name
Seed.ANON_PREFIX = getConfig("peernameprefix", "_anon"); Seed.ANON_PREFIX = getConfig("peernameprefix", "_anon");
@ -359,7 +359,7 @@ public final class Switchboard extends serverSwitch {
this.workPath = getDataPath(SwitchboardConstants.WORK_PATH, SwitchboardConstants.WORK_PATH_DEFAULT); this.workPath = getDataPath(SwitchboardConstants.WORK_PATH, SwitchboardConstants.WORK_PATH_DEFAULT);
this.workPath.mkdirs(); this.workPath.mkdirs();
// if default work files exist, copy them (don't overwrite existing!) // if default work files exist, copy them (don't overwrite existing!)
File defaultWorkPath = new File("defaults/data/work"); File defaultWorkPath = new File(appPath, "defaults/data/work");
if (defaultWorkPath.list() != null) { if (defaultWorkPath.list() != null) {
for (String fs : defaultWorkPath.list()) { for (String fs : defaultWorkPath.list()) {
File wf = new File(this.workPath, fs); File wf = new File(this.workPath, fs);
@ -730,7 +730,7 @@ public final class Switchboard extends serverSwitch {
// copy opensearch heuristic config (if not exist) // copy opensearch heuristic config (if not exist)
final File osdConfig = new File(getDataPath(), "DATA/SETTINGS/heuristicopensearch.conf"); final File osdConfig = new File(getDataPath(), "DATA/SETTINGS/heuristicopensearch.conf");
if (!osdConfig.exists()) { if (!osdConfig.exists()) {
final File osdDefaultConfig = new File("defaults/heuristicopensearch.conf"); final File osdDefaultConfig = new File(appPath, "defaults/heuristicopensearch.conf");
this.log.info("heuristic.opensearch list Path = " + osdDefaultConfig.getAbsolutePath()); this.log.info("heuristic.opensearch list Path = " + osdDefaultConfig.getAbsolutePath());
try { try {
Files.copy(osdDefaultConfig, osdConfig); Files.copy(osdDefaultConfig, osdConfig);
@ -781,7 +781,7 @@ public final class Switchboard extends serverSwitch {
} }
// init html parser evaluation scheme // init html parser evaluation scheme
File parserPropertiesPath = new File("defaults/"); File parserPropertiesPath = new File(appPath, "defaults/");
String[] settingsList = parserPropertiesPath.list(); String[] settingsList = parserPropertiesPath.list();
for ( final String l : settingsList ) { for ( final String l : settingsList ) {
if ( l.startsWith("parser.") && l.endsWith(".properties") ) { if ( l.startsWith("parser.") && l.endsWith(".properties") ) {
@ -865,7 +865,7 @@ public final class Switchboard extends serverSwitch {
// load oai tables // load oai tables
final Map<String, File> oaiFriends = final Map<String, File> oaiFriends =
OAIListFriendsLoader.loadListFriendsSources( OAIListFriendsLoader.loadListFriendsSources(
new File("defaults/oaiListFriendsSource.xml"), new File(appPath, "defaults/oaiListFriendsSource.xml"),
getDataPath()); getDataPath());
OAIListFriendsLoader.init(this.loader, oaiFriends, ClientIdentification.yacyInternetCrawlerAgent); OAIListFriendsLoader.init(this.loader, oaiFriends, ClientIdentification.yacyInternetCrawlerAgent);
this.crawlQueues = new CrawlQueues(this, this.queuesRoot); this.crawlQueues = new CrawlQueues(this, this.queuesRoot);
@ -2413,7 +2413,7 @@ public final class Switchboard extends serverSwitch {
final Properties profile = new Properties(); final Properties profile = new Properties();
FileInputStream fileIn = null; FileInputStream fileIn = null;
try { try {
fileIn = new FileInputStream(new File("DATA/SETTINGS/profile.txt")); fileIn = new FileInputStream(new File(this.dataPath, "DATA/SETTINGS/profile.txt"));
profile.load(fileIn); profile.load(fileIn);
} catch (final IOException e ) { } catch (final IOException e ) {
} finally { } finally {

69
source/net/yacy/search/index/Fulltext.java
Unescape View File

@ -618,13 +618,17 @@ public final class Fulltext {
} }
} }
public static enum ExportFormat {
text, html, rss, solr;
}
// export methods // export methods
public Export export(final File f, final String filter, final String query, final int format, final boolean dom) { public Export export(final File f, final String filter, final String query, final ExportFormat format, final boolean dom, final boolean text) {
if ((this.exportthread != null) && (this.exportthread.isAlive())) { if ((this.exportthread != null) && (this.exportthread.isAlive())) {
ConcurrentLog.warn("LURL-EXPORT", "cannot start another export thread, already one running"); ConcurrentLog.warn("LURL-EXPORT", "cannot start another export thread, already one running");
return this.exportthread; return this.exportthread;
} }
this.exportthread = new Export(f, filter, query, format, dom); this.exportthread = new Export(f, filter, query, format, dom, text);
this.exportthread.start(); this.exportthread.start();
return this.exportthread; return this.exportthread;
} }
@ -638,10 +642,10 @@ public final class Fulltext {
private final Pattern pattern; private final Pattern pattern;
private int count; private int count;
private String failure, query; private String failure, query;
private final int format; private final ExportFormat format;
private final boolean dom; private final boolean dom, text;
private Export(final File f, final String filter, final String query, final int format, boolean dom) { private Export(final File f, final String filter, final String query, final ExportFormat format, final boolean dom, final boolean text) {
// format: 0=text, 1=html, 2=rss/xml // format: 0=text, 1=html, 2=rss/xml
this.f = f; this.f = f;
this.pattern = filter == null ? null : Pattern.compile(filter); this.pattern = filter == null ? null : Pattern.compile(filter);
@ -650,6 +654,7 @@ public final class Fulltext {
this.failure = null; this.failure = null;
this.format = format; this.format = format;
this.dom = dom; this.dom = dom;
this.text = text;
//if ((dom) && (format == 2)) dom = false; //if ((dom) && (format == 2)) dom = false;
} }
@ -658,13 +663,13 @@ public final class Fulltext {
try { try {
final File parentf = this.f.getParentFile(); final File parentf = this.f.getParentFile();
if (parentf != null) parentf.mkdirs(); if (parentf != null) parentf.mkdirs();
OutputStream os = new FileOutputStream(this.format == 3 ? new File(this.f.getAbsolutePath() + ".gz") : this.f); OutputStream os = new FileOutputStream(this.format == ExportFormat.solr ? new File(this.f.getAbsolutePath() + ".gz") : this.f);
if (this.format == 3) os = new GZIPOutputStream(os, 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}}; if (this.format == ExportFormat.solr) os = new GZIPOutputStream(os, 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}};
final PrintWriter pw = new PrintWriter(new BufferedOutputStream(os)); final PrintWriter pw = new PrintWriter(new BufferedOutputStream(os));
if (this.format == 1) { if (this.format == ExportFormat.html) {
pw.println("<html><head></head><body>"); pw.println("<html><head></head><body>");
} }
if (this.format == 2) { if (this.format == ExportFormat.rss) {
pw.println("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"); pw.println("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
pw.println("<?xml-stylesheet type='text/xsl' href='/yacysearch.xsl' version='1.0'?>"); pw.println("<?xml-stylesheet type='text/xsl' href='/yacysearch.xsl' version='1.0'?>");
pw.println("<rss version=\"2.0\" xmlns:yacy=\"http://www.yacy.net/\" xmlns:opensearch=\"http://a9.com/-/spec/opensearch/1.1/\" xmlns:atom=\"http://www.w3.org/2005/Atom\">"); pw.println("<rss version=\"2.0\" xmlns:yacy=\"http://www.yacy.net/\" xmlns:opensearch=\"http://a9.com/-/spec/opensearch/1.1/\" xmlns:atom=\"http://www.w3.org/2005/Atom\">");
@ -673,7 +678,7 @@ public final class Fulltext {
pw.println("<description></description>"); pw.println("<description></description>");
pw.println("<link>http://yacy.net</link>"); pw.println("<link>http://yacy.net</link>");
} }
if (this.format == 3) { if (this.format == ExportFormat.solr) {
pw.println("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"); pw.println("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
pw.println("<response>"); pw.println("<response>");
pw.println("<result>"); pw.println("<result>");
@ -683,12 +688,25 @@ public final class Fulltext {
ReversibleScoreMap<String> stats = scores.get(CollectionSchema.host_s.getSolrFieldName()); ReversibleScoreMap<String> stats = scores.get(CollectionSchema.host_s.getSolrFieldName());
for (final String host: stats) { for (final String host: stats) {
if (this.pattern != null && !this.pattern.matcher(host).matches()) continue; if (this.pattern != null && !this.pattern.matcher(host).matches()) continue;
if (this.format == 0) pw.println(host); if (this.format == ExportFormat.text) pw.println(host);
if (this.format == 1) pw.println("<a href=\"http://" + host + "\">" + host + "</a><br>"); if (this.format == ExportFormat.html) pw.println("<a href=\"http://" + host + "\">" + host + "</a><br>");
this.count++; this.count++;
} }
} else { } else {
if (this.format < 3) { if (this.format == ExportFormat.solr || (this.text && this.format == ExportFormat.text)) {
BlockingQueue<SolrDocument> docs = Fulltext.this.getDefaultConnector().concurrentDocumentsByQuery(this.query + " AND " + CollectionSchema.httpstatus_i.getSolrFieldName() + ":200", null, 0, 100000000, Long.MAX_VALUE, 100, 1, true);
SolrDocument doc;
while ((doc = docs.take()) != AbstractSolrConnector.POISON_DOCUMENT) {
String url = getStringFrom(doc.getFieldValue(CollectionSchema.sku.getSolrFieldName()));
if (this.pattern != null && !this.pattern.matcher(url).matches()) continue;
CRIgnoreWriter sw = new CRIgnoreWriter();
if (this.text) sw.write((String) doc.getFieldValue(CollectionSchema.text_t.getSolrFieldName())); else EnhancedXMLResponseWriter.writeDoc(sw, doc);
sw.close();
String d = sw.toString();
pw.println(d);
this.count++;
}
} else {
BlockingQueue<SolrDocument> docs = Fulltext.this.getDefaultConnector().concurrentDocumentsByQuery(this.query + " AND " + CollectionSchema.httpstatus_i.getSolrFieldName() + ":200", null, 0, 100000000, Long.MAX_VALUE, 100, 1, true, BlockingQueue<SolrDocument> docs = Fulltext.this.getDefaultConnector().concurrentDocumentsByQuery(this.query + " AND " + CollectionSchema.httpstatus_i.getSolrFieldName() + ":200", null, 0, 100000000, Long.MAX_VALUE, 100, 1, true,
CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName(), CollectionSchema.title.getSolrFieldName(), CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName(), CollectionSchema.title.getSolrFieldName(),
CollectionSchema.author.getSolrFieldName(), CollectionSchema.description_txt.getSolrFieldName(), CollectionSchema.size_i.getSolrFieldName(), CollectionSchema.last_modified.getSolrFieldName()); CollectionSchema.author.getSolrFieldName(), CollectionSchema.description_txt.getSolrFieldName(), CollectionSchema.size_i.getSolrFieldName(), CollectionSchema.last_modified.getSolrFieldName());
@ -705,13 +723,13 @@ public final class Fulltext {
size = (Integer) doc.getFieldValue(CollectionSchema.size_i.getSolrFieldName()); size = (Integer) doc.getFieldValue(CollectionSchema.size_i.getSolrFieldName());
date = (Date) doc.getFieldValue(CollectionSchema.last_modified.getSolrFieldName()); date = (Date) doc.getFieldValue(CollectionSchema.last_modified.getSolrFieldName());
if (this.pattern != null && !this.pattern.matcher(url).matches()) continue; if (this.pattern != null && !this.pattern.matcher(url).matches()) continue;
if (this.format == 0) { if (this.format == ExportFormat.text) {
pw.println(url); pw.println(url);
} }
if (this.format == 1) { if (this.format == ExportFormat.html) {
if (title != null) pw.println("<a href=\"" + MultiProtocolURL.escape(url) + "\">" + CharacterCoding.unicode2xml(title, true) + "</a>"); if (title != null) pw.println("<a href=\"" + MultiProtocolURL.escape(url) + "\">" + CharacterCoding.unicode2xml(title, true) + "</a>");
} }
if (this.format == 2) { if (this.format == ExportFormat.rss) {
pw.println("<item>"); pw.println("<item>");
if (title != null) pw.println("<title>" + CharacterCoding.unicode2xml(title, true) + "</title>"); if (title != null) pw.println("<title>" + CharacterCoding.unicode2xml(title, true) + "</title>");
pw.println("<link>" + MultiProtocolURL.escape(url) + "</link>"); pw.println("<link>" + MultiProtocolURL.escape(url) + "</link>");
@ -724,29 +742,16 @@ public final class Fulltext {
} }
this.count++; this.count++;
} }
} else {
BlockingQueue<SolrDocument> docs = Fulltext.this.getDefaultConnector().concurrentDocumentsByQuery(this.query + " AND " + CollectionSchema.httpstatus_i.getSolrFieldName() + ":200", null, 0, 100000000, Long.MAX_VALUE, 100, 1, true);
SolrDocument doc;
while ((doc = docs.take()) != AbstractSolrConnector.POISON_DOCUMENT) {
String url = getStringFrom(doc.getFieldValue(CollectionSchema.sku.getSolrFieldName()));
if (this.pattern != null && !this.pattern.matcher(url).matches()) continue;
CRIgnoreWriter sw = new CRIgnoreWriter();
EnhancedXMLResponseWriter.writeDoc(sw, doc);
sw.close();
String d = sw.toString();
pw.println(d);
this.count++;
}
} }
} }
if (this.format == 1) { if (this.format == ExportFormat.html) {
pw.println("</body></html>"); pw.println("</body></html>");
} }
if (this.format == 2) { if (this.format == ExportFormat.rss) {
pw.println("</channel>"); pw.println("</channel>");
pw.println("</rss>"); pw.println("</rss>");
} }
if (this.format == 3) { if (this.format == ExportFormat.solr) {
pw.println("</result>"); pw.println("</result>");
pw.println("</response>"); pw.println("</response>");
} }

2
source/net/yacy/yacy.java
Unescape View File

@ -598,8 +598,6 @@ public final class yacy {
public static void main(String args[]) { public static void main(String args[]) {
try { try {
System.setProperty("jsse.enableSNIExtension", "false"); // fix a ssl problem in Java 1.7, see http://teknosrc.com/javax-net-ssl-sslprotocolexception-handshake-alert-unrecognized_name-solved/
// check assertion status // check assertion status
//ClassLoader.getSystemClassLoader().setDefaultAssertionStatus(true); //ClassLoader.getSystemClassLoader().setDefaultAssertionStatus(true);
boolean assertionenabled = false; boolean assertionenabled = false;

4
startYACY.bat
Unescape View File

@ -18,7 +18,9 @@ if exist DATA\SETTINGS\httpProxy.conf GoTo :RENAMEINDEX
if exist DATA\SETTINGS\yacy.conf GoTo :GETSTARTOPTS if exist DATA\SETTINGS\yacy.conf GoTo :GETSTARTOPTS
:STARTJAVA :STARTJAVA
set javacmd=%javacmd% -Djava.awt.headless=true -Dsolr.directoryFactory=solr.MMapDirectoryFactory -Dfile.encoding=UTF-8 set javacmd=%javacmd% -Djava.awt.headless=true -Dsolr.directoryFactory=solr.MMapDirectoryFactory -Dfile.encoding=UTF-8 -Djsse.enableSNIExtension=false
Rem -Djsse.enableSNIExtension=false fix a ssl problem in Java 1.7, see http://teknosrc.com/javax-net-ssl-sslprotocolexception-handshake-alert-unrecognized_name-solved/
Rem Starting YaCy Rem Starting YaCy
Echo Generated classpath:%CLASSPATH% Echo Generated classpath:%CLASSPATH%
Echo JRE Parameters:%javacmd% Echo JRE Parameters:%javacmd%

4
startYACY.sh
Unescape View File

@ -6,7 +6,9 @@ PIDFILE="yacy.pid"
OS="`uname`" OS="`uname`"
#get javastart args #get javastart args
JAVA_ARGS="-server -Djava.awt.headless=true -Dfile.encoding=UTF-8"; JAVA_ARGS="-server -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djsse.enableSNIExtension=false";
#rem -Djsse.enableSNIExtension=false fix a ssl problem in Java 1.7, see http://teknosrc.com/javax-net-ssl-sslprotocolexception-handshake-alert-unrecognized_name-solved/
#JAVA_ARGS="-verbose:gc -XX:+PrintGCTimeStamps -XX:+PrintGCDetails $JAVA_ARGS"; #JAVA_ARGS="-verbose:gc -XX:+PrintGCTimeStamps -XX:+PrintGCDetails $JAVA_ARGS";
#check if OS is Sun Solaris or one of the OpenSolaris distributions and use different version of id if necessary #check if OS is Sun Solaris or one of the OpenSolaris distributions and use different version of id if necessary

4
startYACY_debug.bat
Unescape View File

@ -17,7 +17,9 @@ if exist DATA\SETTINGS\httpProxy.conf GoTo :RENAMEINDEX
if exist DATA\SETTINGS\yacy.conf GoTo :GETSTARTOPTS if exist DATA\SETTINGS\yacy.conf GoTo :GETSTARTOPTS
:STARTJAVA :STARTJAVA
set javacmd=%javacmd% -XX:-UseGCOverheadLimit -Djava.awt.headless=true -Dfile.encoding=UTF-8 set javacmd=%javacmd% -XX:-UseGCOverheadLimit -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djsse.enableSNIExtension=false
Rem -Djsse.enableSNIExtension=false fix a ssl problem in Java 1.7, see http://teknosrc.com/javax-net-ssl-sslprotocolexception-handshake-alert-unrecognized_name-solved/
Rem Starting YaCy Rem Starting YaCy
Echo Generated classpath:%CLASSPATH% Echo Generated classpath:%CLASSPATH%
Echo JRE Parameters:%javacmd% Echo JRE Parameters:%javacmd%

Write Preview
Loading…
Cancel
Save