boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of git@github.com:yacy/yacy_search_server.git

Browse Source
pull/9/merge
Michael Peter Christen 10 years ago
parent b94bd7f20a 5ba9924289
commit 11a848da5a
61 changed files with 289 additions and 327 deletions
Show Stats Download Patch File Download Diff File

60
.classpath
Unescape View File

@ -16,7 +16,7 @@
<classpathentry kind="lib" path="lib/J7Zip-modified.jar"/>
<classpathentry kind="lib" path="lib/webcat-0.1-swf.jar"/>
<classpathentry kind="lib" path="lib/commons-jxpath-1.3.jar"/>
<classpathentry kind="lib" path="lib/jsch-0.1.52.jar"/>
<classpathentry kind="lib" path="lib/jsch-0.1.53.jar"/>
<classpathentry kind="lib" path="lib/jakarta-oro-2.0.8.jar"/>
<classpathentry kind="lib" path="lib/apache-mime4j-0.6.jar"/>
<classpathentry kind="lib" path="lib/commons-fileupload-1.3.1.jar"/>
@ -28,10 +28,10 @@
<classpathentry kind="lib" path="lib/commons-lang-2.6.jar"/>
<classpathentry kind="lib" path="lib/jaudiotagger-2.0.4-20111207.115108-15.jar"/>
<classpathentry kind="lib" path="lib/commons-codec-1.10.jar"/>
<classpathentry kind="lib" path="lib/jcl-over-slf4j-1.7.9.jar"/>
<classpathentry kind="lib" path="lib/jcl-over-slf4j-1.7.12.jar"/>
<classpathentry kind="lib" path="lib/wstx-asl-3.2.9.jar"/>
<classpathentry kind="lib" path="lib/slf4j-jdk14-1.7.9.jar"/>
<classpathentry kind="lib" path="lib/log4j-over-slf4j-1.7.9.jar"/>
<classpathentry kind="lib" path="lib/slf4j-jdk14-1.7.12.jar"/>
<classpathentry kind="lib" path="lib/log4j-over-slf4j-1.7.12.jar"/>
<classpathentry kind="lib" path="lib/httpcore-4.4.1.jar"/>
<classpathentry kind="lib" path="lib/metadata-extractor-2.8.1.jar"/>
<classpathentry kind="lib" path="lib/xmpcore-5.1.2.jar"/>
@ -40,15 +40,15 @@
<classpathentry kind="lib" path="lib/spatial4j-0.4.1.jar"/>
<classpathentry kind="lib" path="lib/zookeeper-3.4.6.jar"/>
<classpathentry kind="lib" path="lib/org.restlet.jar"/>
<classpathentry kind="lib" path="lib/fontbox-1.8.9.jar"/>
<classpathentry kind="lib" path="lib/jempbox-1.8.9.jar"/>
<classpathentry kind="lib" path="lib/pdfbox-1.8.9.jar"/>
<classpathentry kind="lib" path="lib/fontbox-1.8.10.jar"/>
<classpathentry kind="lib" path="lib/jempbox-1.8.10.jar"/>
<classpathentry kind="lib" path="lib/pdfbox-1.8.10.jar"/>
<classpathentry kind="lib" path="lib/bcmail-jdk15-1.46.jar"/>
<classpathentry kind="lib" path="lib/bcprov-jdk15-1.46.jar"/>
<classpathentry kind="lib" path="lib/poi-3.12-20150511.jar"/>
<classpathentry kind="lib" path="lib/poi-scratchpad-3.12-20150511.jar"/>
<classpathentry kind="lib" path="lib/commons-io-2.4.jar"/>
<classpathentry kind="lib" path="lib/slf4j-api-1.7.9.jar"/>
<classpathentry kind="lib" path="lib/slf4j-api-1.7.12.jar"/>
<classpathentry kind="lib" path="lib/chardet.jar"/>
<classpathentry kind="lib" path="lib/jetty-client-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-continuation-9.2.11.v20150529.jar"/>
@ -64,34 +64,34 @@
<classpathentry kind="lib" path="lib/jetty-util-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-webapp-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/jetty-xml-9.2.11.v20150529.jar"/>
<classpathentry kind="lib" path="lib/httpclient-4.4.1.jar"/>
<classpathentry kind="lib" path="lib/httpmime-4.4.1.jar"/>
<classpathentry kind="lib" path="lib/httpclient-4.5.jar"/>
<classpathentry kind="lib" path="lib/httpmime-4.5.jar"/>
<classpathentry kind="lib" path="lib/noggit-0.6.jar"/>
<classpathentry kind="lib" path="lib/solr-core-5.2.0.jar" sourcepath="/Volumes/Uno/noBackup/Downloads/solr-5.2.0/solr/core/src/java"/>
<classpathentry kind="lib" path="lib/solr-solrj-5.2.0.jar" sourcepath="/Users/admin/Downloads/solr-5.2.1/solr/solrj/src/java"/>
<classpathentry kind="lib" path="lib/solr-core-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/solr-solrj-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/stax2-api-3.1.4.jar"/>
<classpathentry kind="lib" path="lib/woodstox-core-asl-4.4.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-analyzers-common-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-analyzers-phonetic-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-classification-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-codecs-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-core-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-facet-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-grouping-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-highlighter-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-join-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-memory-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-misc-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-queries-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-queryparser-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-spatial-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-suggest-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-backward-codecs-5.2.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-analyzers-common-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-analyzers-phonetic-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-classification-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-codecs-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-core-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-facet-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-grouping-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-highlighter-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-join-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-memory-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-misc-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-queries-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-queryparser-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-spatial-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-suggest-5.2.1.jar"/>
<classpathentry kind="lib" path="lib/lucene-backward-codecs-5.2.1.jar"/>
<classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/>
<classpathentry kind="lib" path="lib/icu4j-core.jar"/>
<classpathentry kind="lib" path="lib/icu4j-55_1.jar"/>
<classpathentry kind="lib" path="lib/htmllexer.jar"/>
<classpathentry kind="lib" path="lib/jsoup-1.8.2.jar"/>
<classpathentry kind="lib" path="lib/javax.servlet-api-3.1.0.jar"/>
<classpathentry kind="lib" path="lib/weupnp-0.1.2.jar"/>
<classpathentry kind="lib" path="lib/weupnp-0.1.3.jar"/>
<classpathentry kind="output" path="gen"/>
</classpath>

60
build.xml
Unescape View File

@ -172,21 +172,21 @@
<pathelement location="${lib}/commons-jxpath-1.3.jar" />
<pathelement location="${lib}/commons-lang-2.6.jar" />
<pathelement location="${lib}/commons-logging-1.2.jar" />
<pathelement location="${lib}/fontbox-1.8.9.jar" />
<pathelement location="${lib}/fontbox-1.8.10.jar" />
<pathelement location="${lib}/geronimo-stax-api_1.0_spec-1.0.1.jar" />
<pathelement location="${lib}/guava-18.0.jar" />
<pathelement location="${lib}/htmllexer.jar" />
<pathelement location="${lib}/httpclient-4.4.1.jar" />
<pathelement location="${lib}/httpclient-4.5.jar" />
<pathelement location="${lib}/httpcore-4.4.1.jar" />
<pathelement location="${lib}/httpmime-4.4.1.jar" />
<pathelement location="${lib}/icu4j-core.jar" />
<pathelement location="${lib}/httpmime-4.5.jar" />
<pathelement location="${lib}/icu4j-55_1.jar" />
<pathelement location="${lib}/J7Zip-modified.jar" />
<pathelement location="${lib}/jakarta-oro-2.0.8.jar" />
<pathelement location="${lib}/jaudiotagger-2.0.4-20111207.115108-15.jar" />
<pathelement location="${lib}/javax.servlet-api-3.1.0.jar" />
<pathelement location="${lib}/jcifs-1.3.17.jar" />
<pathelement location="${lib}/jcl-over-slf4j-1.7.9.jar" />
<pathelement location="${lib}/jempbox-1.8.9.jar" />
<pathelement location="${lib}/jcl-over-slf4j-1.7.12.jar" />
<pathelement location="${lib}/jempbox-1.8.10.jar" />
<pathelement location="${lib}/jetty-client-9.2.11.v20150529.jar" />
<pathelement location="${lib}/jetty-continuation-9.2.11.v20150529.jar" />
<pathelement location="${lib}/jetty-deploy-9.2.11.v20150529.jar" />
@ -201,39 +201,39 @@
<pathelement location="${lib}/jetty-util-9.2.11.v20150529.jar" />
<pathelement location="${lib}/jetty-webapp-9.2.11.v20150529.jar" />
<pathelement location="${lib}/jetty-xml-9.2.11.v20150529.jar" />
<pathelement location="${lib}/jsch-0.1.52.jar" />
<pathelement location="${lib}/jsch-0.1.53.jar" />
<pathelement location="${lib}/json-simple-1.1.1.jar" />
<pathelement location="${lib}/jsoup-1.8.2.jar" />
<pathelement location="${lib}/log4j-over-slf4j-1.7.9.jar" />
<pathelement location="${lib}/lucene-analyzers-common-5.2.0.jar" />
<pathelement location="${lib}/lucene-analyzers-phonetic-5.2.0.jar" />
<pathelement location="${lib}/lucene-backward-codecs-5.2.0.jar" />
<pathelement location="${lib}/lucene-classification-5.2.0.jar" />
<pathelement location="${lib}/lucene-codecs-5.2.0.jar" />
<pathelement location="${lib}/lucene-core-5.2.0.jar" />
<pathelement location="${lib}/lucene-facet-5.2.0.jar" />
<pathelement location="${lib}/lucene-grouping-5.2.0.jar" />
<pathelement location="${lib}/lucene-highlighter-5.2.0.jar" />
<pathelement location="${lib}/lucene-join-5.2.0.jar" />
<pathelement location="${lib}/lucene-memory-5.2.0.jar" />
<pathelement location="${lib}/lucene-misc-5.2.0.jar" />
<pathelement location="${lib}/lucene-queries-5.2.0.jar" />
<pathelement location="${lib}/lucene-queryparser-5.2.0.jar" />
<pathelement location="${lib}/lucene-spatial-5.2.0.jar" />
<pathelement location="${lib}/lucene-suggest-5.2.0.jar" />
<pathelement location="${lib}/log4j-over-slf4j-1.7.12.jar" />
<pathelement location="${lib}/lucene-analyzers-common-5.2.1.jar" />
<pathelement location="${lib}/lucene-analyzers-phonetic-5.2.1.jar" />
<pathelement location="${lib}/lucene-backward-codecs-5.2.1.jar" />
<pathelement location="${lib}/lucene-classification-5.2.1.jar" />
<pathelement location="${lib}/lucene-codecs-5.2.1.jar" />
<pathelement location="${lib}/lucene-core-5.2.1.jar" />
<pathelement location="${lib}/lucene-facet-5.2.1.jar" />
<pathelement location="${lib}/lucene-grouping-5.2.1.jar" />
<pathelement location="${lib}/lucene-highlighter-5.2.1.jar" />
<pathelement location="${lib}/lucene-join-5.2.1.jar" />
<pathelement location="${lib}/lucene-memory-5.2.1.jar" />
<pathelement location="${lib}/lucene-misc-5.2.1.jar" />
<pathelement location="${lib}/lucene-queries-5.2.1.jar" />
<pathelement location="${lib}/lucene-queryparser-5.2.1.jar" />
<pathelement location="${lib}/lucene-spatial-5.2.1.jar" />
<pathelement location="${lib}/lucene-suggest-5.2.1.jar" />
<pathelement location="${lib}/metadata-extractor-2.8.1.jar" />
<pathelement location="${lib}/noggit-0.5.jar" />
<pathelement location="${lib}/org.restlet.jar" />
<pathelement location="${lib}/pdfbox-1.8.9.jar" />
<pathelement location="${lib}/pdfbox-1.8.10.jar" />
<pathelement location="${lib}/poi-3.12-20150511.jar" />
<pathelement location="${lib}/poi-scratchpad-3.12-20150511.jar" />
<pathelement location="${lib}/slf4j-api-1.7.9.jar" />
<pathelement location="${lib}/slf4j-jdk14-1.7.9.jar" />
<pathelement location="${lib}/solr-core-5.2.0.jar" />
<pathelement location="${lib}/solr-solrj-5.2.0.jar" />
<pathelement location="${lib}/slf4j-api-1.7.12.jar" />
<pathelement location="${lib}/slf4j-jdk14-1.7.12.jar" />
<pathelement location="${lib}/solr-core-5.2.1.jar" />
<pathelement location="${lib}/solr-solrj-5.2.1.jar" />
<pathelement location="${lib}/spatial4j-0.4.1.jar" />
<pathelement location="${lib}/webcat-0.1-swf.jar" />
<pathelement location="${lib}/weupnp-0.1.2.jar" />
<pathelement location="${lib}/weupnp-0.1.3.jar" />
<pathelement location="${lib}/wstx-asl-3.2.9.jar" />
<pathelement location="${lib}/xercesImpl.jar" />
<pathelement location="${lib}/xml-apis.jar" />

1
defaults/yacy.init
Unescape View File

@ -103,6 +103,7 @@ filesize.max.other = 8589934591
# In case of privately managed networks, this configuration must be changed BEFORE it is released
# to the members of the separated network peers.
network.unit.definition = defaults/yacy.network.freeworld.unit
#network.unit.definition = defaults/yacy.network.webportal.unit
#network.unit.definition = defaults/yacy.network.intranet.unit
# distinguish intranet/internet IPs:

8
htroot/IndexExport_p.html
Unescape View File

@ -39,19 +39,21 @@
<dt>Only Domain:</dt>
<dd><input type="radio" name="format" value="dom-text" /> Plain Text List (domains only)<br />
<input type="radio" name="format" value="dom-html" /> HTML (domains as URLs, no title)</dd>
<dt>Only Text:</dt>
<dd><input type="radio" name="format" value="text-text" /> Fulltext of Search Index Text</dd>
</dl>
</dd>
<dt>&nbsp;</dt>
<dd><input type="submit" name="lurlexport" value="Export URLs" class="btn btn-primary" style="width:240px;"/>
<dd><input type="submit" name="lurlexport" value="Export" class="btn btn-primary" style="width:240px;"/>
</dd>
</dl>
</fieldset>
</form>::
<div class="alert alert-info" style="text-decoration:blink">Export to file #[exportfile]# is running .. #[urlcount]# URLs so far</div>::
<div class="alert alert-info" style="text-decoration:blink">Export to file #[exportfile]# is running .. #[urlcount]# Documents so far</div>::
#(/lurlexport)#
#(lurlexportfinished)#::
<div class="alert alert-success">Finished export of #[urlcount]# URLs to file <a href="file://#[exportfile]#" target="_">#[exportfile]#</a><br/>
<div class="alert alert-success">Finished export of #[urlcount]# Documents to file <a href="file://#[exportfile]#" target="_">#[exportfile]#</a><br/>
<em>Import this file by moving it to DATA/SURROGATES/in</em></div>::
#(/lurlexportfinished)#

21
htroot/IndexExport_p.java
Unescape View File

@ -93,27 +93,28 @@ public class IndexExport_p {
if (post.containsKey("lurlexport")) {
// parse format
int format = 0;
Fulltext.ExportFormat format = Fulltext.ExportFormat.text;
final String fname = post.get("format", "url-text");
final boolean dom = fname.startsWith("dom"); // if dom== false complete urls are exported, otherwise only the domain
if (fname.endsWith("text")) format = 0;
if (fname.endsWith("html")) format = 1;
if (fname.endsWith("rss")) format = 2;
if (fname.endsWith("solr")) format = 3;
final boolean text = fname.startsWith("text");
if (fname.endsWith("text")) format = Fulltext.ExportFormat.text;
if (fname.endsWith("html")) format = Fulltext.ExportFormat.html;
if (fname.endsWith("rss")) format = Fulltext.ExportFormat.rss;
if (fname.endsWith("solr")) format = Fulltext.ExportFormat.solr;
// extend export file name
String s = post.get("exportfile", "");
if (s.indexOf('.',0) < 0) {
if (format == 0) s = s + ".txt";
if (format == 1) s = s + ".html";
if (format == 2 ) s = s + "_rss.xml";
if (format == 3) s = s + "_full.xml";
if (format == Fulltext.ExportFormat.text) s = s + ".txt";
if (format == Fulltext.ExportFormat.html) s = s + ".html";
if (format == Fulltext.ExportFormat.rss ) s = s + "_rss.xml";
if (format == Fulltext.ExportFormat.solr) s = s + "_full.xml";
}
final File f = new File(s);
f.getParentFile().mkdirs();
final String filter = post.get("exportfilter", ".*");
final String query = post.get("exportquery", "*:*");
final Fulltext.Export running = segment.fulltext().export(f, filter, query, format, dom);
final Fulltext.Export running = segment.fulltext().export(f, filter, query, format, dom, text);
prop.put("lurlexport_exportfile", s);
prop.put("lurlexport_urlcount", running.count());

0
lib/fontbox-1.8.9.License → lib/fontbox-1.8.10.License
Unescape View File

BIN
lib/fontbox-1.8.9.jar → lib/fontbox-1.8.10.jar
View File

Binary file not shown.

0
lib/httpclient-4.4.1.License → lib/httpclient-4.5.License
Unescape View File

BIN
lib/httpclient-4.4.1.jar → lib/httpclient-4.5.jar
View File

Binary file not shown.

0
lib/httpmime-4.4.1.License → lib/httpmime-4.5.License
Unescape View File

BIN
lib/httpmime-4.4.1.jar → lib/httpmime-4.5.jar
View File

Binary file not shown.

BIN
lib/icu4j-55_1.jar
View File

Binary file not shown.

BIN
lib/icu4j-core.jar
View File

Binary file not shown.

BIN
lib/jcl-over-slf4j-1.7.12.jar
View File

Binary file not shown.

BIN
lib/jcl-over-slf4j-1.7.9.jar
View File

Binary file not shown.

0
lib/jempbox-1.8.9.License → lib/jempbox-1.8.10.License
Unescape View File

BIN
lib/jempbox-1.8.9.jar → lib/jempbox-1.8.10.jar
View File

Binary file not shown.

BIN
lib/jsch-0.1.52.jar
View File

Binary file not shown.

2
lib/jsch-0.1.52.License → lib/jsch-0.1.53.License
Unescape View File

@ -2,7 +2,7 @@ JSch 0.0.* was released under the GNU LGPL license. Later, we have switched
over to a BSD-style license.
------------------------------------------------------------------------------
Copyright (c) 2002,2003,2004 Atsuhiko Yamanaka, JCraft,Inc.
Copyright (c) 2002-2015 Atsuhiko Yamanaka, JCraft,Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without

BIN
lib/jsch-0.1.53.jar
View File

Binary file not shown.

BIN
lib/log4j-over-slf4j-1.7.12.jar
View File

Binary file not shown.

BIN
lib/log4j-over-slf4j-1.7.9.jar
View File

Binary file not shown.

BIN
lib/lucene-analyzers-common-5.2.0.jar → lib/lucene-analyzers-common-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-analyzers-phonetic-5.2.0.jar → lib/lucene-analyzers-phonetic-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-backward-codecs-5.2.0.jar → lib/lucene-backward-codecs-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-classification-5.2.0.jar → lib/lucene-classification-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-codecs-5.2.0.jar → lib/lucene-codecs-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-core-5.2.0.jar → lib/lucene-core-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-facet-5.2.0.jar → lib/lucene-facet-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-grouping-5.2.0.jar → lib/lucene-grouping-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-highlighter-5.2.0.jar → lib/lucene-highlighter-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-join-5.2.0.jar → lib/lucene-join-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-memory-5.2.0.jar → lib/lucene-memory-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-misc-5.2.0.jar → lib/lucene-misc-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-queries-5.2.0.jar → lib/lucene-queries-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-queryparser-5.2.0.jar → lib/lucene-queryparser-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-spatial-5.2.0.jar → lib/lucene-spatial-5.2.1.jar
View File

Binary file not shown.

BIN
lib/lucene-suggest-5.2.0.jar → lib/lucene-suggest-5.2.1.jar
View File

Binary file not shown.

0
lib/pdfbox-1.8.9.License → lib/pdfbox-1.8.10.License
Unescape View File

BIN
lib/pdfbox-1.8.9.jar → lib/pdfbox-1.8.10.jar
View File

Binary file not shown.

BIN
lib/slf4j-api-1.7.12.jar
View File

Binary file not shown.

BIN
lib/slf4j-api-1.7.9.jar
View File

Binary file not shown.

BIN
lib/slf4j-jdk14-1.7.12.jar
View File

Binary file not shown.

BIN
lib/slf4j-jdk14-1.7.9.jar
View File

Binary file not shown.

BIN
lib/solr-core-5.2.0.jar → lib/solr-core-5.2.1.jar
View File

Binary file not shown.

BIN
lib/solr-solrj-5.2.0.jar → lib/solr-solrj-5.2.1.jar
View File

Binary file not shown.

BIN
lib/weupnp-0.1.2.jar
View File

Binary file not shown.

BIN
lib/weupnp-0.1.3.jar
View File

Binary file not shown.

2
nbproject/project.xml
Unescape View File

@ -83,7 +83,7 @@
<compilation-unit>
<package-root>source</package-root>
<package-root>htroot</package-root>
<classpath mode="compile">lib/J7Zip-modified.jar;lib/apache-mime4j-0.6.jar;lib/bcmail-jdk15-1.46.jar;lib/bcprov-jdk15-1.46.jar;lib/chardet.jar;lib/commons-codec-1.10.jar;lib/commons-compress-1.9.jar;lib/commons-fileupload-1.3.1.jar;lib/commons-io-2.4.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.2.jar;lib/fontbox-1.8.9.jar;lib/geronimo-stax-api_1.0_spec-1.0.1.jar;lib/guava-18.0.jar;lib/htmllexer.jar;lib/httpclient-4.4.1.jar;lib/httpcore-4.4.1.jar;lib/httpmime-4.4.1.jar;lib/icu4j-core.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/javax.servlet-api-3.1.0.jar;lib/jcifs-1.3.17.jar;lib/jcl-over-slf4j-1.7.9.jar;lib/jempbox-1.8.9.jar;lib/jetty-client-9.2.11.v20150529.jar;lib/jetty-continuation-9.2.11.v20150529.jar;lib/jetty-deploy-9.2.11.v20150529.jar;lib/jetty-http-9.2.11.v20150529.jar;lib/jetty-io-9.2.11.v20150529.jar;lib/jetty-jmx-9.2.11.v20150529.jar;lib/jetty-proxy-9.2.11.v20150529.jar;lib/jetty-security-9.2.11.v20150529.jar;lib/jetty-server-9.2.11.v20150529.jar;lib/jetty-servlet-9.2.11.v20150529.jar;lib/jetty-servlets-9.2.11.v20150529.jar;lib/jetty-util-9.2.11.v20150529.jar;lib/jetty-webapp-9.2.11.v20150529.jar;lib/jetty-xml-9.2.11.v20150529.jar;lib/jsch-0.1.52.jar;lib/json-simple-1.1.1.jar;lib/jsoup-1.8.2.jar;lib/log4j-over-slf4j-1.7.9.jar;lib/lucene-analyzers-common-5.2.0.jar;lib/lucene-analyzers-phonetic-5.2.0.jar;lib/lucene-backward-codecs-5.2.0.jar;lib/lucene-classification-5.2.0.jar;lib/lucene-codecs-5.2.0.jar;lib/lucene-core-5.2.0.jar;lib/lucene-facet-5.2.0.jar;lib/lucene-grouping-5.2.0.jar;lib/lucene-highlighter-5.2.0.jar;lib/lucene-join-5.2.0.jar;lib/lucene-memory-5.2.0.jar;lib/lucene-misc-5.2.0.jar;lib/lucene-queries-5.2.0.jar;lib/lucene-queryparser-5.2.0.jar;lib/lucene-spatial-5.2.0.jar;lib/lucene-suggest-5.2.0.jar;lib/metadata-extractor-2.8.1.jar;lib/noggit-0.6.jar;lib/org.restlet.jar;lib/pdfbox-1.8.9.jar;lib/poi-3.12-20150511.jar;lib/poi-scratchpad-3.12-20150511.jar;lib/slf4j-api-1.7.9.jar;lib/slf4j-jdk14-1.7.9.jar;lib/solr-core-5.2.0.jar;lib/solr-solrj-5.2.0.jar;lib/spatial4j-0.4.1.jar;lib/stax2-api-3.1.4.jar;lib/webcat-0.1-swf.jar;lib/weupnp-0.1.2.jar;lib/woodstox-core-asl-4.4.1.jar;lib/wstx-asl-3.2.9.jar;lib/xercesImpl.jar;lib/xml-apis.jar;lib/xmpcore-5.1.2.jar;lib/zookeeper-3.4.6.jar</classpath>
<classpath mode="compile">lib/J7Zip-modified.jar;lib/apache-mime4j-0.6.jar;lib/bcmail-jdk15-1.46.jar;lib/bcprov-jdk15-1.46.jar;lib/chardet.jar;lib/commons-codec-1.10.jar;lib/commons-compress-1.9.jar;lib/commons-fileupload-1.3.1.jar;lib/commons-io-2.4.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.2.jar;lib/fontbox-1.8.10.jar;lib/geronimo-stax-api_1.0_spec-1.0.1.jar;lib/guava-18.0.jar;lib/htmllexer.jar;lib/httpclient-4.5.jar;lib/httpcore-4.4.1.jar;lib/httpmime-4.5.jar;lib/icu4j-55_1.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/javax.servlet-api-3.1.0.jar;lib/jcifs-1.3.17.jar;lib/jcl-over-slf4j-1.7.12.jar;lib/jempbox-1.8.10.jar;lib/jetty-client-9.2.11.v20150529.jar;lib/jetty-continuation-9.2.11.v20150529.jar;lib/jetty-deploy-9.2.11.v20150529.jar;lib/jetty-http-9.2.11.v20150529.jar;lib/jetty-io-9.2.11.v20150529.jar;lib/jetty-jmx-9.2.11.v20150529.jar;lib/jetty-proxy-9.2.11.v20150529.jar;lib/jetty-security-9.2.11.v20150529.jar;lib/jetty-server-9.2.11.v20150529.jar;lib/jetty-servlet-9.2.11.v20150529.jar;lib/jetty-servlets-9.2.11.v20150529.jar;lib/jetty-util-9.2.11.v20150529.jar;lib/jetty-webapp-9.2.11.v20150529.jar;lib/jetty-xml-9.2.11.v20150529.jar;lib/jsch-0.1.53.jar;lib/json-simple-1.1.1.jar;lib/jsoup-1.8.2.jar;lib/log4j-over-slf4j-1.7.12.jar;lib/lucene-analyzers-common-5.2.1.jar;lib/lucene-analyzers-phonetic-5.2.1.jar;lib/lucene-backward-codecs-5.2.1.jar;lib/lucene-classification-5.2.1.jar;lib/lucene-codecs-5.2.1.jar;lib/lucene-core-5.2.1.jar;lib/lucene-facet-5.2.1.jar;lib/lucene-grouping-5.2.1.jar;lib/lucene-highlighter-5.2.1.jar;lib/lucene-join-5.2.1.jar;lib/lucene-memory-5.2.1.jar;lib/lucene-misc-5.2.1.jar;lib/lucene-queries-5.2.1.jar;lib/lucene-queryparser-5.2.1.jar;lib/lucene-spatial-5.2.1.jar;lib/lucene-suggest-5.2.1.jar;lib/metadata-extractor-2.8.1.jar;lib/noggit-0.6.jar;lib/org.restlet.jar;lib/pdfbox-1.8.10.jar;lib/poi-3.12-20150511.jar;lib/poi-scratchpad-3.12-20150511.jar;lib/slf4j-api-1.7.12.jar;lib/slf4j-jdk14-1.7.12.jar;lib/solr-core-5.2.1.jar;lib/solr-solrj-5.2.1.jar;lib/spatial4j-0.4.1.jar;lib/stax2-api-3.1.4.jar;lib/webcat-0.1-swf.jar;lib/weupnp-0.1.3.jar;lib/woodstox-core-asl-4.4.1.jar;lib/wstx-asl-3.2.9.jar;lib/xercesImpl.jar;lib/xml-apis.jar;lib/xmpcore-5.1.2.jar;lib/zookeeper-3.4.6.jar</classpath>
<built-to>lib/yacycore.jar</built-to>
<source-level>1.7</source-level>
</compilation-unit>

84
pom.xml
Unescape View File

@ -366,7 +366,7 @@
<dependency>
<groupId>org.apache.pdfbox</groupId>
<artifactId>fontbox</artifactId>
<version>1.8.9</version>
<version>1.8.10</version>
</dependency>
<dependency>
<groupId>org.apache.geronimo.specs</groupId>
@ -386,7 +386,7 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.4</version>
<version>4.5</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
@ -396,12 +396,12 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId>
<version>4.4</version>
<version>4.5</version>
</dependency>
<dependency>
<groupId>com.ibm.icu</groupId>
<artifactId>icu4j</artifactId>
<version>4.8.1.1</version>
<version>55.1</version>
</dependency>
<dependency>
<groupId>org</groupId>
@ -421,17 +421,17 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
<version>1.7.9</version>
<version>1.7.12</version>
</dependency>
<dependency>
<groupId>org.apache.pdfbox</groupId>
<artifactId>jempbox</artifactId>
<version>1.8.9</version>
<version>1.8.10</version>
</dependency>
<dependency>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
<version>0.1.52</version>
<version>0.1.53</version>
</dependency>
<dependency>
<groupId>com.googlecode.json-simple</groupId>
@ -443,66 +443,16 @@
<artifactId>jsoup</artifactId>
<version>1.8.2</version>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>log4j-over-slf4j</artifactId>
<version>1.7.9</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
<version>${solr.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-phonetic</artifactId>
<version>${solr.version}</version>
<version>1.7.12</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<version>${solr.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-grouping</artifactId>
<version>${solr.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-highlighter</artifactId>
<version>${solr.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-memory</artifactId>
<version>${solr.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-misc</artifactId>
<version>${solr.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queries</artifactId>
<version>${solr.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-spatial</artifactId>
<version>${solr.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-suggest</artifactId>
<version>${solr.version}</version>
</dependency>
<dependency>
<groupId>com.drewnoakes</groupId>
<artifactId>metadata-extractor</artifactId>
@ -516,7 +466,7 @@
<dependency>
<groupId>org.apache.pdfbox</groupId>
<artifactId>pdfbox</artifactId>
<version>1.8.9</version>
<version>1.8.10</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
@ -536,12 +486,12 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.9</version>
<version>1.7.12</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-jdk14</artifactId>
<version>1.7.9</version>
<version>1.7.12</version>
</dependency>
<dependency>
<groupId>org.apache.solr</groupId>
@ -552,16 +502,6 @@
<groupId>org.apache.solr</groupId>
<artifactId>solr-core</artifactId>
<version>${solr.version}</version>
<exclusions>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>javax.servlet</artifactId>
<groupId>org.eclipse.jetty.orbit</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
@ -631,7 +571,7 @@
<dependency>
<groupId>org.bitlet</groupId>
<artifactId>weupnp</artifactId>
<version>0.1.2</version>
<version>0.1.3</version>
</dependency>
<dependency>
<groupId>org.codehaus.woodstox</groupId>

12
source/net/yacy/cora/document/id/MultiProtocolURL.java
Unescape View File

@ -843,14 +843,12 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
final int q = fileName.lastIndexOf('?');
if (q < 0) {
return fileName.substring(p + 1).toLowerCase();
} else {
// check last dot in query part
if (p > q) {
return ""; // TODO: last . after ? (file.ext?param=one.txt)
} else {
return fileName.substring(p + 1, q).toLowerCase();
}
}
// check last dot in query part
if (p > q) {
return ""; // TODO: last . after ? (file.ext?param=one.txt)
}
return fileName.substring(p + 1, q).toLowerCase();
}
public String getPath() {

7
source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java
Unescape View File

@ -205,7 +205,8 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
String fq = req.getParams().get(CommonParams.FQ);
String sort = req.getParams().get(CommonParams.SORT);
String threadname = Thread.currentThread().getName();
if (q != null) Thread.currentThread().setName("solr query: q = " + q + (fq == null ? "" : ", fq = " + fq) + (sort == null ? "" : ", sort = " + sort)); // for debugging in Threaddump
if (q != null) Thread.currentThread().setName("solr query: q=" + q + (fq == null ? "" : "&fq = " + fq) + (sort == null ? "" : "&sort = " + sort)); // for debugging in Threaddump
ConcurrentLog.info("EmbeddedSolrConnector.query", "QUERY: q=" + q + (fq == null ? "" : "&" + fq.toString()) + (sort == null ? "" : "&sort = " + sort) + " SolrQueryRequest=" + req.toString());
SolrQueryResponse rsp = new SolrQueryResponse();
NamedList<Object> responseHeader = new SimpleOrderedMap<Object>();
@ -334,8 +335,8 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
if (q != null) {
StringBuilder fqa = new StringBuilder();
if (fq != null) for (String f: fq) fqa.append("fq=").append(f).append(' ');
Thread.currentThread().setName("solr query: q = " + q + (fq == null ? "" : ", " + fqa.toString()) + (fl == null ? "" : ", fl=" + fl));
//System.out.println("solr query: q = " + q + (fq == null ? "" : ", " + fqa.toString()) + (fl == null ? "" : ", fl=" + fl));
Thread.currentThread().setName("solr query: q=" + q + (fq == null ? "" : "&" + fqa.toString()) + (fl == null ? "" : "&fl=" + fl));
ConcurrentLog.info("EmbeddedSolrConnector.getResponseByParams", "QUERY: q=" + q + (fq == null ? "" : "&" + fqa.toString()) + (fl == null ? "" : "&fl=" + fl) + " PARAMS: " + params.toString());
}
QueryResponse rsp;
try {

22
source/net/yacy/crawler/data/Cache.java
Unescape View File

@ -86,6 +86,15 @@ public final class Cache {
responseHeaderDB = new MapHeap(dbfile, Word.commonHashLength, Base64Order.enhancedCoder, 2048, 100, ' ');
} catch (final IOException e) {
ConcurrentLog.logException(e);
// try a healing
if (dbfile.exists()) {
dbfile.delete();
try {
responseHeaderDB = new MapHeap(dbfile, Word.commonHashLength, Base64Order.enhancedCoder, 2048, 100, ' ');
} catch (final IOException ee) {
ConcurrentLog.logException(e);
}
}
}
// open the cache file
try {
@ -94,8 +103,19 @@ public final class Cache {
fileDB = new Compressor(fileDBunbuffered, 6 * 1024 * 1024);
} catch (final IOException e) {
ConcurrentLog.logException(e);
// try a healing
if (cachePath.exists()) {
cachePath.delete();
try {
fileDBunbuffered = new ArrayStack(new File(cachePath, FILE_DB_NAME), prefix, Base64Order.enhancedCoder, 12, 1024 * 1024 * 2, false, true);
fileDBunbuffered.setMaxSize(maxCacheSize);
fileDB = new Compressor(fileDBunbuffered, 6 * 1024 * 1024);
} catch (final IOException ee) {
ConcurrentLog.logException(e);
}
}
}
ConcurrentLog.info("Cache", "initialized cache database responseHeaderDB.size() = " + responseHeaderDB.size() + ", fileDB.size() = " + fileDB.size());
ConcurrentLog.info("Cache", "initialized cache database responseHeaderDB.size() = " + (responseHeaderDB == null ? "NULL" : responseHeaderDB.size()) + ", fileDB.size() = " + (fileDB == null ? "NULL" : fileDB.size()));
// clean up the responseHeaderDB which cannot be cleaned the same way as the cache files.
// We do this as a concurrent job only once after start-up silently

29
source/net/yacy/http/AbstractRemoteHandler.java
Unescape View File

@ -64,15 +64,28 @@ abstract public class AbstractRemoteHandler extends ConnectHandler implements Ha
new Thread() {
@Override
public void run() {
final InetAddress localInetAddress = Domains.myPublicLocalIP();
if (localInetAddress != null) {
if (!localVirtualHostNames.contains(localInetAddress.getHostName())) {
localVirtualHostNames.add(localInetAddress.getHostName());
localVirtualHostNames.add(localInetAddress.getHostAddress()); // same as getServer().getURI().getHost()
for (InetAddress localInetAddress : Domains.myPublicIPv4()) {
if (localInetAddress != null) {
if (!localVirtualHostNames.contains(localInetAddress.getHostName())) {
localVirtualHostNames.add(localInetAddress.getHostName());
localVirtualHostNames.add(localInetAddress.getHostAddress()); // same as getServer().getURI().getHost()
}
if (!localVirtualHostNames.contains(localInetAddress.getCanonicalHostName())) {
localVirtualHostNames.add(localInetAddress.getCanonicalHostName());
}
}
if (!localVirtualHostNames.contains(localInetAddress.getCanonicalHostName())) {
localVirtualHostNames.add(localInetAddress.getCanonicalHostName());
}
for (InetAddress localInetAddress : Domains.myPublicIPv6()) {
if (localInetAddress != null) {
if (!localVirtualHostNames.contains(localInetAddress.getHostName())) {
localVirtualHostNames.add(localInetAddress.getHostName());
localVirtualHostNames.add(localInetAddress.getHostAddress()); // same as getServer().getURI().getHost()
}
if (!localVirtualHostNames.contains(localInetAddress.getCanonicalHostName())) {
localVirtualHostNames.add(localInetAddress.getCanonicalHostName());
}
}
}
if (sb.peers != null) {

23
source/net/yacy/peers/Transmission.java
Unescape View File

@ -83,13 +83,11 @@ public class Transmission {
* to multiple peers and to ensure that all entries in the indexContainers
* have a reference in the urls
* - a set of yacy seeds which will shrink as the containers are transmitted to them
* - a counter that gives the number of sucessful and unsuccessful transmissions so far
*/
private final Seed dhtTarget;
private final ReferenceContainerCache<WordReference> containers;
private final HandleSet references;
private final HandleSet badReferences;
private int hit, miss;
/**
* generate a new dispatcher target. such a target is defined with a primary target and
@ -103,8 +101,6 @@ public class Transmission {
this.containers = new ReferenceContainerCache<WordReference>(Segment.wordReferenceFactory, Segment.wordOrder, Word.commonHashLength);
this.references = new RowHandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0);
this.badReferences = new RowHandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0);
this.hit = 0;
this.miss = 0;
}
/*
@ -211,28 +207,11 @@ public class Transmission {
return this.dhtTarget;
}
/**
* return the number of successful transmissions
* @return
*/
public int hit() {
return this.hit;
}
/**
* return the number of unsuccessful transmissions
* @return
*/
public int miss() {
return this.miss;
}
public boolean transmit() {
// transferring selected words to remote peer
if (this.dhtTarget == Transmission.this.seeds.mySeed() || this.dhtTarget.hash.equals(Transmission.this.seeds.mySeed().hash)) {
// target is my own peer. This is easy. Just restore the indexContainer
restore();
this.hit++;
Transmission.this.log.info("Transfer of chunk to myself-target");
return true;
}
@ -256,13 +235,11 @@ public class Transmission {
// if the peer has set a pause time and we are in flush mode (index transfer)
// then we pause for a while now
Transmission.this.log.info("Transfer finished of chunk to target " + this.dhtTarget.hash + "/" + this.dhtTarget.getName());
this.hit++;
return true;
}
Transmission.this.log.info(
"Index transfer to peer " + this.dhtTarget.getName() + ":" + this.dhtTarget.hash +
" failed: " + error);
this.miss++;
// write information that peer does not receive index transmissions
Transmission.this.log.info("Transfer failed of chunk to target " + this.dhtTarget.hash + "/" + this.dhtTarget.getName() + ": " + error);
// get possibly newer target Info

12
source/net/yacy/search/Switchboard.java
Unescape View File

@ -315,7 +315,7 @@ public final class Switchboard extends serverSwitch {
// set loglevel and log
setLog(new ConcurrentLog("SWITCHBOARD"));
AccessTracker.setDumpFile(new File("DATA/LOG/queries.log"));
AccessTracker.setDumpFile(new File(dataPath, "DATA/LOG/queries.log"));
// set default peer name
Seed.ANON_PREFIX = getConfig("peernameprefix", "_anon");
@ -359,7 +359,7 @@ public final class Switchboard extends serverSwitch {
this.workPath = getDataPath(SwitchboardConstants.WORK_PATH, SwitchboardConstants.WORK_PATH_DEFAULT);
this.workPath.mkdirs();
// if default work files exist, copy them (don't overwrite existing!)
File defaultWorkPath = new File("defaults/data/work");
File defaultWorkPath = new File(appPath, "defaults/data/work");
if (defaultWorkPath.list() != null) {
for (String fs : defaultWorkPath.list()) {
File wf = new File(this.workPath, fs);
@ -730,7 +730,7 @@ public final class Switchboard extends serverSwitch {
// copy opensearch heuristic config (if not exist)
final File osdConfig = new File(getDataPath(), "DATA/SETTINGS/heuristicopensearch.conf");
if (!osdConfig.exists()) {
final File osdDefaultConfig = new File("defaults/heuristicopensearch.conf");
final File osdDefaultConfig = new File(appPath, "defaults/heuristicopensearch.conf");
this.log.info("heuristic.opensearch list Path = " + osdDefaultConfig.getAbsolutePath());
try {
Files.copy(osdDefaultConfig, osdConfig);
@ -781,7 +781,7 @@ public final class Switchboard extends serverSwitch {
}
// init html parser evaluation scheme
File parserPropertiesPath = new File("defaults/");
File parserPropertiesPath = new File(appPath, "defaults/");
String[] settingsList = parserPropertiesPath.list();
for ( final String l : settingsList ) {
if ( l.startsWith("parser.") && l.endsWith(".properties") ) {
@ -865,7 +865,7 @@ public final class Switchboard extends serverSwitch {
// load oai tables
final Map<String, File> oaiFriends =
OAIListFriendsLoader.loadListFriendsSources(
new File("defaults/oaiListFriendsSource.xml"),
new File(appPath, "defaults/oaiListFriendsSource.xml"),
getDataPath());
OAIListFriendsLoader.init(this.loader, oaiFriends, ClientIdentification.yacyInternetCrawlerAgent);
this.crawlQueues = new CrawlQueues(this, this.queuesRoot);
@ -2413,7 +2413,7 @@ public final class Switchboard extends serverSwitch {
final Properties profile = new Properties();
FileInputStream fileIn = null;
try {
fileIn = new FileInputStream(new File("DATA/SETTINGS/profile.txt"));
fileIn = new FileInputStream(new File(this.dataPath, "DATA/SETTINGS/profile.txt"));
profile.load(fileIn);
} catch (final IOException e ) {
} finally {

69
source/net/yacy/search/index/Fulltext.java
Unescape View File

@ -618,13 +618,17 @@ public final class Fulltext {
}
}
public static enum ExportFormat {
text, html, rss, solr;
}
// export methods
public Export export(final File f, final String filter, final String query, final int format, final boolean dom) {
public Export export(final File f, final String filter, final String query, final ExportFormat format, final boolean dom, final boolean text) {
if ((this.exportthread != null) && (this.exportthread.isAlive())) {
ConcurrentLog.warn("LURL-EXPORT", "cannot start another export thread, already one running");
return this.exportthread;
}
this.exportthread = new Export(f, filter, query, format, dom);
this.exportthread = new Export(f, filter, query, format, dom, text);
this.exportthread.start();
return this.exportthread;
}
@ -638,10 +642,10 @@ public final class Fulltext {
private final Pattern pattern;
private int count;
private String failure, query;
private final int format;
private final boolean dom;
private final ExportFormat format;
private final boolean dom, text;
private Export(final File f, final String filter, final String query, final int format, boolean dom) {
private Export(final File f, final String filter, final String query, final ExportFormat format, final boolean dom, final boolean text) {
// format: 0=text, 1=html, 2=rss/xml
this.f = f;
this.pattern = filter == null ? null : Pattern.compile(filter);
@ -650,6 +654,7 @@ public final class Fulltext {
this.failure = null;
this.format = format;
this.dom = dom;
this.text = text;
//if ((dom) && (format == 2)) dom = false;
}
@ -658,13 +663,13 @@ public final class Fulltext {
try {
final File parentf = this.f.getParentFile();
if (parentf != null) parentf.mkdirs();
OutputStream os = new FileOutputStream(this.format == 3 ? new File(this.f.getAbsolutePath() + ".gz") : this.f);
if (this.format == 3) os = new GZIPOutputStream(os, 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}};
OutputStream os = new FileOutputStream(this.format == ExportFormat.solr ? new File(this.f.getAbsolutePath() + ".gz") : this.f);
if (this.format == ExportFormat.solr) os = new GZIPOutputStream(os, 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}};
final PrintWriter pw = new PrintWriter(new BufferedOutputStream(os));
if (this.format == 1) {
if (this.format == ExportFormat.html) {
pw.println("<html><head></head><body>");
}
if (this.format == 2) {
if (this.format == ExportFormat.rss) {
pw.println("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
pw.println("<?xml-stylesheet type='text/xsl' href='/yacysearch.xsl' version='1.0'?>");
pw.println("<rss version=\"2.0\" xmlns:yacy=\"http://www.yacy.net/\" xmlns:opensearch=\"http://a9.com/-/spec/opensearch/1.1/\" xmlns:atom=\"http://www.w3.org/2005/Atom\">");
@ -673,7 +678,7 @@ public final class Fulltext {
pw.println("<description></description>");
pw.println("<link>http://yacy.net</link>");
}
if (this.format == 3) {
if (this.format == ExportFormat.solr) {
pw.println("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
pw.println("<response>");
pw.println("<result>");
@ -683,12 +688,25 @@ public final class Fulltext {
ReversibleScoreMap<String> stats = scores.get(CollectionSchema.host_s.getSolrFieldName());
for (final String host: stats) {
if (this.pattern != null && !this.pattern.matcher(host).matches()) continue;
if (this.format == 0) pw.println(host);
if (this.format == 1) pw.println("<a href=\"http://" + host + "\">" + host + "</a><br>");
if (this.format == ExportFormat.text) pw.println(host);
if (this.format == ExportFormat.html) pw.println("<a href=\"http://" + host + "\">" + host + "</a><br>");
this.count++;
}
} else {
if (this.format < 3) {
if (this.format == ExportFormat.solr || (this.text && this.format == ExportFormat.text)) {
BlockingQueue<SolrDocument> docs = Fulltext.this.getDefaultConnector().concurrentDocumentsByQuery(this.query + " AND " + CollectionSchema.httpstatus_i.getSolrFieldName() + ":200", null, 0, 100000000, Long.MAX_VALUE, 100, 1, true);
SolrDocument doc;
while ((doc = docs.take()) != AbstractSolrConnector.POISON_DOCUMENT) {
String url = getStringFrom(doc.getFieldValue(CollectionSchema.sku.getSolrFieldName()));
if (this.pattern != null && !this.pattern.matcher(url).matches()) continue;
CRIgnoreWriter sw = new CRIgnoreWriter();
if (this.text) sw.write((String) doc.getFieldValue(CollectionSchema.text_t.getSolrFieldName())); else EnhancedXMLResponseWriter.writeDoc(sw, doc);
sw.close();
String d = sw.toString();
pw.println(d);
this.count++;
}
} else {
BlockingQueue<SolrDocument> docs = Fulltext.this.getDefaultConnector().concurrentDocumentsByQuery(this.query + " AND " + CollectionSchema.httpstatus_i.getSolrFieldName() + ":200", null, 0, 100000000, Long.MAX_VALUE, 100, 1, true,
CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName(), CollectionSchema.title.getSolrFieldName(),
CollectionSchema.author.getSolrFieldName(), CollectionSchema.description_txt.getSolrFieldName(), CollectionSchema.size_i.getSolrFieldName(), CollectionSchema.last_modified.getSolrFieldName());
@ -705,13 +723,13 @@ public final class Fulltext {
size = (Integer) doc.getFieldValue(CollectionSchema.size_i.getSolrFieldName());
date = (Date) doc.getFieldValue(CollectionSchema.last_modified.getSolrFieldName());
if (this.pattern != null && !this.pattern.matcher(url).matches()) continue;
if (this.format == 0) {
if (this.format == ExportFormat.text) {
pw.println(url);
}
if (this.format == 1) {
if (this.format == ExportFormat.html) {
if (title != null) pw.println("<a href=\"" + MultiProtocolURL.escape(url) + "\">" + CharacterCoding.unicode2xml(title, true) + "</a>");
}
if (this.format == 2) {
if (this.format == ExportFormat.rss) {
pw.println("<item>");
if (title != null) pw.println("<title>" + CharacterCoding.unicode2xml(title, true) + "</title>");
pw.println("<link>" + MultiProtocolURL.escape(url) + "</link>");
@ -724,29 +742,16 @@ public final class Fulltext {
}
this.count++;
}
} else {
BlockingQueue<SolrDocument> docs = Fulltext.this.getDefaultConnector().concurrentDocumentsByQuery(this.query + " AND " + CollectionSchema.httpstatus_i.getSolrFieldName() + ":200", null, 0, 100000000, Long.MAX_VALUE, 100, 1, true);
SolrDocument doc;
while ((doc = docs.take()) != AbstractSolrConnector.POISON_DOCUMENT) {
String url = getStringFrom(doc.getFieldValue(CollectionSchema.sku.getSolrFieldName()));
if (this.pattern != null && !this.pattern.matcher(url).matches()) continue;
CRIgnoreWriter sw = new CRIgnoreWriter();
EnhancedXMLResponseWriter.writeDoc(sw, doc);
sw.close();
String d = sw.toString();
pw.println(d);
this.count++;
}
}
}
if (this.format == 1) {
if (this.format == ExportFormat.html) {
pw.println("</body></html>");
}
if (this.format == 2) {
if (this.format == ExportFormat.rss) {
pw.println("</channel>");
pw.println("</rss>");
}
if (this.format == 3) {
if (this.format == ExportFormat.solr) {
pw.println("</result>");
pw.println("</response>");
}

2
source/net/yacy/yacy.java
Unescape View File

@ -598,8 +598,6 @@ public final class yacy {
public static void main(String args[]) {
try {
System.setProperty("jsse.enableSNIExtension", "false"); // fix a ssl problem in Java 1.7, see http://teknosrc.com/javax-net-ssl-sslprotocolexception-handshake-alert-unrecognized_name-solved/
// check assertion status
//ClassLoader.getSystemClassLoader().setDefaultAssertionStatus(true);
boolean assertionenabled = false;

4
startYACY.bat
Unescape View File

@ -18,7 +18,9 @@ if exist DATA\SETTINGS\httpProxy.conf GoTo :RENAMEINDEX
if exist DATA\SETTINGS\yacy.conf GoTo :GETSTARTOPTS
:STARTJAVA
set javacmd=%javacmd% -Djava.awt.headless=true -Dsolr.directoryFactory=solr.MMapDirectoryFactory -Dfile.encoding=UTF-8
set javacmd=%javacmd% -Djava.awt.headless=true -Dsolr.directoryFactory=solr.MMapDirectoryFactory -Dfile.encoding=UTF-8 -Djsse.enableSNIExtension=false
Rem -Djsse.enableSNIExtension=false fix a ssl problem in Java 1.7, see http://teknosrc.com/javax-net-ssl-sslprotocolexception-handshake-alert-unrecognized_name-solved/
Rem Starting YaCy
Echo Generated classpath:%CLASSPATH%
Echo JRE Parameters:%javacmd%

4
startYACY.sh
Unescape View File

@ -6,7 +6,9 @@ PIDFILE="yacy.pid"
OS="`uname`"
#get javastart args
JAVA_ARGS="-server -Djava.awt.headless=true -Dfile.encoding=UTF-8";
JAVA_ARGS="-server -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djsse.enableSNIExtension=false";
#rem -Djsse.enableSNIExtension=false fix a ssl problem in Java 1.7, see http://teknosrc.com/javax-net-ssl-sslprotocolexception-handshake-alert-unrecognized_name-solved/
#JAVA_ARGS="-verbose:gc -XX:+PrintGCTimeStamps -XX:+PrintGCDetails $JAVA_ARGS";
#check if OS is Sun Solaris or one of the OpenSolaris distributions and use different version of id if necessary

4
startYACY_debug.bat
Unescape View File

@ -17,7 +17,9 @@ if exist DATA\SETTINGS\httpProxy.conf GoTo :RENAMEINDEX
if exist DATA\SETTINGS\yacy.conf GoTo :GETSTARTOPTS
:STARTJAVA
set javacmd=%javacmd% -XX:-UseGCOverheadLimit -Djava.awt.headless=true -Dfile.encoding=UTF-8
set javacmd=%javacmd% -XX:-UseGCOverheadLimit -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djsse.enableSNIExtension=false
Rem -Djsse.enableSNIExtension=false fix a ssl problem in Java 1.7, see http://teknosrc.com/javax-net-ssl-sslprotocolexception-handshake-alert-unrecognized_name-solved/
Rem Starting YaCy
Echo Generated classpath:%CLASSPATH%
Echo JRE Parameters:%javacmd%

Write Preview
Loading…
Cancel
Save