diff --git a/.classpath b/.classpath index 9f6b72b3e..073f0d841 100644 --- a/.classpath +++ b/.classpath @@ -43,8 +43,8 @@ - - + + @@ -65,26 +65,26 @@ - - + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + diff --git a/build.xml b/build.xml index a93c2befc..db2e4ce71 100644 --- a/build.xml +++ b/build.xml @@ -204,32 +204,32 @@ - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - + + - - + + diff --git a/htroot/yacysearchitem.java b/htroot/yacysearchitem.java index ec50e3cc9..a38cc523c 100644 --- a/htroot/yacysearchitem.java +++ b/htroot/yacysearchitem.java @@ -316,7 +316,7 @@ public class yacysearchitem { final String license = URLLicense.aquireLicense(image.imageUrl); // this is just the license key to get the image forwarded through the YaCy thumbnail viewer, not an actual lawful license //sb.loader.loadIfNotExistBackground(image.imageUrl, 1024 * 1024 * 10, null, ClientIdentification.yacyIntranetCrawlerAgent); - prop.putHTML("content_item_hrefCache", "ViewImage." + ("gif.png.svg".contains(imageUrlExt) ? imageUrlExt : "png") + "?maxwidth=128&maxheight=128&code="+license+"&isStatic=true&quadratic=&url=" + imageUrlstring); + prop.putHTML("content_item_hrefCache", "ViewImage." + (!imageUrlExt.isEmpty() && "gif.png.svg".contains(imageUrlExt) ? imageUrlExt : "png") + "?maxwidth=128&maxheight=128&code="+license+"&isStatic=true&quadratic=&url=" + imageUrlstring); prop.putHTML("content_item_href", imageUrlstring); prop.putHTML("content_item_target", target); prop.put("content_item_code", license); diff --git a/htroot/yacysearchtrailer.java b/htroot/yacysearchtrailer.java index 35141ab4d..d60994fa7 100644 --- a/htroot/yacysearchtrailer.java +++ b/htroot/yacysearchtrailer.java @@ -26,10 +26,9 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.Map; -import org.apache.solr.schema.TrieDateField; +import org.apache.solr.util.DateFormatUtil; import net.yacy.cora.date.AbstractFormatter; -import net.yacy.cora.document.analysis.Classification; import net.yacy.cora.document.analysis.Classification.ContentDomain; import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.lod.vocabulary.Tagging; @@ -401,13 +400,13 @@ public class yacysearchtrailer { String shortname = name.substring(0, 10); long d; Date dd; - try {dd = TrieDateField.parseDate(name); d = dd.getTime();} catch (ParseException e) {continue;} + try {dd = DateFormatUtil.parseDate(name); d = dd.getTime();} catch (ParseException e) {continue;} if (fromconstraint != null && dd.before(fromconstraint)) continue; if (toconstraint != null && dd.after(toconstraint)) break; if (dx > 0) { while (d - dx > AbstractFormatter.dayMillis) { dx += AbstractFormatter.dayMillis; - String sn = TrieDateField.formatExternal(new Date(dx)).substring(0, 10); + String sn = DateFormatUtil.formatExternal(new Date(dx)).substring(0, 10); prop.put("nav-dates_element_" + i + "_on", 0); prop.put(fileType, "nav-dates_element_" + i + "_name", sn); prop.put("nav-dates_element_" + i + "_count", 0); diff --git a/lib/lucene-analyzers-common-5.2.1.jar b/lib/lucene-analyzers-common-5.3.1.jar similarity index 89% rename from lib/lucene-analyzers-common-5.2.1.jar rename to lib/lucene-analyzers-common-5.3.1.jar index aaa26a135..a8713840c 100644 Binary files a/lib/lucene-analyzers-common-5.2.1.jar and b/lib/lucene-analyzers-common-5.3.1.jar differ diff --git a/lib/lucene-analyzers-phonetic-5.2.1.jar b/lib/lucene-analyzers-phonetic-5.3.1.jar similarity index 89% rename from lib/lucene-analyzers-phonetic-5.2.1.jar rename to lib/lucene-analyzers-phonetic-5.3.1.jar index 838b8daf2..5493461f7 100644 Binary files a/lib/lucene-analyzers-phonetic-5.2.1.jar and b/lib/lucene-analyzers-phonetic-5.3.1.jar differ diff --git a/lib/lucene-backward-codecs-5.2.1.jar b/lib/lucene-backward-codecs-5.3.1.jar similarity index 67% rename from lib/lucene-backward-codecs-5.2.1.jar rename to lib/lucene-backward-codecs-5.3.1.jar index bbdfff8b2..8fef66c3d 100644 Binary files a/lib/lucene-backward-codecs-5.2.1.jar and b/lib/lucene-backward-codecs-5.3.1.jar differ diff --git a/lib/lucene-classification-5.2.1.jar b/lib/lucene-classification-5.2.1.jar deleted file mode 100644 index 9901b5a97..000000000 Binary files a/lib/lucene-classification-5.2.1.jar and /dev/null differ diff --git a/lib/lucene-classification-5.3.1.jar b/lib/lucene-classification-5.3.1.jar new file mode 100644 index 000000000..e575bb81b Binary files /dev/null and b/lib/lucene-classification-5.3.1.jar differ diff --git a/lib/lucene-codecs-5.2.1.jar b/lib/lucene-codecs-5.3.1.jar similarity index 64% rename from lib/lucene-codecs-5.2.1.jar rename to lib/lucene-codecs-5.3.1.jar index be7e95778..33097bc69 100644 Binary files a/lib/lucene-codecs-5.2.1.jar and b/lib/lucene-codecs-5.3.1.jar differ diff --git a/lib/lucene-core-5.2.1.jar b/lib/lucene-core-5.3.1.jar similarity index 60% rename from lib/lucene-core-5.2.1.jar rename to lib/lucene-core-5.3.1.jar index 18b887f79..9ffa9173b 100644 Binary files a/lib/lucene-core-5.2.1.jar and b/lib/lucene-core-5.3.1.jar differ diff --git a/lib/lucene-facet-5.2.1.jar b/lib/lucene-facet-5.3.1.jar similarity index 62% rename from lib/lucene-facet-5.2.1.jar rename to lib/lucene-facet-5.3.1.jar index fb96a545d..7262908b8 100644 Binary files a/lib/lucene-facet-5.2.1.jar and b/lib/lucene-facet-5.3.1.jar differ diff --git a/lib/lucene-grouping-5.2.1.jar b/lib/lucene-grouping-5.3.1.jar similarity index 82% rename from lib/lucene-grouping-5.2.1.jar rename to lib/lucene-grouping-5.3.1.jar index 6d7b46dd1..dd1c8cdd6 100644 Binary files a/lib/lucene-grouping-5.2.1.jar and b/lib/lucene-grouping-5.3.1.jar differ diff --git a/lib/lucene-highlighter-5.2.1.jar b/lib/lucene-highlighter-5.3.1.jar similarity index 57% rename from lib/lucene-highlighter-5.2.1.jar rename to lib/lucene-highlighter-5.3.1.jar index e81ecbd7c..1c1a514d8 100644 Binary files a/lib/lucene-highlighter-5.2.1.jar and b/lib/lucene-highlighter-5.3.1.jar differ diff --git a/lib/lucene-join-5.2.1.jar b/lib/lucene-join-5.2.1.jar deleted file mode 100644 index 570275134..000000000 Binary files a/lib/lucene-join-5.2.1.jar and /dev/null differ diff --git a/lib/lucene-join-5.3.1.jar b/lib/lucene-join-5.3.1.jar new file mode 100644 index 000000000..2741a513a Binary files /dev/null and b/lib/lucene-join-5.3.1.jar differ diff --git a/lib/lucene-memory-5.2.1.jar b/lib/lucene-memory-5.2.1.jar deleted file mode 100644 index 7c4eb8ec2..000000000 Binary files a/lib/lucene-memory-5.2.1.jar and /dev/null differ diff --git a/lib/lucene-memory-5.3.1.jar b/lib/lucene-memory-5.3.1.jar new file mode 100644 index 000000000..e3acb1fe7 Binary files /dev/null and b/lib/lucene-memory-5.3.1.jar differ diff --git a/lib/lucene-misc-5.2.1.jar b/lib/lucene-misc-5.3.1.jar similarity index 74% rename from lib/lucene-misc-5.2.1.jar rename to lib/lucene-misc-5.3.1.jar index ce6eeba16..f0993f26b 100644 Binary files a/lib/lucene-misc-5.2.1.jar and b/lib/lucene-misc-5.3.1.jar differ diff --git a/lib/lucene-queries-5.2.1.jar b/lib/lucene-queries-5.3.1.jar similarity index 59% rename from lib/lucene-queries-5.2.1.jar rename to lib/lucene-queries-5.3.1.jar index 01742fa90..366de9f1c 100644 Binary files a/lib/lucene-queries-5.2.1.jar and b/lib/lucene-queries-5.3.1.jar differ diff --git a/lib/lucene-queryparser-5.2.1.jar b/lib/lucene-queryparser-5.3.1.jar similarity index 76% rename from lib/lucene-queryparser-5.2.1.jar rename to lib/lucene-queryparser-5.3.1.jar index ee7599359..12a5e63a3 100644 Binary files a/lib/lucene-queryparser-5.2.1.jar and b/lib/lucene-queryparser-5.3.1.jar differ diff --git a/lib/lucene-spatial-5.2.1.jar b/lib/lucene-spatial-5.3.1.jar similarity index 50% rename from lib/lucene-spatial-5.2.1.jar rename to lib/lucene-spatial-5.3.1.jar index 51b36233f..93b28ddbe 100644 Binary files a/lib/lucene-spatial-5.2.1.jar and b/lib/lucene-spatial-5.3.1.jar differ diff --git a/lib/lucene-suggest-5.2.1.jar b/lib/lucene-suggest-5.2.1.jar deleted file mode 100644 index e30c5d41d..000000000 Binary files a/lib/lucene-suggest-5.2.1.jar and /dev/null differ diff --git a/lib/lucene-suggest-5.3.1.jar b/lib/lucene-suggest-5.3.1.jar new file mode 100644 index 000000000..cdc19645d Binary files /dev/null and b/lib/lucene-suggest-5.3.1.jar differ diff --git a/lib/poi-3.12-20150511.jar b/lib/poi-3.12-20150511.jar deleted file mode 100644 index a4f42634f..000000000 Binary files a/lib/poi-3.12-20150511.jar and /dev/null differ diff --git a/lib/poi-3.13-20150929.jar b/lib/poi-3.13-20150929.jar new file mode 100644 index 000000000..a9e380085 Binary files /dev/null and b/lib/poi-3.13-20150929.jar differ diff --git a/lib/poi-3.12.License b/lib/poi-3.13.License similarity index 100% rename from lib/poi-3.12.License rename to lib/poi-3.13.License diff --git a/lib/poi-scratchpad-3.12-20150511.jar b/lib/poi-scratchpad-3.12-20150511.jar deleted file mode 100644 index 941f5b4c0..000000000 Binary files a/lib/poi-scratchpad-3.12-20150511.jar and /dev/null differ diff --git a/lib/poi-scratchpad-3.13-20150929.jar b/lib/poi-scratchpad-3.13-20150929.jar new file mode 100644 index 000000000..735eb7ded Binary files /dev/null and b/lib/poi-scratchpad-3.13-20150929.jar differ diff --git a/lib/solr-core-5.2.1.jar b/lib/solr-core-5.3.1.jar similarity index 60% rename from lib/solr-core-5.2.1.jar rename to lib/solr-core-5.3.1.jar index cd2844a21..59c543c3b 100644 Binary files a/lib/solr-core-5.2.1.jar and b/lib/solr-core-5.3.1.jar differ diff --git a/lib/solr-solrj-5.2.1.jar b/lib/solr-solrj-5.3.1.jar similarity index 61% rename from lib/solr-solrj-5.2.1.jar rename to lib/solr-solrj-5.3.1.jar index 1e33bd0e8..c8927e7ee 100644 Binary files a/lib/solr-solrj-5.2.1.jar and b/lib/solr-solrj-5.3.1.jar differ diff --git a/nbproject/project.xml b/nbproject/project.xml index 585323ea2..c51ef375f 100644 --- a/nbproject/project.xml +++ b/nbproject/project.xml @@ -83,7 +83,7 @@ source htroot - lib/J7Zip-modified.jar;lib/apache-mime4j-0.6.jar;lib/bcmail-jdk15-1.46.jar;lib/bcprov-jdk15-1.46.jar;lib/chardet.jar;lib/commons-codec-1.10.jar;lib/commons-compress-1.9.jar;lib/commons-fileupload-1.3.1.jar;lib/commons-io-2.4.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.2.jar;lib/fontbox-1.8.10.jar;lib/guava-18.0.jar;lib/htmllexer.jar;lib/httpclient-4.5.jar;lib/httpcore-4.4.1.jar;lib/httpmime-4.5.jar;lib/icu4j-55_1.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/javax.servlet-api-3.1.0.jar;lib/jcifs-1.3.17.jar;lib/jcl-over-slf4j-1.7.12.jar;lib/jempbox-1.8.10.jar;lib/jetty-client-9.2.11.v20150529.jar;lib/jetty-continuation-9.2.11.v20150529.jar;lib/jetty-deploy-9.2.11.v20150529.jar;lib/jetty-http-9.2.11.v20150529.jar;lib/jetty-io-9.2.11.v20150529.jar;lib/jetty-jmx-9.2.11.v20150529.jar;lib/jetty-proxy-9.2.11.v20150529.jar;lib/jetty-security-9.2.11.v20150529.jar;lib/jetty-server-9.2.11.v20150529.jar;lib/jetty-servlet-9.2.11.v20150529.jar;lib/jetty-servlets-9.2.11.v20150529.jar;lib/jetty-util-9.2.11.v20150529.jar;lib/jetty-webapp-9.2.11.v20150529.jar;lib/jetty-xml-9.2.11.v20150529.jar;lib/jsch-0.1.53.jar;lib/json-simple-1.1.1.jar;lib/jsoup-1.8.3.jar;lib/log4j-over-slf4j-1.7.12.jar;lib/lucene-analyzers-common-5.2.1.jar;lib/lucene-analyzers-phonetic-5.2.1.jar;lib/lucene-backward-codecs-5.2.1.jar;lib/lucene-classification-5.2.1.jar;lib/lucene-codecs-5.2.1.jar;lib/lucene-core-5.2.1.jar;lib/lucene-facet-5.2.1.jar;lib/lucene-grouping-5.2.1.jar;lib/lucene-highlighter-5.2.1.jar;lib/lucene-join-5.2.1.jar;lib/lucene-memory-5.2.1.jar;lib/lucene-misc-5.2.1.jar;lib/lucene-queries-5.2.1.jar;lib/lucene-queryparser-5.2.1.jar;lib/lucene-spatial-5.2.1.jar;lib/lucene-suggest-5.2.1.jar;lib/metadata-extractor-2.8.1.jar;lib/noggit-0.6.jar;lib/org.restlet.jar;lib/pdfbox-1.8.10.jar;lib/poi-3.12-20150511.jar;lib/poi-scratchpad-3.12-20150511.jar;lib/slf4j-api-1.7.12.jar;lib/slf4j-jdk14-1.7.12.jar;lib/solr-core-5.2.1.jar;lib/solr-solrj-5.2.1.jar;lib/spatial4j-0.4.1.jar;lib/stax2-api-3.1.4.jar;lib/webcat-0.1-swf.jar;lib/weupnp-0.1.3.jar;lib/woodstox-core-asl-4.4.1.jar;lib/xercesImpl.jar;lib/xml-apis.jar;lib/xmpcore-5.1.2.jar;lib/zookeeper-3.4.6.jar + lib/J7Zip-modified.jar;lib/apache-mime4j-0.6.jar;lib/bcmail-jdk15-1.46.jar;lib/bcprov-jdk15-1.46.jar;lib/chardet.jar;lib/commons-codec-1.10.jar;lib/commons-compress-1.9.jar;lib/commons-fileupload-1.3.1.jar;lib/commons-io-2.4.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.2.jar;lib/fontbox-1.8.10.jar;lib/guava-18.0.jar;lib/htmllexer.jar;lib/httpclient-4.5.jar;lib/httpcore-4.4.1.jar;lib/httpmime-4.5.jar;lib/icu4j-55_1.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/javax.servlet-api-3.1.0.jar;lib/jcifs-1.3.17.jar;lib/jcl-over-slf4j-1.7.12.jar;lib/jempbox-1.8.10.jar;lib/jetty-client-9.2.11.v20150529.jar;lib/jetty-continuation-9.2.11.v20150529.jar;lib/jetty-deploy-9.2.11.v20150529.jar;lib/jetty-http-9.2.11.v20150529.jar;lib/jetty-io-9.2.11.v20150529.jar;lib/jetty-jmx-9.2.11.v20150529.jar;lib/jetty-proxy-9.2.11.v20150529.jar;lib/jetty-security-9.2.11.v20150529.jar;lib/jetty-server-9.2.11.v20150529.jar;lib/jetty-servlet-9.2.11.v20150529.jar;lib/jetty-servlets-9.2.11.v20150529.jar;lib/jetty-util-9.2.11.v20150529.jar;lib/jetty-webapp-9.2.11.v20150529.jar;lib/jetty-xml-9.2.11.v20150529.jar;lib/jsch-0.1.53.jar;lib/json-simple-1.1.1.jar;lib/jsoup-1.8.3.jar;lib/log4j-over-slf4j-1.7.12.jar;lib/lucene-analyzers-common-5.3.1.jar;lib/lucene-analyzers-phonetic-5.3.1.jar;lib/lucene-backward-codecs-5.3.1.jar;lib/lucene-classification-5.3.1.jar;lib/lucene-codecs-5.3.1.jar;lib/lucene-core-5.3.1.jar;lib/lucene-facet-5.3.1.jar;lib/lucene-grouping-5.3.1.jar;lib/lucene-highlighter-5.3.1.jar;lib/lucene-join-5.3.1.jar;lib/lucene-memory-5.3.1.jar;lib/lucene-misc-5.3.1.jar;lib/lucene-queries-5.3.1.jar;lib/lucene-queryparser-5.3.1.jar;lib/lucene-spatial-5.3.1.jar;lib/lucene-suggest-5.3.1.jar;lib/metadata-extractor-2.8.1.jar;lib/noggit-0.6.jar;lib/org.restlet.jar;lib/pdfbox-1.8.10.jar;lib/poi-3.13-20150929.jar;lib/poi-scratchpad-3.13-20150929.jar;lib/slf4j-api-1.7.12.jar;lib/slf4j-jdk14-1.7.12.jar;lib/solr-core-5.3.1.jar;lib/solr-solrj-5.3.1.jar;lib/spatial4j-0.4.1.jar;lib/stax2-api-3.1.4.jar;lib/webcat-0.1-swf.jar;lib/weupnp-0.1.3.jar;lib/woodstox-core-asl-4.4.1.jar;lib/xercesImpl.jar;lib/xml-apis.jar;lib/xmpcore-5.1.2.jar;lib/zookeeper-3.4.6.jar lib/yacycore.jar 1.7 diff --git a/pom.xml b/pom.xml index c86c59db0..53c467dc1 100644 --- a/pom.xml +++ b/pom.xml @@ -34,7 +34,7 @@ 1.7 1.7 - 5.2.1 + 5.3.1 9.2.11.v20150529 @@ -498,12 +498,12 @@ org.apache.poi poi - 3.12 + 3.13 org.apache.poi poi-scratchpad - 3.12 + 3.13 sax diff --git a/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java index 3c92c4fab..eabb456cc 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java @@ -48,10 +48,10 @@ import org.apache.solr.schema.FieldType; import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; import org.apache.solr.schema.TextField; -import org.apache.solr.schema.TrieDateField; import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.util.DateFormatUtil; public class EnhancedXMLResponseWriter implements QueryResponseWriter { @@ -270,7 +270,7 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter { } else if (typeName.equals(SolrType.num_long.printName())) { writeTag(writer, "long", name, value, true); } else if (typeName.equals(SolrType.date.printName())) { - writeTag(writer, "date", name, TrieDateField.formatExternal(new Date(Long.parseLong(value))), true); + writeTag(writer, "date", name, DateFormatUtil.formatExternal(new Date(Long.parseLong(value))), true); } else if (typeName.equals(SolrType.num_float.printName())) { writeTag(writer, "float", name, value, true); } else if (typeName.equals(SolrType.num_double.printName())) { @@ -288,7 +288,7 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter { } else if (value instanceof Long) { writeTag(writer, "long", name, ((Long) value).toString(), true); } else if (value instanceof Date) { - writeTag(writer, "date", name, TrieDateField.formatExternal((Date) value), true); + writeTag(writer, "date", name, DateFormatUtil.formatExternal((Date) value), true); } else if (value instanceof Float) { writeTag(writer, "float", name, ((Float) value).toString(), true); } else if (value instanceof Double) { diff --git a/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java index ebff21190..59e5c1f99 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java @@ -45,10 +45,10 @@ import org.apache.solr.schema.FieldType; import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; import org.apache.solr.schema.TextField; -import org.apache.solr.schema.TrieDateField; import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.util.DateFormatUtil; public class HTMLResponseWriter implements QueryResponseWriter { @@ -223,7 +223,7 @@ public class HTMLResponseWriter implements QueryResponseWriter { if (typeName.equals(SolrType.bool.printName())) { return "F".equals(value) ? "false" : "true"; } else if (typeName.equals(SolrType.date.printName())) { - return TrieDateField.formatExternal(new Date(Long.parseLong(value))); + return DateFormatUtil.formatExternal(new Date(Long.parseLong(value))); } return value; } diff --git a/source/net/yacy/crawler/HostQueue.java b/source/net/yacy/crawler/HostQueue.java index 6157efb80..3638f4534 100644 --- a/source/net/yacy/crawler/HostQueue.java +++ b/source/net/yacy/crawler/HostQueue.java @@ -98,7 +98,7 @@ public class HostQueue implements Balancer { int p = filename.lastIndexOf('.'); if (p < 0) throw new RuntimeException("hostPath name must contain a dot: " + filename); this.hostName = filename.substring(0, p); - this.port = Integer.parseInt(filename.substring(p + 1)); + this.port = Integer.parseInt(filename.substring(p + 1)); // consider "host.com" contains dot but no required port -> will throw exception init(); } diff --git a/source/net/yacy/crawler/RecrawlBusyThread.java b/source/net/yacy/crawler/RecrawlBusyThread.java index 840f9fc90..cc9bc1bf6 100644 --- a/source/net/yacy/crawler/RecrawlBusyThread.java +++ b/source/net/yacy/crawler/RecrawlBusyThread.java @@ -57,6 +57,7 @@ public class RecrawlBusyThread extends AbstractBusyThread { final Switchboard sb; private final Set urlstack; // buffer of urls to recrawl public long urlsfound = 0; + private String solrSortBy; public RecrawlBusyThread(Switchboard xsb) { super(3000, 1000); // set lower limits of cycle delay @@ -66,6 +67,9 @@ public class RecrawlBusyThread extends AbstractBusyThread { this.sb = xsb; urlstack = new HashSet(); + // workaround to prevent solr exception on existing index (not fully reindexed) since intro of schema with docvalues + // org.apache.solr.core.SolrCore java.lang.IllegalStateException: unexpected docvalues type NONE for field 'load_date_dt' (expected=NUMERIC). Use UninvertingReader or index with docvalues. + solrSortBy = null; // CollectionSchema.load_date_dt.getSolrFieldName() + " asc"; } /** @@ -167,8 +171,8 @@ public class RecrawlBusyThread extends AbstractBusyThread { if (!solrConnector.isClosed()) { try { // query all or only httpstatus=200 depending on includefailed flag - docList = solrConnector.getDocumentListByQuery(this.includefailed ? currentQuery : currentQuery + " AND (" + CollectionSchema.httpstatus_i.name() + ":200)", - CollectionSchema.load_date_dt.getSolrFieldName() + " asc", this.chunkstart, this.chunksize, CollectionSchema.sku.getSolrFieldName()); + docList = solrConnector.getDocumentListByQuery(this.includefailed ? currentQuery : currentQuery + " AND (" + CollectionSchema.httpstatus_i.name() + ":200)", + this.solrSortBy, this.chunkstart, this.chunksize, CollectionSchema.sku.getSolrFieldName()); this.urlsfound = docList.getNumFound(); } catch (Throwable e) { this.urlsfound = 0; diff --git a/source/net/yacy/document/parser/images/genericImageParser.java b/source/net/yacy/document/parser/images/genericImageParser.java index 4f69b7eb6..c6370f00b 100644 --- a/source/net/yacy/document/parser/images/genericImageParser.java +++ b/source/net/yacy/document/parser/images/genericImageParser.java @@ -129,6 +129,12 @@ public class genericImageParser extends AbstractParser implements Parser { byte[] b; try { b = FileUtils.read(source); + // check jpeg file signature (magic number FF D8 FF) + if ((b[0] != (byte) 0xFF) // cast to signed byte (-1) + || (b[1] != (byte) 0xD8) //cast to signed byte (-40) + || (b[2] != (byte) 0xFF)) { + throw new Parser.Failure("File has no jpeg signature", location); + } } catch (final IOException e) { ConcurrentLog.logException(e); throw new Parser.Failure(e.getMessage(), location); diff --git a/source/net/yacy/search/query/QueryModifier.java b/source/net/yacy/search/query/QueryModifier.java index f3db2f233..aac89403f 100644 --- a/source/net/yacy/search/query/QueryModifier.java +++ b/source/net/yacy/search/query/QueryModifier.java @@ -26,7 +26,7 @@ import java.util.Date; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.MultiMapSolrParams; -import org.apache.solr.schema.TrieDateField; +import org.apache.solr.util.DateFormatUtil; import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.util.CommonPattern; @@ -355,7 +355,7 @@ public class QueryModifier { Date onDate = DateDetection.parseLine(onDescription, timezoneOffset); StringBuilder filterQuery = new StringBuilder(20); if (onDate != null) { - String dstr = TrieDateField.formatExternal(onDate); + String dstr = DateFormatUtil.formatExternal(onDate); filterQuery.append(CollectionSchema.dates_in_content_dts.getSolrFieldName()).append(":[").append(dstr).append(" TO ").append(dstr).append(']'); } return filterQuery.toString(); @@ -366,8 +366,8 @@ public class QueryModifier { Date toDate = to == null || to.equals("*") ? null : DateDetection.parseLine(to, timezoneOffset); StringBuilder filterQuery = new StringBuilder(20); if (fromDate != null && toDate != null) { - String dstrFrom = fromDate == null ? "*" : TrieDateField.formatExternal(fromDate); - String dstrTo = toDate == null ? "*" : TrieDateField.formatExternal(toDate); + String dstrFrom = fromDate == null ? "*" : DateFormatUtil.formatExternal(fromDate); + String dstrTo = toDate == null ? "*" : DateFormatUtil.formatExternal(toDate); filterQuery.append(CollectionSchema.dates_in_content_dts.getSolrFieldName()).append(":[").append(dstrFrom).append(" TO ").append(dstrTo).append(']'); } return filterQuery.toString(); diff --git a/source/net/yacy/search/query/QueryParams.java b/source/net/yacy/search/query/QueryParams.java index fea69f626..2e4bcfa3f 100644 --- a/source/net/yacy/search/query/QueryParams.java +++ b/source/net/yacy/search/query/QueryParams.java @@ -73,7 +73,7 @@ import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrQuery.SortClause; import org.apache.solr.common.params.DisMaxParams; import org.apache.solr.common.params.FacetParams; -import org.apache.solr.schema.TrieDateField; +import org.apache.solr.util.DateFormatUtil; public final class QueryParams { @@ -465,8 +465,8 @@ public final class QueryParams { for (String field: this.facetfields) params.addFacetField("{!ex=" + field + "}" + field); // params.addFacetField("{!ex=" + field + "}" + field); if (this.facetfields.contains(CollectionSchema.dates_in_content_dts.name())) { params.setParam(FacetParams.FACET_RANGE, CollectionSchema.dates_in_content_dts.name()); - String start = TrieDateField.formatExternal(new Date(System.currentTimeMillis() - 1000L * 60L * 60L * 24L * 3)); - String end = TrieDateField.formatExternal(new Date(System.currentTimeMillis() + 1000L * 60L * 60L * 24L * 3)); + String start = DateFormatUtil.formatExternal(new Date(System.currentTimeMillis() - 1000L * 60L * 60L * 24L * 3)); + String end = DateFormatUtil.formatExternal(new Date(System.currentTimeMillis() + 1000L * 60L * 60L * 24L * 3)); params.setParam("f." + CollectionSchema.dates_in_content_dts.getSolrFieldName() + ".facet.range.start", start); params.setParam("f." + CollectionSchema.dates_in_content_dts.getSolrFieldName() + ".facet.range.end", end); params.setParam("f." + CollectionSchema.dates_in_content_dts.getSolrFieldName() + ".facet.range.gap", "+1DAY"); diff --git a/source/net/yacy/server/serverObjects.java b/source/net/yacy/server/serverObjects.java index 1d3688be7..d5be88e51 100644 --- a/source/net/yacy/server/serverObjects.java +++ b/source/net/yacy/server/serverObjects.java @@ -69,6 +69,7 @@ import net.yacy.search.Switchboard; import net.yacy.search.schema.CollectionSchema; import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.FacetParams; import org.apache.solr.common.params.MultiMapSolrParams; @@ -552,7 +553,7 @@ public class serverObjects implements Serializable, Cloneable { if (facets != null && facets.length > 0) { this.remove("facet"); this.put("facet", "true"); - for (int i = 0; i < facets.length; i++) this.add("facet.field", facets[i].getSolrFieldName()); + for (int i = 0; i < facets.length; i++) this.add(FacetParams.FACET_FIELD, facets[i].getSolrFieldName()); } return this.map; }