Merge remote-tracking branch 'regerdev/master'

nbproject/project.xml

@@ -77,7 +77,7 @@
             <compilation-unit>
                 <package-root>source</package-root>
                 <package-root>htroot</package-root>
-                <classpath mode="compile">lib/activation.jar;lib/apache-mime4j-0.6.jar;lib/apache-solr-core-3.6.1.jar;lib/apache-solr-solrj-3.6.1.jar;lib/arq-2.8.7.jar;lib/bcmail-jdk15-145.jar;lib/bcprov-jdk15-145.jar;lib/commons-codec-1.6.jar;lib/commons-compress-1.4.1.jar;lib/commons-fileupload-1.2.2.jar;lib/commons-httpclient-3.1.jar;lib/commons-io-2.1.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.1.1.jar;lib/fontbox-1.7.0.jar;lib/geronimo-stax-api_1.0_spec-1.0.1.jar;lib/guava-r05.jar;lib/htmllexer.jar;lib/htmlparser.jar;lib/httpclient-4.2.2.jar;lib/httpcore-4.2.2.jar;lib/httpmime-4.2.2.jar;lib/icu4j-core.jar;lib/iri-0.8.jar;lib/J7Zip-modified.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/jcifs-1.3.15.jar;lib/jcl-over-slf4j-1.6.1.jar;lib/jempbox-1.7.0.jar;lib/jena-2.6.4.jar;lib/jetty-6.1.26-patched-JETTY-1340.jar;lib/jetty-util-6.1.26-patched-JETTY-1340.jar;lib/jsch-0.1.42.jar;lib/json-simple-1.1.jar;lib/jsoup-1.6.3.jar;lib/log4j-1.2.16.jar;lib/log4j-over-slf4j-1.6.1.jar;lib/lucene-analyzers-3.6.1.jar;lib/lucene-core-3.6.1.jar;lib/lucene-highlighter-3.6.1.jar;lib/lucene-phonetic-3.6.1.jar;lib/lucene-spatial-3.6.1.jar;lib/lucene-spellchecker-3.6.1.jar;lib/metadata-extractor-2.4.0-beta-1.jar;lib/mysql-connector-java-5.1.12-bin.jar;lib/pdfbox-1.7.0.jar;lib/poi-3.6-20091214.jar;lib/poi-scratchpad-3.6-20091214.jar;lib/sax-2.0.1.jar;lib/servlet-api-2.5-20081211.jar;lib/slf4j-api-1.6.1.jar;lib/slf4j-jdk14-1.6.1.jar;lib/webcat-0.1-swf.jar;lib/wstx-asl-3.2.7.jar;lib/xercesImpl.jar;lib/xml-apis.jar</classpath>
+                <classpath mode="compile">lib/activation.jar;lib/apache-mime4j-0.6.jar;lib/apache-solr-core-4.0.0.jar;lib/apache-solr-solrj-4.0.0.jar;lib/arq-2.8.7.jar;lib/bcmail-jdk15-145.jar;lib/bcprov-jdk15-145.jar;lib/commons-codec-1.7.jar;lib/commons-compress-1.4.1.jar;lib/commons-fileupload-1.2.2.jar;lib/commons-httpclient-3.1.jar;lib/commons-io-2.1.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.1.1.jar;lib/fontbox-1.7.0.jar;lib/geronimo-stax-api_1.0_spec-1.0.1.jar;lib/guava-r05.jar;lib/htmllexer.jar;lib/htmlparser.jar;lib/httpclient-4.2.2.jar;lib/httpcore-4.2.2.jar;lib/httpmime-4.2.2.jar;lib/icu4j-core.jar;lib/iri-0.8.jar;lib/J7Zip-modified.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/jcifs-1.3.15.jar;lib/jcl-over-slf4j-1.6.1.jar;lib/jempbox-1.7.0.jar;lib/jena-2.6.4.jar;lib/jetty-6.1.26-patched-JETTY-1340.jar;lib/jetty-util-6.1.26-patched-JETTY-1340.jar;lib/jsch-0.1.42.jar;lib/json-simple-1.1.jar;lib/jsoup-1.6.3.jar;lib/log4j-1.2.16.jar;lib/log4j-over-slf4j-1.6.1.jar;lib/lucene-analyzers-4.0.0.jar;lib/lucene-core-4.0.0.jar;lib/lucene-highlighter-4.0.0.jar;lib/lucene-phonetic-4.0.0.jar;lib/lucene-spatial-4.0.0.jar;lib/lucene-spellchecker-4.0.0.jar;lib/metadata-extractor-2.4.0-beta-1.jar;lib/mysql-connector-java-5.1.12-bin.jar;lib/pdfbox-1.7.0.jar;lib/poi-3.6-20091214.jar;lib/poi-scratchpad-3.6-20091214.jar;lib/sax-2.0.1.jar;lib/servlet-api-2.5-20081211.jar;lib/slf4j-api-1.6.1.jar;lib/slf4j-jdk14-1.6.1.jar;lib/webcat-0.1-swf.jar;lib/wstx-asl-3.2.7.jar;lib/xercesImpl.jar;lib/xml-apis.jar</classpath>
                 <source-level>1.6</source-level>
             </compilation-unit>
         </java-data>

source/net/yacy/kelondro/data/meta/URIMetadataRow.java

@@ -163,7 +163,7 @@ public class URIMetadataRow {
         this.entry.setCol(col_dt, dt.isEmpty() ? new byte[]{(byte) 't'} : new byte[]{(byte) dt.charAt(0)});
         final String flags = prop.getProperty("flags", "AAAAAA");
         this.entry.setCol(col_flags, (flags.length() > 6) ? QueryParams.empty_constraint.bytes() : (new Bitfield(4, flags)).bytes());
-        this.entry.setCol(col_lang, UTF8.getBytes(prop.getProperty("lang", "uk")));
+        this.entry.setCol(col_lang, UTF8.getBytes(prop.getProperty("lang", "")));
         this.entry.setCol(col_llocal, Integer.parseInt(prop.getProperty("llocal", "0")));
         this.entry.setCol(col_lother, Integer.parseInt(prop.getProperty("lother", "0")));
         this.entry.setCol(col_limage, Integer.parseInt(prop.getProperty("limage", "0")));
@@ -333,9 +333,9 @@ public class URIMetadataRow {
     public byte[] language() {
         byte[] b = this.entry.getColBytes(col_lang, true);
         if ((b == null || b[0] == (byte)'[') && this.metadata().url != null) {
-            String tld = this.metadata().url.getTLD();
+            String lang = this.metadata().url.language(); // calculate lang by TLD
-            if (tld.length() < 2 || tld.length() > 2) return ASCII.getBytes("en");
+            this.entry.setCol(col_lang, UTF8.getBytes(lang)); //remember calculation
-            return ASCII.getBytes(tld);
+            return ASCII.getBytes(lang);
         }
         return b;
     }
@@ -511,12 +511,20 @@ public class URIMetadataRow {
         public double lat() {
             if (this.latlon == null || this.latlon.isEmpty()) return 0.0d;
             final int p = this.latlon.indexOf(',');
-            return p < 0 ? 0.0f : Double.parseDouble(this.latlon.substring(0, p));
+            if (p < 0) {
+                return 0.0d;
+            } else { // old index entries might contain text "NaN,NaN"
+                return this.latlon.charAt(0) > '9' ? 0.0d : Double.parseDouble(this.latlon.substring(0, p));
+            }
         }
         public double lon() {
             if (this.latlon == null || this.latlon.isEmpty()) return 0.0d;
             final int p = this.latlon.indexOf(',');
-            return p < 0 ? 0.0f : Double.parseDouble(this.latlon.substring(p + 1));
+            if (p < 0) {
+                return 0.0d;
+            } else { // old index entries might contain text "NaN,NaN"
+                return this.latlon.charAt(p + 1) > '9' ? 0.0d : Double.parseDouble(this.latlon.substring(p + 1));
+            }
         }
     }