boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Fix Metadata handling

Browse Source 
- language default on missing lang property to "uk" (fix set to nothing)
-  language set to TLD (added call to existing language calculation from TLD)
-  coordinate number exception on possible lat/lon content of "NaN,NaN"

adjust Netbeans IDE classpath (for Solr/Lucene 4.0.0 jars)
pull/1/head
reger 12 years ago
parent 19d1f474ce
commit 633fbe9188
2 changed files with 616 additions and 608 deletions
Show Stats Download Patch File Download Diff File

2
nbproject/project.xml
Unescape View File

@ -77,7 +77,7 @@
<compilation-unit>
<package-root>source</package-root>
<package-root>htroot</package-root>
<classpath mode="compile">lib/activation.jar;lib/apache-mime4j-0.6.jar;lib/apache-solr-core-3.6.1.jar;lib/apache-solr-solrj-3.6.1.jar;lib/arq-2.8.7.jar;lib/bcmail-jdk15-145.jar;lib/bcprov-jdk15-145.jar;lib/commons-codec-1.6.jar;lib/commons-compress-1.4.1.jar;lib/commons-fileupload-1.2.2.jar;lib/commons-httpclient-3.1.jar;lib/commons-io-2.1.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.1.1.jar;lib/fontbox-1.7.0.jar;lib/geronimo-stax-api_1.0_spec-1.0.1.jar;lib/guava-r05.jar;lib/htmllexer.jar;lib/htmlparser.jar;lib/httpclient-4.2.2.jar;lib/httpcore-4.2.2.jar;lib/httpmime-4.2.2.jar;lib/icu4j-core.jar;lib/iri-0.8.jar;lib/J7Zip-modified.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/jcifs-1.3.15.jar;lib/jcl-over-slf4j-1.6.1.jar;lib/jempbox-1.7.0.jar;lib/jena-2.6.4.jar;lib/jetty-6.1.26-patched-JETTY-1340.jar;lib/jetty-util-6.1.26-patched-JETTY-1340.jar;lib/jsch-0.1.42.jar;lib/json-simple-1.1.jar;lib/jsoup-1.6.3.jar;lib/log4j-1.2.16.jar;lib/log4j-over-slf4j-1.6.1.jar;lib/lucene-analyzers-3.6.1.jar;lib/lucene-core-3.6.1.jar;lib/lucene-highlighter-3.6.1.jar;lib/lucene-phonetic-3.6.1.jar;lib/lucene-spatial-3.6.1.jar;lib/lucene-spellchecker-3.6.1.jar;lib/metadata-extractor-2.4.0-beta-1.jar;lib/mysql-connector-java-5.1.12-bin.jar;lib/pdfbox-1.7.0.jar;lib/poi-3.6-20091214.jar;lib/poi-scratchpad-3.6-20091214.jar;lib/sax-2.0.1.jar;lib/servlet-api-2.5-20081211.jar;lib/slf4j-api-1.6.1.jar;lib/slf4j-jdk14-1.6.1.jar;lib/webcat-0.1-swf.jar;lib/wstx-asl-3.2.7.jar;lib/xercesImpl.jar;lib/xml-apis.jar</classpath>
<classpath mode="compile">lib/activation.jar;lib/apache-mime4j-0.6.jar;lib/apache-solr-core-4.0.0.jar;lib/apache-solr-solrj-4.0.0.jar;lib/arq-2.8.7.jar;lib/bcmail-jdk15-145.jar;lib/bcprov-jdk15-145.jar;lib/commons-codec-1.7.jar;lib/commons-compress-1.4.1.jar;lib/commons-fileupload-1.2.2.jar;lib/commons-httpclient-3.1.jar;lib/commons-io-2.1.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.1.1.jar;lib/fontbox-1.7.0.jar;lib/geronimo-stax-api_1.0_spec-1.0.1.jar;lib/guava-r05.jar;lib/htmllexer.jar;lib/htmlparser.jar;lib/httpclient-4.2.2.jar;lib/httpcore-4.2.2.jar;lib/httpmime-4.2.2.jar;lib/icu4j-core.jar;lib/iri-0.8.jar;lib/J7Zip-modified.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/jcifs-1.3.15.jar;lib/jcl-over-slf4j-1.6.1.jar;lib/jempbox-1.7.0.jar;lib/jena-2.6.4.jar;lib/jetty-6.1.26-patched-JETTY-1340.jar;lib/jetty-util-6.1.26-patched-JETTY-1340.jar;lib/jsch-0.1.42.jar;lib/json-simple-1.1.jar;lib/jsoup-1.6.3.jar;lib/log4j-1.2.16.jar;lib/log4j-over-slf4j-1.6.1.jar;lib/lucene-analyzers-4.0.0.jar;lib/lucene-core-4.0.0.jar;lib/lucene-highlighter-4.0.0.jar;lib/lucene-phonetic-4.0.0.jar;lib/lucene-spatial-4.0.0.jar;lib/lucene-spellchecker-4.0.0.jar;lib/metadata-extractor-2.4.0-beta-1.jar;lib/mysql-connector-java-5.1.12-bin.jar;lib/pdfbox-1.7.0.jar;lib/poi-3.6-20091214.jar;lib/poi-scratchpad-3.6-20091214.jar;lib/sax-2.0.1.jar;lib/servlet-api-2.5-20081211.jar;lib/slf4j-api-1.6.1.jar;lib/slf4j-jdk14-1.6.1.jar;lib/webcat-0.1-swf.jar;lib/wstx-asl-3.2.7.jar;lib/xercesImpl.jar;lib/xml-apis.jar</classpath>
<source-level>1.6</source-level>
</compilation-unit>
</java-data>

20
source/net/yacy/kelondro/data/meta/URIMetadataRow.java
Unescape View File

@ -163,7 +163,7 @@ public class URIMetadataRow {
this.entry.setCol(col_dt, dt.isEmpty() ? new byte[]{(byte) 't'} : new byte[]{(byte) dt.charAt(0)});
final String flags = prop.getProperty("flags", "AAAAAA");
this.entry.setCol(col_flags, (flags.length() > 6) ? QueryParams.empty_constraint.bytes() : (new Bitfield(4, flags)).bytes());
this.entry.setCol(col_lang, UTF8.getBytes(prop.getProperty("lang", "uk")));
this.entry.setCol(col_lang, UTF8.getBytes(prop.getProperty("lang", "")));
this.entry.setCol(col_llocal, Integer.parseInt(prop.getProperty("llocal", "0")));
this.entry.setCol(col_lother, Integer.parseInt(prop.getProperty("lother", "0")));
this.entry.setCol(col_limage, Integer.parseInt(prop.getProperty("limage", "0")));
@ -333,9 +333,9 @@ public class URIMetadataRow {
public byte[] language() {
byte[] b = this.entry.getColBytes(col_lang, true);
if ((b == null || b[0] == (byte)'[') && this.metadata().url != null) {
String tld = this.metadata().url.getTLD();
if (tld.length() < 2 || tld.length() > 2) return ASCII.getBytes("en");
return ASCII.getBytes(tld);
String lang = this.metadata().url.language(); // calculate lang by TLD
this.entry.setCol(col_lang, UTF8.getBytes(lang)); //remember calculation
return ASCII.getBytes(lang);
}
return b;
}
@ -511,12 +511,20 @@ public class URIMetadataRow {
public double lat() {
if (this.latlon == null || this.latlon.isEmpty()) return 0.0d;
final int p = this.latlon.indexOf(',');
return p < 0 ? 0.0f : Double.parseDouble(this.latlon.substring(0, p));
if (p < 0) {
return 0.0d;
} else { // old index entries might contain text "NaN,NaN"
return this.latlon.charAt(0) > '9' ? 0.0d : Double.parseDouble(this.latlon.substring(0, p));
}
}
public double lon() {
if (this.latlon == null || this.latlon.isEmpty()) return 0.0d;
final int p = this.latlon.indexOf(',');
return p < 0 ? 0.0f : Double.parseDouble(this.latlon.substring(p + 1));
if (p < 0) {
return 0.0d;
} else { // old index entries might contain text "NaN,NaN"
return this.latlon.charAt(p + 1) > '9' ? 0.0d : Double.parseDouble(this.latlon.substring(p + 1));
}
}
}

Write Preview
Loading…
Cancel
Save