yacy_search_server/source/net/yacy/document/content/DCEntry.java

// DCEntry.java
// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 15.04.2009 on http://yacy.net
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA


package net.yacy.document.content;

import com.ibm.icu.util.ULocale;

import java.io.IOException;
import java.io.OutputStreamWriter;
import java.net.MalformedURLException;
import java.nio.charset.StandardCharsets;
import java.text.Collator;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.TreeMap;

import org.apache.solr.common.params.MultiMapSolrParams;

import net.yacy.cora.date.ISO8601Formatter;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.lod.vocabulary.DublinCore;
import net.yacy.cora.lod.vocabulary.Geo;
import net.yacy.cora.util.CommonPattern;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.document.Document;
import net.yacy.document.parser.genericParser;
import net.yacy.search.schema.CollectionSchema;

public class DCEntry extends MultiMapSolrParams {

    private static final long    serialVersionUID = -2050291583515701559L;

    // use a collator to relax when distinguishing between lowercase und uppercase letters
    private static final Collator insensitiveCollator = Collator.getInstance(Locale.US);
    static {
        insensitiveCollator.setStrength(Collator.SECONDARY);
        insensitiveCollator.setDecomposition(Collator.NO_DECOMPOSITION);
    }
    public  static final DCEntry poison = new DCEntry();

    public DCEntry() {
        super(new TreeMap<String, String[]>((Collator) insensitiveCollator.clone()));
    }

    public DCEntry(
            DigestURL url,
            Date date,
            String title,
            String author,
            String body,
            double lat,
            double lon
            ) {
        super(new TreeMap<String, String[]>((Collator) insensitiveCollator.clone()));
        this.getMap().put(DublinCore.Identifier.getURIref(), new String[]{url.toNormalform(true)});
        this.getMap().put(DublinCore.Date.getURIref(), new String[]{ISO8601Formatter.FORMATTER.format(date)});
        this.getMap().put(DublinCore.Title.getURIref(), new String[]{title});
        this.getMap().put(DublinCore.Creator.getURIref(), new String[]{author});
        this.getMap().put(DublinCore.Description.getURIref(), new String[]{body});
        this.getMap().put(Geo.Lat.getURIref(), new String[]{Double.toString(lat)});
        this.getMap().put(Geo.Long.getURIref(), new String[]{Double.toString(lon)});
    }

    /*
    DC according to rfc 5013

    * dc_title
    * dc_creator
    * dc_subject
    * dc_description
    * dc_publisher
    dc_contributor
    dc_date
    dc_type
    * dc_format
    * dc_identifier
    * dc_source
    dc_language
    dc_relation
    dc_coverage
    dc_rights
         */
    public Date getDate() {
        String d = this.get("docdatetime");
        if (d == null) d = this.get("date");
        if (d == null) d = this.get(DublinCore.Date.getURIref());
        if (d == null) d = this.get("last-modified");
        if (d == null) return null;
        if (d.isEmpty()) return null;
        try {
            Date x = ISO8601Formatter.FORMATTER.parse(d, 0).getTime();
            Date now = new Date();
            return x.after(now) ? now : x;
        } catch (final ParseException e) {
            ConcurrentLog.logException(e);
            return new Date();
        }
    }

    /**
     * get Identifier (url) (so far only used for surrogate processing)
     * @param useRelationAsAlternative true = take relation if no identifier resolves to url
     * @return this entry identifier url
     */
    public DigestURL getIdentifier(boolean useRelationAsAlternative) {
        // identifier may be included multiple times (with all kinds of syntax - example is from on record)
        // <dc:identifier>Astronomy and Astrophysics, 539, A99, 2012</dc:identifier>
        // <dc:identifier>http://hdl.handle.net/2104/8302</dc:identifier>
        // <dc:identifier>10.1051/0004-6361/201117940</dc:identifier>
        String u = this.get("url");
        if (u == null) u = this.get("sku");
        
        if (u == null) {
            final String[] urls = this.getParams(DublinCore.Identifier.getURIref());
            if (urls == null) {
                return useRelationAsAlternative ? getRelation() : null;
            }
            if (urls.length > 0) { // check best also with 1 in case it's not http urn
                // select one that fits
                u = bestU(urls);
            }
        }

        try {
            return new DigestURL(u);
        } catch (final MalformedURLException e) {
            if (useRelationAsAlternative) {
                DigestURL relation = this.getRelation();
                if (relation != null) return relation;
                ConcurrentLog.warn("DCEntry", "getIdentifier: url is bad, relation also: " + e.getMessage());
            }
            ConcurrentLog.warn("DCEntry", "getIdentifier: url is bad: " + e.getMessage());
            return null;
        }
    }

    public DigestURL getRelation() {
        String u = this.get(DublinCore.Relation.getURIref());
        if (u == null) return null;
        String[] urls = CommonPattern.SEMICOLON.split(u);
        if (urls.length > 1) {
            // select one that fits
            u = bestU(urls);
        }
        try {
            return new DigestURL(u);
        } catch (final MalformedURLException e) {
            ConcurrentLog.warn("DCEntry", "getRelation: url is bad: " + e.getMessage());
            return null;
        }
    }

    /**
     * From the given list of urls selects the best to access the resource,
     * possibly expanding short identifier for doi nbn or itef with resolved url
     * @param urls list of identifier / url
     * @return best/resolved url
     */
    private String bestU(String[] urls) {
        if (urls.length > 1) { // with only one ... no choice
            for (String uu: urls) {
                if (uu.startsWith("http://") && (uu.endsWith(".html") || uu.endsWith(".htm") || uu.endsWith(".pdf") || uu.endsWith(".doc") || uu.endsWith(".rss") || uu.endsWith(".xml"))) return uu;
            }
            for (String uu: urls) {
                if (uu.startsWith("http://")) return uu;
            }
            for (String uu: urls) {
                if (uu.startsWith("https://")) return uu;
            }
            for (String uu: urls) {
                if (uu.startsWith("ftp://")) return uu;
            }
        } // but check urn:/doi: resolve
        for (String uu: urls) {
            //doi identifier can be resolved through dx.doi.org:
            //   http://dx.doi.org/doi:10.12775/SIT.2010.010
            //or http://dx.doi.org/10.12775/SIT.2010.010
            if (uu.startsWith("DOI:") || uu.startsWith("doi:")) // saw it upper & lower-case
                return "http://dx.doi.org/" + uu;

            //urn identifier koennen ueber den resolver der d-nb aufgeloest werden:
            //http://nbn-resolving.de/urn:nbn:de:bsz:960-opus-1860
            if (uu.startsWith("urn:nbn:")) return "http://nbn-resolving.de/" + uu;

            // resolver service http://wm-urn.org/ (for nbn, isbn,issn,rfc) example http://wm-urn.org/urn:ietf:rfc:2141
            if (uu.startsWith("urn:ietf:rfc")) return "http://wm-urn.org/" + uu;
        }
        return urls[0];
    }

    //modified by copperdust; Ukraine, 2012
    public String getLanguage() {//final language computation
        String l = this.get(DublinCore.Language.getURIref());//from document metainfo
        // OAI uses often 3-char languages (ISO639-2) convert to ISO639-1 2-char code)
        if (l != null && l.length() == 3) {
            if (l.startsWith("ger") || l.startsWith("deu")) l = "de";
            else if (l.startsWith("eng")) l = "en";
            else if (l.startsWith("rus")) l = "ru";
            else if (l.startsWith("jpn")) l = "ja";
            else if (l.startsWith("ita")) l = "it";
            else if (l.startsWith("por")) l = "pt";
            else if (l.startsWith("pol")) l = "pl";
            else if (l.startsWith("spa")) l = "es";
            else if (l.startsWith("ukr")) l = "uk";
            else if (l.startsWith("chi") || l.startsWith("zho")) l = "zh";
            else if (l.startsWith("fre") || l.startsWith("fra")) l = "fr";
            else if (l.startsWith("eus") || l.startsWith("baq")) l = "eu";
            else if (l.startsWith("gre") || l.startsWith("ell")) l = "el";
            else {
                // icu.ULocale performs a normalization (of ISO639-2/T) to ISO639-1 2-char language code
                // (fyi: ISO639-2 allows (T)erminology and (B)ibliographic (e.g. chi=zh and zho=zh), ICU handles (T) )
                ULocale loc = new ULocale(l);
                l = loc.getLanguage();
            }
            return l;
        }
        if (l == null) l = getIdentifier(true).language(); // determine from identifier-url.TLD
        if (l == null) return this.get("language");//from TLD
        return l;
    }

    public String getType() {
        String t = this.get(DublinCore.Type.getURIref());
        if (t == null) return "";
        return t;
    }

    public String getFormat() {
        String t = this.get(DublinCore.Format.getURIref());
        if (t == null) return "";
        return t;
    }

    public String getSource() {
        String t = this.get(DublinCore.Source.getURIref());
        if (t == null) return "";
        return t;
    }

    public String getRights() {
        String t = this.get(DublinCore.Rights.getURIref());
        if (t == null) return "";
        return t;
    }

    public String getTitle() {
        String t = this.get("title");
        if (t == null) t = this.get(DublinCore.Title.getURIref());
        t = stripCDATA(t);
        if (t == null) return "";
        return t;
    }

    public String getPublisher() {
        String t = this.get(DublinCore.Publisher.getURIref());
        t = stripCDATA(t);
        if (t == null) return "";
        return t;
    }

    public String getCreator() {
        String t = this.get("author");
        if (t == null) t = this.get(DublinCore.Creator.getURIref());
        t = stripCDATA(t);
        if (t == null) return "";
        return t;
    }

    public List<String> getDescriptions() {
        String[] t = this.getParams(DublinCore.Description.getURIref());
        List<String> descriptions = new ArrayList<String>();
        if (t == null) return descriptions;
        for (String s: t) descriptions.add(stripCDATA(s));
        return descriptions;
    }

    /**
     * return list of subjects (keywords)
     * @return string list or null
     */
    public String[] getSubject() {
        String t = this.get("categories");
        String[] tx;
        if (t != null) {
            t = stripCDATA(t);
            return CommonPattern.SEMICOLON.split(t);
        }
        tx = this.getParams(DublinCore.Subject.getURIref());
        
        if (tx != null) {
            for (int i = 0; i < tx.length; i++) {
                tx[i] = stripCDATA(tx[i]);
            }
        }
        return tx;
    }

    public double getLon() {
        String t = this.get(Geo.Long.getURIref());
        if (t == null) t = this.get("geo:lon"); // try geo:long with possible typing error
        t = stripCDATA(t);
        if (t == null) return 0.0d;
        return Double.parseDouble(t);
    }

    public double getLat() {
        String t = this.get(Geo.Lat.getURIref());
        t = stripCDATA(t);
        if (t == null) return 0.0d;
        return Double.parseDouble(t);
    }

    private static String stripCDATA(String s) {
        if (s == null) return null;
        s = s.trim();
        if (s.startsWith("<![CDATA[")) s = s.substring(9);
        if (s.endsWith("]]")) s = s.substring(0, s.length() - 2);
        return s;
    }

    public Document document() {
        HashSet<String> languages = new HashSet<String>();
        languages.add(getLanguage());
        List<String> t = new ArrayList<String>(1);
        t.add(getTitle());
        
        // for processing during indexing, embed entry as source scraperObject in a standard parserobj object
        genericParser parserobj = new genericParser(); // init the simplest parser with DCEntry as source/scraperObject used during indexing

        Document document = new Document(
            getIdentifier(true),
            "text/html",
            StandardCharsets.UTF_8.name(),
            parserobj,
            languages,
            getSubject(), // might be null
            t,
            getCreator(),
            getPublisher(),
            null,
            getDescriptions(),
            getLon(), getLat(),
            get(CollectionSchema.text_t.name(), ""),
            null,
            null,
            null,
            false,
            getDate());
        document.setScraperObject(this); // TODO: used during indexing to access some possible but special YaCy meta tags in surrogate source ( <md:solrfilename>value ) -> optimize/find alternative
        return document;
    }

    public void writeXML(OutputStreamWriter os) throws IOException {
        Document doc = document();
        if (doc != null) {
            doc.writeXML(os);
        }
    }
}