wttr.in/bin/proxy.py

# vim: fileencoding=utf-8

"""

The proxy server acts as a backend for the wttr.in service.

It caches the answers and handles various data sources transforming their
answers into format supported by the wttr.in service.

If WTTRIN_TEST is specified, it works in a special test mode:
it does not fetch and does not store the data in the cache,
but is using the fake data from "test/proxy-data".

"""
from __future__ import print_function

from gevent.pywsgi import WSGIServer
from gevent.monkey import patch_all

patch_all()

# pylint: disable=wrong-import-position,wrong-import-order
import sys
import os
import time
import json
import hashlib
import re

import requests
import cyrtranslit

from flask import Flask, request

APP = Flask(__name__)


MYDIR = os.path.abspath(os.path.dirname(os.path.dirname("__file__")))
sys.path.append("%s/lib/" % MYDIR)

import proxy_log
import globals
from globals import (
    PROXY_CACHEDIR,
    PROXY_PORT,
    USE_METNO,
    USER_AGENT,
    MISSING_TRANSLATION_LOG,
)
from metno import create_standard_json_from_metno, metno_request
from translations import PROXY_LANGS

# pylint: enable=wrong-import-position

proxy_logger = proxy_log.LoggerWWO(globals.PROXY_LOG_ACCESS, globals.PROXY_LOG_ERRORS)


def is_testmode():
    """Server is running in the wttr.in test mode"""

    return "WTTRIN_TEST" in os.environ


def load_translations():
    """
    load all translations
    """
    translations = {}

    for f_name in PROXY_LANGS:
        f_name = "share/translations/%s.txt" % f_name
        translation = {}
        lang = f_name.split("/")[-1].split(".", 1)[0]
        with open(f_name, "r") as f_file:
            for line in f_file:
                if ":" not in line:
                    continue
                if line.count(":") == 3:
                    _, trans, orig, _ = line.strip().split(":", 4)
                else:
                    _, trans, orig = line.strip().split(":", 3)
                trans = trans.strip()
                orig = orig.strip()

                translation[orig.lower()] = trans
        translations[lang] = translation
    return translations


TRANSLATIONS = load_translations()


def _is_metno():
    return USE_METNO


def _find_srv_for_query(path, query):  # pylint: disable=unused-argument
    if _is_metno():
        return "https://api.met.no"
    return "http://api.worldweatheronline.com"


def _cache_file(path, query):
    """Return cache file name for specified `path` and `query`
    and for the current time.

    Do smooth load on the server, expiration time
    is slightly varied basing on the path+query sha1 hash digest.
    """

    digest = hashlib.sha1(("%s %s" % (path, query)).encode("utf-8")).hexdigest()
    digest_number = ord(digest[0].upper())
    expiry_interval = 60 * (digest_number + 180)

    timestamp = "%010d" % (int(time.time()) // expiry_interval * expiry_interval)
    filename = os.path.join(PROXY_CACHEDIR, timestamp, path, query)

    return filename


def _load_content_and_headers(path, query):
    if is_testmode():
        cache_file = "test/proxy-data/data1"
    else:
        cache_file = _cache_file(path, query)
    try:
        return (
            open(cache_file, "r").read(),
            json.loads(open(cache_file + ".headers", "r").read()),
        )
    except IOError:
        return None, None


def _touch_empty_file(path, query):
    cache_file = _cache_file(path, query)
    cache_dir = os.path.dirname(cache_file)
    if not os.path.exists(cache_dir):
        os.makedirs(cache_dir)
    open(cache_file, "w").write("")


def _save_content_and_headers(path, query, content, headers):
    cache_file = _cache_file(path, query)
    cache_dir = os.path.dirname(cache_file)
    if not os.path.exists(cache_dir):
        os.makedirs(cache_dir)
    open(cache_file + ".headers", "w").write(json.dumps(headers))
    open(cache_file, "wb").write(content)


def translate(text, lang):
    """
    Translate `text` into `lang`.
    If `text` is comma-separated, translate each term independently.
    If no translation found, leave it untouched.
    """

    def _log_unknown_translation(lang, text):
        with open(MISSING_TRANSLATION_LOG % lang, "a") as f_missing_translation:
            f_missing_translation.write(text + "\n")

    if "," in text:
        terms = text.split(",")
        translated_terms = [translate(term.strip(), lang) for term in terms]
        return ", ".join(translated_terms)

    if lang not in TRANSLATIONS:
        _log_unknown_translation(lang, "UNKNOWN_LANGUAGE")
        return text

    if text.lower() not in TRANSLATIONS.get(lang, {}):
        _log_unknown_translation(lang, text)
        return text

    translated = TRANSLATIONS.get(lang, {}).get(text.lower(), text)
    return translated


def cyr(to_translate):
    """
    Transliterate `to_translate` from latin into cyrillic
    """
    return cyrtranslit.to_cyrillic(to_translate)


def _patch_greek(original):
    return original.replace("Ηλιόλουστη/ο", "Ηλιόλουστη")


def add_translations(content, lang):
    """
    Add `lang` translation to `content` (JSON)
    returned by the data source
    """

    if content == "{}":
        return {}

    languages_to_translate = TRANSLATIONS.keys()
    try:
        d = json.loads(content)  # pylint: disable=invalid-name
    except (ValueError, TypeError) as exception:
        print("---")
        print(exception)
        print("---")
        return {}

    try:
        weather_condition = d["data"]["current_condition"][0]["weatherDesc"][0][
            "value"
        ].capitalize()
        d["data"]["current_condition"][0]["weatherDesc"][0]["value"] = weather_condition
        if lang in languages_to_translate:
            d["data"]["current_condition"][0]["lang_%s" % lang] = [
                {"value": translate(weather_condition, lang)}
            ]
        elif lang == "sr":
            d["data"]["current_condition"][0]["lang_%s" % lang] = [
                {
                    "value": cyr(
                        d["data"]["current_condition"][0]["lang_%s" % lang][0]["value"]
                    )
                }
            ]
        elif lang == "el":
            d["data"]["current_condition"][0]["lang_%s" % lang] = [
                {
                    "value": _patch_greek(
                        d["data"]["current_condition"][0]["lang_%s" % lang][0]["value"]
                    )
                }
            ]
        elif lang == "sr-lat":
            d["data"]["current_condition"][0]["lang_%s" % lang] = [
                {"value": d["data"]["current_condition"][0]["lang_sr"][0]["value"]}
            ]

        fixed_weather = []
        for w in d["data"]["weather"]:  # pylint: disable=invalid-name
            fixed_hourly = []
            for h in w["hourly"]:  # pylint: disable=invalid-name
                weather_condition = h["weatherDesc"][0]["value"]
                if lang in languages_to_translate:
                    h["lang_%s" % lang] = [
                        {"value": translate(weather_condition, lang)}
                    ]
                elif lang == "sr":
                    h["lang_%s" % lang] = [
                        {"value": cyr(h["lang_%s" % lang][0]["value"])}
                    ]
                elif lang == "el":
                    h["lang_%s" % lang] = [
                        {"value": _patch_greek(h["lang_%s" % lang][0]["value"])}
                    ]
                elif lang == "sr-lat":
                    h["lang_%s" % lang] = [{"value": h["lang_sr"][0]["value"]}]
                fixed_hourly.append(h)
            w["hourly"] = fixed_hourly
            fixed_weather.append(w)
        d["data"]["weather"] = fixed_weather

        content = json.dumps(d)
    except (IndexError, ValueError) as exception:
        print(exception)
    return content


def _fetch_content_and_headers(path, query_string, **kwargs):
    content, headers = _load_content_and_headers(path, query_string)

    if content is None:
        srv = _find_srv_for_query(path, query_string)
        url = "%s/%s?%s" % (srv, path, query_string)

        attempts = 10
        response = None
        error = ""
        while attempts:
            try:
                response = requests.get(url, timeout=2, **kwargs)
            except requests.ReadTimeout:
                attempts -= 1
                continue
            try:
                data = json.loads(response.content)
                error = data.get("data", {}).get("error", "")
                if error:
                    try:
                        error = error[0]["msg"]
                    except (ValueError, IndexError):
                        error = "invalid error format: %s" % error
                break
            except ValueError:
                attempts -= 1
                error = "invalid response"

        proxy_logger.log(query_string, error)
        _touch_empty_file(path, query_string)
        if response:
            headers = {}
            headers["Content-Type"] = response.headers["content-type"]
            _save_content_and_headers(path, query_string, response.content, headers)
            content = response.content
        else:
            content = "{}"
    else:
        print("cache found")
    return content, headers


def _make_query(path, query_string):

    if _is_metno():
        path, query, days = metno_request(path, query_string)
        if USER_AGENT == "":
            raise ValueError(
                "User agent must be set to adhere to metno ToS: https://api.met.no/doc/TermsOfService"
            )
        content, headers = _fetch_content_and_headers(
            path, query, headers={"User-Agent": USER_AGENT}
        )
        content = create_standard_json_from_metno(content, days)
    else:
        # WWO tweaks
        query_string += "&extra=localObsTime"
        query_string += "&includelocation=yes"
        content, headers = _fetch_content_and_headers(path, query_string)

    return content, headers


def _normalize_query_string(query_string):
    # Normalized query string has the following fixes:
    # 1. Uses , for the coordinates separation
    # 2. Limits number of digits after .

    coord_match = re.search(r"q=[^&]*", query_string)
    coords_str = coord_match.group(0)
    coords = re.findall(r"[-0-9.]+", coords_str)
    if len(coords) != 2:
        return query_string

    lat = str(round(float(coords[0]), 2))
    lng = str(round(float(coords[1]), 2))
    query_string = re.sub(r"q=[^&]*", "q=" + lat + "," + lng, query_string)
    print(query_string)

    # nqs = query_string.replace("%2C", ",")
    # lat, lng = nqs.split(",", 1)
    # nqs = f"{lat:.2f},{lng:.2f}"

    return query_string


@APP.route("/<path:path>")
def proxy(path):
    """
    Main proxy function. Handles incoming HTTP queries.
    """

    lang = request.args.get("lang", "en")
    query_string = request.query_string.decode("utf-8")
    query_string = _normalize_query_string(query_string)
    query_string = query_string.replace("sr-lat", "sr")
    query_string = query_string.replace("lang=None", "lang=en")
    content = ""
    headers = ""

    content, headers = _make_query(path, query_string)

    # _log_query(path, query_string, error)

    content = add_translations(content, lang)

    return content, 200, headers


if __name__ == "__main__":
    # app.run(host='0.0.0.0', port=5001, debug=False)
    # app.debug = True

    if len(sys.argv) == 1:
        bind_addr = "0.0.0.0"
        SERVER = WSGIServer((bind_addr, PROXY_PORT), APP)
        SERVER.serve_forever()
    else:
        print("running single request from command line arg")
        APP.testing = True
        with APP.test_client() as c:
            resp = c.get(sys.argv[1])
            print("Status: " + resp.status)
            # print('Headers: ' + dumps(resp.headers))
            print(resp.data.decode("utf-8"))