"use strict";
// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2019-2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
    if (mod && mod.__esModule) return mod;
    var result = {};
    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
    __setModuleDefault(result, mod);
    return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const assert_1 = __importDefault(require("assert"));
const string_interp_1 = __importDefault(require("string-interp"));
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const node_gettext_1 = __importDefault(require("node-gettext"));
const gettextParser = __importStar(require("gettext-parser"));
const Units = __importStar(require("thingtalk-units"));
const base_1 = __importDefault(require("./tokenizer/base"));
function capitalize(word) {
    return word[0].toUpperCase() + word.substring(1);
}
/**
 * Base class for all code that is specific to a certain natural language
 * in Genie.
 */
class DefaultLanguagePack {
    // do not use ngettext, use ICU syntax `${foo:plural:one{}other{}}` instead
    constructor(locale) {
        this.locale = locale;
        this._gt = new node_gettext_1.default();
        this._gt.setLocale(locale);
        this.gettext = this._gt.dgettext.bind(this._gt, 'genie-toolkit');
        this._ = this.gettext;
        if (!/^en(-|$)/.test(locale))
            this._loadTranslations();
    }
    _loadTranslations() {
        // try the path relative to our build location first (in dist/lib/dialogue-agent)
        let modir = path.resolve(path.dirname(module.filename), '../../../po');
        if (!fs.existsSync(modir)) {
            // if that fails, try the path relative to our source location
            // (running with ts-node)
            modir = path.resolve(path.dirname(module.filename), '../../po');
            assert_1.default(fs.existsSync(modir));
        }
        const split = this.locale.split(/[-_.@]/);
        let mo = modir + '/' + split.join('_') + '.mo';
        while (!fs.existsSync(mo) && split.length) {
            split.pop();
            mo = modir + '/' + split.join('_') + '.mo';
        }
        if (split.length === 0) {
            console.error(`No translations found for locale ${this.locale}`);
            return;
        }
        try {
            const loaded = gettextParser.mo.parse(fs.readFileSync(mo), 'utf-8');
            this._gt.addTranslations(this.locale, 'genie-toolkit', loaded);
        }
        catch (e) {
            console.log(`Failed to load translations for ${this.locale}: ${e.message}`);
        }
    }
    /**
     * Return an instance of the tokenizer used by this language.
     */
    getTokenizer() {
        if (this._tokenizer)
            return this._tokenizer;
        return this._tokenizer = new base_1.default();
    }
    /**
     * Apply final touches to a newly generated synthetic sentence
     *
     * This function should correct coreferences, conjugations and other
     * grammar/readability issues that are too inconvenient to prevent
     * using the templates.
     */
    postprocessSynthetic(sentence, program, rng, forTarget) {
        return sentence;
    }
    /**
     * Convert a tokenized sentence back into a correctly spaced, correctly
     * punctuated sentence.
     *
     * This is a low-level method called by {@link DefaultLanguagePack#detokenizeSentence}.
     * It can be used to detokenize one token at a time.
     */
    detokenize(sentence, prevtoken, token) {
        if (sentence && !this._NO_SPACE_TOKENS.has(token))
            sentence += ' ';
        sentence += token;
        return sentence;
    }
    /**
     * Convert a tokenized sentence back into a correctly spaced, correctly
     * punctuated sentence.
     *
     * This is used for sentences presented to an MTurk worker for paraphrasing,
     * and it is used for the agent replies before they are shown to the user.
     */
    detokenizeSentence(tokens) {
        let sentence = '';
        let prevToken = '';
        for (const token of tokens) {
            sentence = this.detokenize(sentence, prevToken, token);
            prevToken = token;
        }
        return sentence;
    }
    /**
     * Retrieve the list of units to use for a given base unit. This defaults
     * to metric units, but subclasses can override to choose a different
     * unit.
     *
     * The best unit for a given value (i.e., the one with the fewest digits)
     * will be chosen to display. If there are ties, the first unit will be chosen.
     */
    _getPossibleUnits(baseUnit) {
        switch (baseUnit) {
            case 'ms':
                return ['ms', 's', 'min', 'h', 'day', 'week', 'mon', 'year', 'decade', 'century'];
            case 'm':
                return ['mm', 'cm', 'm', 'km', 'ly'];
            case 'm2':
                return ['mm2', 'cm2', 'm2', 'km2'];
            case 'm3':
                // prefer liquid over solid units
                return ['ml', 'cl', 'hl', 'l', 'mm3', 'cm3', 'm3', 'km3'];
            case 'mps':
                return ['mps', 'kmph'];
            case 'kg':
                return ['mg', 'g', 'kg'];
            case 'Pa':
                return ['Pa', 'bar'];
            case 'C':
                return ['C', 'K'];
            case 'kcal':
                return ['kcal', 'kJ'];
            case 'byte':
                // avoid IEC units
                return ['byte', 'KB', 'MB', 'GB', 'TB'];
            case 'W':
                return ['W', 'kW'];
            default:
                return [baseUnit];
        }
    }
    _getBestUnit(value, baseUnit) {
        // default to metric units
        const possibleUnits = this._getPossibleUnits(baseUnit).map((unit) => {
            const transformed = Units.transformFromBaseUnit(value, unit);
            // score how close the representation is to 1
            // add a penalty for 0.something
            const cost = Math.abs(Math.log10(transformed)) + (2 * +(Math.abs(transformed) < 1));
            return { unit, cost };
        });
        possibleUnits.sort((a, b) => {
            return a.cost - b.cost;
        });
        return possibleUnits[0].unit;
    }
    _measureToString(value, unit, precision = 1) {
        const transformed = Units.transformFromBaseUnit(value, unit);
        // TODO use value.toLocaleString() with unit formatting
        //      it depends on https://github.com/tc39/proposal-unified-intl-numberformat
        //      which is part of ES2020 (so node 14 or later?)
        return this._numberToString(transformed, precision) + ' ' + unit;
    }
    _numberToString(value, precision = 1) {
        return value.toLocaleString(this.locale, {
            minimumFractionDigits: 0,
            maximumFractionDigits: precision
        });
    }
    _dateToString(date, timezone, options) {
        if (!options) {
            options = {
                weekday: undefined,
                day: 'numeric',
                month: 'long',
                year: 'numeric',
            };
        }
        options.timeZone = timezone;
        return date.toLocaleDateString(this.locale, options);
    }
    /**
     * Convert a date object to a user-visible string, displaying _only_ the time part.
     *
     * This is a small wrapper over {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toLocaleString|Date.toLocaleString}
     * that applies the correct timezone.
     *
     * @param {Date} date - the time to display
     * @param {Object} [options] - additional options to pass to `toLocaleString`
     * @return {string} the formatted time
     */
    _timeToString(date, timezone, options) {
        if (!options) {
            options = {
                hour: 'numeric',
                minute: '2-digit',
                timeZoneName: undefined,
            };
            if (date.getSeconds() !== 0)
                options.second = '2-digit';
        }
        options.timeZone = timezone;
        return date.toLocaleTimeString(this.locale, options);
    }
    _dateAndTimeToString(date, timezone, options = {}) {
        options.timeZone = timezone;
        return date.toLocaleString(this.locale, options);
    }
    getDefaultTemperatureUnit() {
        return 'C';
    }
    displayEntity(token, entityValue, delegate) {
        if (token.startsWith('GENERIC_ENTITY_')) {
            const entity = entityValue;
            return (entity.display || entity.value);
        }
        if (token.startsWith('QUOTED_STRING_'))
            return String(entityValue).replace(/[ \t\v\r\n]+/g, ' ').trim();
        if (token.startsWith('USERNAME_'))
            return '@' + entityValue;
        if (token.startsWith('HASHTAG_'))
            return '#' + entityValue;
        if (token.startsWith('MEASURE_')) {
            const [, baseUnit] = /^MEASURE_([A-Za-z0-9_]+)_[0-9]+$/.exec(token);
            const entity = entityValue;
            let fromUnit = entity.unit;
            if (fromUnit.startsWith('default')) {
                switch (fromUnit) {
                    case 'defaultTemperature':
                        fromUnit = delegate.getPreferredUnit('temperature') || this.getDefaultTemperatureUnit();
                        break;
                    default:
                        throw new TypeError('Unexpected default unit ' + fromUnit);
                }
            }
            const value = Units.transformToBaseUnit(entity.value, fromUnit);
            let toUnit;
            if (baseUnit === 'C') {
                const maybeUnit = delegate.getPreferredUnit('temperature');
                toUnit = maybeUnit || this.getDefaultTemperatureUnit();
            }
            else {
                toUnit = this._getBestUnit(value, baseUnit);
            }
            return this._measureToString(value, toUnit);
        }
        if (token.startsWith('NUMBER_'))
            return this._numberToString(entityValue);
        if (token.startsWith('CURRENCY_')) {
            const entity = entityValue;
            const options = {
                style: 'currency',
                currency: entity.unit.toUpperCase()
            };
            return entity.value.toLocaleString(this.locale, options);
        }
        if (token.startsWith('TIME_')) {
            const entity = entityValue;
            const time = new Date;
            time.setHours(entity.hour);
            time.setMinutes(entity.minute);
            time.setSeconds(entity.second);
            time.setMilliseconds(0);
            return this._timeToString(time, delegate.timezone);
        }
        if (token.startsWith('DATE_')) {
            const date = entityValue;
            // check for midnight local, and midnight UTC, to mean date without time
            if ((date.getHours() === 0 && date.getMinutes() === 0 && date.getSeconds() === 0) ||
                (date.getUTCHours() === 0 && date.getUTCMinutes() === 0 && date.getUTCSeconds() === 0))
                return this._dateToString(date, delegate.timezone);
            else
                return this._dateAndTimeToString(date, delegate.timezone);
        }
        if (token.startsWith('LOCATION_')) {
            const loc = entityValue;
            if (loc.display) {
                return loc.display;
            }
            else {
                return string_interp_1.default(this._("[Latitude: ${loc.latitude:.3} deg, Longitude: ${loc.longitude:.3} deg]"), { loc }, { locale: this.locale, timezone: delegate.timezone }) || '';
            }
        }
        return String(entityValue);
    }
    /**
     * Post-process a sentence generated by the neural NLG for display to
     * the user.
     *
     * This includes true-casing, detokenizing, and replacing entity tokens
     * with actual values.
     */
    postprocessNLG(answer, entities, delegate) {
        // small hack, fix untokenized commas generated by describe (which uses I18n.ListFormat)
        answer = answer.replace(/([^ ]),/g, '$1 ,');
        const tokens = answer.split(' ').map((token) => {
            if (token in entities)
                return this.displayEntity(token, entities[token], delegate);
            // capitalize certain tokens that should be capitalized in English
            if (this.MUST_CAPITALIZE_TOKEN.has(token))
                return capitalize(token);
            return token;
        });
        answer = this.detokenizeSentence(tokens);
        // simple true-casing: uppercase all letters at the beginning of the sentence
        // and after a period, question or exclamation mark
        answer = answer.replace(/(^|[.?!] )([a-z])/g, (_, prefix, letter) => prefix + letter.toUpperCase());
        // remove duplicate spaces
        answer = answer.replace(/\s+/g, ' ');
        // sometimes, we end up with two periods at the end of a sentence, because
        // a #[result] phrase includes a period, or because a value includes a period
        // (this happens with jokes)
        // clean that up
        answer = answer.replace(/\.\.$/, '.');
        return answer;
    }
    /**
     * Convert a word or phrase to plural form.
     *
     * This function should return `undefined` if there is no plural form
     * of the given phrase.
     */
    pluralize(phrase) {
        // no plural form
        return undefined;
    }
    /**
     * Convert a word or verb phrase to past tense.
     *
     * This function should return `undefined` if there is no past tense
     * of the given phrase.
     */
    toVerbPast(phrase) {
        // no past
        return undefined;
    }
    /**
     * Convert a phrase from the side of the user to the side of the agent.
     *
     * This function takes a phrase that talks about "my devices" (uttered by
     * the user) and converts to a phrase that talks about "your devices"
     * uttered by the agent.
     */
    toAgentSideUtterance(phrase) {
        // by default, no change
        return phrase;
    }
    /**
     * Filter out words that cannot be in the dataset, because they would be
     * either tokenized/preprocessed out or they are unlikely to be used with
     * voice.
     */
    isGoodWord(word) {
        // all words are good words
        return true;
    }
    /**
     * Filter out phrases that should not be used as a parameter on their own.
     *
     * This is mainly used to remove phrases that would be syntatically
     * ambiguous, and would not be immediately recognized as a parameter.
     * A good rule of thumb is to filter out all phrases that consist entirely
     * of stop words.
     */
    isGoodSentence(sentence) {
        // all sentences are good words
        return true;
    }
    /**
     * Check if a numeric phrase is valid for the given language.
     *
     * This covers ASCII digits as well as language-specific number systems,
     * like Arabic digits.
     */
    isGoodNumber(number) {
        return /^([0-9|\u0660-\u0669]+)$/.test(number);
    }
    /**
     * Check if a phrase looks like a person name.
     *
     * This is a coarse check that is used to override
     * {@link DefaultLanguagePack#isGoodWord} to account for foreign person
     * names and loan words.
     */
    isGoodPersonName(word) {
        return this.isGoodWord(word) || /^(\w+\s\w\s?\.)$/.test(word);
    }
    /**
     * Add a definite article ("the") to the given phrase.
     *
     * If the language has no concept of definite articles, this function
     * must return `undefined`.
     */
    addDefiniteArticle(phrase) {
        return undefined;
    }
}
exports.default = DefaultLanguagePack;
/**
 * Override the canonical form of argument names for synthetic generation
 * (to generate filters and projections)
 *
 * More than one form can be provided for each argument name, in which case
 * all are used.
 */
DefaultLanguagePack.prototype.ARGUMENT_NAME_OVERRIDES = {};
/**
 * Tokens that can be ignored in the names of entities, by entity type.
 *
 * This should cover abbreviations, prefixes and suffixes that are usually
 * omitted in colloquial speech.
 */
DefaultLanguagePack.prototype.IGNORABLE_TOKENS = {
    'sportradar': ['fc', 'ac', 'us', 'if', 'as', 'rc', 'rb', 'il', 'fk', 'cd', 'cf'],
    'tt:stock_id': ['l.p.', 's.a.', 'plc', 'n.v', 's.a.b', 'c.v.'],
    'org:freedesktop:app_id': ['gnome']
};
/**
 * Interchangeable abbreviations for entity names
 *
 * Each entry in this array is a set (in array form) of abbreviations with the same
 * meaning; while expanding parameters, one of the possible forms is chosen at random
 *
 * Use this to fix tokenization inconsistencies in the entity database, to add
 * colloquial forms, and to add robustness to punctuation.
 */
DefaultLanguagePack.prototype.ABBREVIATIONS = {};
/**
 * Tokens that should not be preceded by a space.
 * This is used by the default {@link DefaultLanguagePack#detokenize}
 * implementation.
 */
DefaultLanguagePack.prototype._NO_SPACE_TOKENS = new Set(['.', ',', '?', '!', ':']);
/**
 * All the different forms in which MTurk workers write "no idea" for a sentence
 * they don't understand.
 *
 * This is usually empirically collected by looking at the results and finding
 * sentences that don't validate or are too short.
 */
DefaultLanguagePack.prototype.NO_IDEA = [];
DefaultLanguagePack.prototype.CHANGE_SUBJECT_TEMPLATES = [];
/**
 * Different ways to add an explicit reference to a skill name for a command.
 */
DefaultLanguagePack.prototype.SINGLE_DEVICE_TEMPLATES = [];
/**
 * A regular expression used to identify a definite article ("the") at the
 * beginning of a (tokenized) phrase.
 *
 * A language without definite articles should leave this to `undefined`.
 */
DefaultLanguagePack.prototype.DEFINITE_ARTICLE_REGEXP = undefined;
/**
 * A set of tokens that must always be capitalized.
 */
DefaultLanguagePack.prototype.MUST_CAPITALIZE_TOKEN = new Set([
    'spotify', 'twitter', 'yelp', 'google', 'facebook',
]);
//# sourceMappingURL=default.js.map