"use strict";
// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Silei Xu <silei@cs.stanford.edu>
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
    if (mod && mod.__esModule) return mod;
    var result = {};
    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
    __setModuleDefault(result, mod);
    return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.execute = exports.initArgparse = void 0;
const fs = __importStar(require("fs"));
const util_1 = __importDefault(require("util"));
const assert_1 = __importDefault(require("assert"));
const ThingTalk = __importStar(require("thingtalk"));
const thingtalk_1 = require("thingtalk");
const StreamUtils = __importStar(require("../../../lib/utils/stream-utils"));
const base_canonical_generator_1 = __importDefault(require("../lib/base-canonical-generator"));
const misc_utils_1 = require("../../../lib/utils/misc-utils");
const utils_1 = require("../lib/utils");
const utils_2 = require("./utils");
const manual_annotations_1 = require("./manual-annotations");
function argnameFromLabel(label) {
    return utils_1.snakecase(label)
        .replace(/'/g, '') // remove apostrophe
        .replace(/,/g, '') // remove comma
        .replace(/_\/_/g, '_or_') // replace slash by or
        .replace(/[(|)]/g, '') // replace parentheses
        .normalize("NFD").replace(/[\u0300-\u036f]/g, ""); // remove accent
}
function getElementType(type) {
    if (type.isArray)
        return getElementType(type.elem);
    return type;
}
async function retrieveProperties(domain, properties) {
    let list = properties.includes('default') ? await utils_2.getPropertyList(domain) : [];
    for (let property of properties) {
        if (property === 'none')
            continue;
        if (property === 'default')
            continue;
        if (property.startsWith('-')) {
            property = property.slice(1);
            let index = list.indexOf(property);
            if (index > -1)
                list.splice(index, 1);
        }
        else if (!list.includes(property)) {
            list.push(property);
        }
    }
    return list;
}
class SchemaProcessor {
    constructor(domains, domainCanonicals, propertiesByDomain, requiredPropertiesByDomain, output, outputEntities, manual, wikidataLabels, schemaorgManifest) {
        this._domains = domains;
        this._domainCanonicals = domainCanonicals;
        this._propertiesByDomain = propertiesByDomain;
        this._requiredPropertiesByDomain = requiredPropertiesByDomain;
        this._output = output;
        this._outputEntities = outputEntities;
        this._entities = utils_1.DEFAULT_ENTITIES.slice();
        this._manual = manual;
        this._wikidataLabels = wikidataLabels;
        this._schemaorgManifest = schemaorgManifest;
        this._schemaorgProperties = {};
    }
    async _getType(domain, domainLabel, property, propertyLabel) {
        if (property in manual_annotations_1.PROPERTY_TYPE_OVERRIDE)
            return manual_annotations_1.PROPERTY_TYPE_OVERRIDE[property];
        const elemType = await this._getElemType(domain, domainLabel, property, propertyLabel);
        if (manual_annotations_1.PROPERTY_FORCE_ARRAY.has(property))
            return new thingtalk_1.Type.Array(elemType);
        if (manual_annotations_1.PROPERTY_FORCE_NOT_ARRAY.has(property))
            return elemType;
        if (elemType.isEntity && elemType.type === 'tt:picture')
            return new thingtalk_1.Type.Array(elemType);
        // TODO: decide if an property has an array type based on data
        return elemType;
    }
    async _getElemType(domain, domainLabel, property, propertyLabel) {
        if (manual_annotations_1.PROPERTY_TYPE_SAME_AS_SUBJECT.has(property))
            return new thingtalk_1.Type.Entity(`org.wikidata:${utils_1.snakecase(domainLabel)}`);
        const enumEntries = await utils_2.getOneOfConstraint(property);
        if (enumEntries.length > 0)
            return new thingtalk_1.Type.Enum(enumEntries.map(utils_1.cleanEnumValue));
        const classes = await utils_2.getClasses(property);
        if (classes.includes('Q18636219')) // Wikidata property with datatype 'time'
            return thingtalk_1.Type.Date;
        if (classes.includes('Q18616084')) // Wikidata property to indicate a language
            return new thingtalk_1.Type.Entity('tt:iso_lang_code');
        if (propertyLabel.startsWith('date of'))
            return thingtalk_1.Type.Date;
        const units = await utils_2.getAllowedUnits(property);
        if (units.length > 0) {
            if (units.includes('kilogram'))
                return new thingtalk_1.Type.Measure('kg');
            if (units.includes('metre') || units.includes('kilometre'))
                return new thingtalk_1.Type.Measure('m');
            if (units.includes('second') || units.includes('year'))
                return new thingtalk_1.Type.Measure('ms');
            if (units.includes('degree Celsius'))
                return new thingtalk_1.Type.Measure('C');
            if (units.includes('metre per second') || units.includes('kilometre per second'))
                return new thingtalk_1.Type.Measure('mps');
            if (units.includes('square metre'))
                return new thingtalk_1.Type.Measure('m2');
            if (units.includes('cubic metre'))
                return new thingtalk_1.Type.Measure('m3');
            if (units.includes('percent'))
                return thingtalk_1.Type.Number;
            if (units.includes('United States dollar'))
                return thingtalk_1.Type.Currency;
            console.error(`Unknown measurement type with unit ${units.join(', ')} for ${property}`);
            return thingtalk_1.Type.Number;
        }
        const range = await utils_2.getRangeConstraint(property);
        if (range)
            return thingtalk_1.Type.Number;
        if (propertyLabel.startsWith('manner of') || propertyLabel.startsWith('cause of'))
            return thingtalk_1.Type.String;
        const subpropertyOf = await utils_2.wikidataQuery(`SELECT ?value WHERE { wd:${property} wdt:P1647 ?value. } `);
        if (subpropertyOf.some((property) => property.value.value === 'http://www.wikidata.org/entity/P18'))
            return new thingtalk_1.Type.Entity('tt:picture');
        if (subpropertyOf.some((property) => property.value.value === 'http://www.wikidata.org/entity/P2699'))
            return new thingtalk_1.Type.Entity('tt:url');
        if (subpropertyOf.some((property) => property.value.value === 'http://www.wikidata.org/entity/P276'))
            return thingtalk_1.Type.Location;
        const types = await utils_2.getValueTypeConstraint(property);
        if (types.length > 0) {
            // human type: Q5: human, Q215627: person
            if (types.some((type) => type.label === 'human' || type.label === 'person'))
                return new thingtalk_1.Type.Entity(`org.wikidata:human`);
            // location type: Q618123: geographic object, Q2221906: geographic location
            if (types.some((type) => type.label === 'geographical object' || type.label === 'geographical location'))
                return thingtalk_1.Type.Location;
        }
        // load equivalent schema.org type if available
        const schemaorgEquivalent = await utils_2.getSchemaorgEquivalent(property);
        if (schemaorgEquivalent && schemaorgEquivalent in this._schemaorgProperties) {
            const schemaorgType = this._schemaorgProperties[schemaorgEquivalent];
            const schemaorgElemType = schemaorgType.isArray ? schemaorgType.elem : schemaorgType;
            if (schemaorgElemType.isEntity && schemaorgElemType.type.startsWith('org.schema')) {
                const entityType = schemaorgElemType.type.substring(schemaorgElemType.type.lastIndexOf(':') + 1).toLowerCase();
                return schemaorgType.isArray ?
                    new thingtalk_1.Type.Array(new thingtalk_1.Type.Entity(`org.wikidata:${entityType}`)) : new thingtalk_1.Type.Entity(`org.wikidata:${entityType}`);
            }
            if (!schemaorgType.isCompound)
                return schemaorgType;
        }
        // majority or arrays of string so this may be better default.
        return thingtalk_1.Type.String;
    }
    async _getArgCanonical(property, label, type) {
        if (this._manual && property in manual_annotations_1.MANUAL_PROPERTY_CANONICAL_OVERRIDE)
            return manual_annotations_1.MANUAL_PROPERTY_CANONICAL_OVERRIDE[property];
        const canonical = {};
        base_canonical_generator_1.default(canonical, label, type);
        if (this._wikidataLabels) {
            const altLabels = await utils_2.getPropertyAltLabels(property);
            if (altLabels) {
                for (let label of altLabels)
                    base_canonical_generator_1.default(canonical, label, type);
            }
        }
        return canonical;
    }
    _addEntity(type, name, has_ner_support) {
        if (!this._entities.some((entity) => entity.type === type))
            this._entities.push({ type, name, is_well_known: false, has_ner_support });
    }
    async run() {
        const queries = {};
        const actions = {};
        // load schema.org manifest if available
        if (this._schemaorgManifest) {
            const library = ThingTalk.Syntax.parse(await util_1.default.promisify(fs.readFile)(this._schemaorgManifest, { encoding: 'utf8' }));
            assert_1.default(library instanceof ThingTalk.Ast.Library && library.classes.length === 1);
            const classDef = library.classes[0];
            for (let fn in classDef.queries) {
                const fndef = classDef.queries[fn];
                for (let argname of fndef.args) {
                    let key = argname;
                    if (argname.includes('.'))
                        key = argname.substring(argname.lastIndexOf('.') + 1);
                    if (!(argname in this._schemaorgProperties))
                        this._schemaorgProperties[key] = fndef.getArgType(argname);
                }
            }
        }
        for (let domain of this._domains) {
            const domainLabel = domain in this._domainCanonicals ? this._domainCanonicals[domain] : await utils_2.getItemLabel(domain);
            const properties = this._propertiesByDomain[domain];
            const args = [
                new thingtalk_1.Ast.ArgumentDef(null, thingtalk_1.Ast.ArgDirection.OUT, 'id', new thingtalk_1.Type.Entity(`org.wikidata:${utils_1.snakecase(domainLabel)}`), {
                    nl: { canonical: { base: ['name'], passive_verb: ['named', 'called'] } }
                })
            ];
            this._addEntity(`org.wikidata:${utils_1.snakecase(domainLabel)}`, utils_1.titleCase(domainLabel), true);
            for (let property of properties) {
                const label = await utils_2.getPropertyLabel(property);
                const name = argnameFromLabel(label);
                const type = await this._getType(domain, domainLabel, property, label);
                const annotations = {
                    nl: { canonical: await this._getArgCanonical(property, label, type) },
                    impl: { wikidata_id: new thingtalk_1.Ast.Value.String(property) }
                };
                const elemType = getElementType(type);
                if (elemType.isString)
                    annotations.impl['string_values'] = new thingtalk_1.Ast.Value.String(`org.wikidata:${utils_1.snakecase(domainLabel)}_${name}`);
                if (elemType.isEntity && elemType.type.startsWith('org.wikidata:'))
                    this._addEntity(elemType.type, utils_1.titleCase(label), true);
                args.push(new thingtalk_1.Ast.ArgumentDef(null, thingtalk_1.Ast.ArgDirection.OUT, name, type, annotations));
            }
            const qualifiers = { is_list: true, is_monitorable: false };
            const annotations = {
                nl: { canonical: misc_utils_1.clean(domainLabel), confirmation: misc_utils_1.clean(domainLabel) },
                impl: { wikidata_subject: new thingtalk_1.Ast.Value.String(domain) }
            };
            if (domain in this._requiredPropertiesByDomain) {
                annotations.impl.required_properties = new thingtalk_1.Ast.Value.Array(this._requiredPropertiesByDomain[domain].map((p) => new thingtalk_1.Ast.Value.String(p)));
            }
            queries[domainLabel] = new thingtalk_1.Ast.FunctionDef(null, 'query', null, utils_1.snakecase(domainLabel), null, qualifiers, args, annotations);
        }
        const imports = [
            new thingtalk_1.Ast.MixinImportStmt(null, ['loader'], 'org.thingpedia.v2', []),
            new thingtalk_1.Ast.MixinImportStmt(null, ['config'], 'org.thingpedia.config.none', [])
        ];
        const classdef = new thingtalk_1.Ast.ClassDef(null, 'org.wikidata', null, { imports, queries, actions }, {
            nl: {
                name: `Wikidata for domain ${this._domains.join(', ')}`,
                description: 'Natural language dialogues over Wikidata knowledge base.'
            },
        }, {
            is_abstract: false
        });
        this._output.end(classdef.prettyprint());
        this._outputEntities.end(JSON.stringify({
            result: 'ok',
            data: this._entities
        }, undefined, 2));
        await StreamUtils.waitFinish(this._output);
        await StreamUtils.waitFinish(this._outputEntities);
    }
}
function initArgparse(subparsers) {
    const parser = subparsers.add_parser('wikidata-process-schema', {
        add_help: true,
        description: "Generate schema.tt given a list of domains."
    });
    parser.add_argument('-o', '--output', {
        required: true,
        type: fs.createWriteStream
    });
    parser.add_argument('--entities', {
        required: true,
        type: fs.createWriteStream
    });
    parser.add_argument('--domains', {
        required: true,
        help: 'domains (by item id) to include in the schema, split by comma (no space)'
    });
    parser.add_argument('--domain-canonicals', {
        required: false,
        help: 'the canonical form for the given domains, used as the query names, split by comma (no space);\n' +
            'if absent, use Wikidata label by default.'
    });
    parser.add_argument('--properties', {
        nargs: '+',
        required: false,
        help: 'properties to include for each domain, properties are split by comma (no space);\n' +
            'use "default" to include properties included in P1963 (properties of this type);\n' +
            'exclude a property by placing a minus sign before its id (no space)'
    });
    parser.add_argument('--manual', {
        action: 'store_true',
        help: 'Enable manual annotations.',
        default: false
    });
    parser.add_argument('--wikidata-labels', {
        action: 'store_true',
        help: 'Enable wikidata labels as annotations.',
        default: false
    });
    parser.add_argument('--schemaorg-manifest', {
        required: false,
        help: 'Path to manifest.tt for schema.org; used for predict the type of wikidata properties'
    });
    parser.add_argument('--required-properties', {
        nargs: '+',
        required: false,
        help: 'the subset of properties that are required to be non-empty for all retrieved entities;\n' +
            'use "none" to indicate no required property needed;\n' +
            'use "default" to include properties included in P1963 (properties of this type);\n' +
            'exclude a property by placing a minus sign before its id (no space)'
    });
}
exports.initArgparse = initArgparse;
async function execute(args) {
    const domains = args.domains.split(',');
    const domainCanonicals = {};
    if (args.domain_canonicals) {
        const canonicals = args.domain_canonicals.split(',');
        assert_1.default.strictEqual(canonicals.length, domains.length);
        for (let i = 0; i < domains.length; i++)
            domainCanonicals[domains[i]] = canonicals[i];
    }
    const requiredPropertiesByDomain = {};
    if (args.required_properties) {
        // if provided, property lists should match the number of domains
        assert_1.default(Array.isArray(args.required_properties) && args.required_properties.length === domains.length);
        for (let i = 0; i < domains.length; i++) {
            const domain = domains[i];
            requiredPropertiesByDomain[domain] = await retrieveProperties(domain, args.required_properties[i].split(','));
        }
    }
    const propertiesByDomain = {};
    if (args.properties) {
        // if provided, property lists should match the number of domains
        assert_1.default(Array.isArray(args.properties) && args.properties.length === domains.length);
        for (let i = 0; i < domains.length; i++) {
            const domain = domains[i];
            const properties = args.properties[i].split(',');
            propertiesByDomain[domain] = await retrieveProperties(domain, properties);
        }
    }
    else {
        for (let domain of domains)
            propertiesByDomain[domain] = await utils_2.getPropertyList(domain);
    }
    const schemaProcessor = new SchemaProcessor(domains, domainCanonicals, propertiesByDomain, requiredPropertiesByDomain, args.output, args.entities, args.manual, args.wikidata_labels, args.schemaorg_manifest);
    schemaProcessor.run();
}
exports.execute = execute;
//# sourceMappingURL=process-schema.js.map