"use strict";
// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
    if (mod && mod.__esModule) return mod;
    var result = {};
    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
    __setModuleDefault(result, mod);
    return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.execute = exports.initArgparse = void 0;
const en_inflectors_1 = require("en-inflectors");
const Tp = __importStar(require("thingpedia"));
const thingtalk_1 = require("thingtalk");
const fs = __importStar(require("fs"));
const util_1 = __importDefault(require("util"));
const misc_utils_1 = require("../../../lib/utils/misc-utils");
const english_1 = __importDefault(require("../../../lib/i18n/english"));
const StreamUtils = __importStar(require("../../../lib/utils/stream-utils"));
const base_canonical_generator_1 = __importDefault(require("../lib/base-canonical-generator"));
const manual_annotations_1 = require("./manual-annotations");
const keepAnnotation = false;
function isHumanEntity(type) {
    if (type instanceof thingtalk_1.Type.Entity)
        return isHumanEntity(type.type);
    if (type instanceof thingtalk_1.Type.Array)
        return isHumanEntity(type.elem);
    if (typeof type !== 'string')
        return false;
    if (['tt:contact', 'tt:username', 'org.wikidata:human'].includes(type))
        return true;
    if (type.startsWith('org.schema') && type.endsWith(':Person'))
        return true;
    return false;
}
function getId(id) {
    if (id.startsWith('http://schema.org/'))
        id = id.substring('http://schema.org/'.length);
    // add "_" prefix for id starts with a number
    if (/^\d/.test(id))
        id = '_' + id;
    return id;
}
function getIncludes(includes) {
    if (Array.isArray(includes))
        return includes.map((incl) => getId(incl['@id']));
    else
        return [getId(includes['@id'])];
}
function getItemType(typename, typeHierarchy) {
    // use conventions on the typename to convert an array type to its element type
    for (let suffix of ['List', 'Collection', 'Section', 'Catalog']) {
        if (typename.endsWith(suffix)) {
            const itemname = typename.substring(0, typename.length - suffix.length);
            if (itemname in typeHierarchy)
                return itemname;
            else
                return 'Thing';
        }
    }
    console.error(`ItemList subclass ${typename} does not have a recognized suffix`);
    return 'Thing';
}
function recursiveAddStringValues(arg, fileId) {
    let type = arg.type;
    while (type.isArray)
        type = type.elem;
    if (fileId in manual_annotations_1.PROPERTIES_NO_FILTER)
        return;
    if ((type.isEntity || type.isLocation) && manual_annotations_1.STRING_FILE_OVERRIDES[fileId]) {
        arg.annotations['string_values'] = new thingtalk_1.Ast.Value.String(manual_annotations_1.STRING_FILE_OVERRIDES[fileId]);
        return;
    }
    if (type.isString) {
        arg.annotations['string_values'] = new thingtalk_1.Ast.Value.String(manual_annotations_1.STRING_FILE_OVERRIDES[fileId] || fileId);
        return;
    }
    if (type.isCompound) {
        for (let field in type.fields) {
            if (field.indexOf('.') >= 0)
                continue;
            recursiveAddStringValues(type.fields[field], fileId + '_' + field);
        }
    }
}
class SchemaProcessor {
    constructor(args) {
        this._domain = args.domain;
        this._output = args.output;
        this._cache = args.cache_file;
        this._className = args.class_name;
        this._url = args.url;
        this._manual = args.manual;
        this._always_base_canonical = args.always_base_canonical;
        this._hasGeo = false;
        this._prefix = `${this._className}:`;
        this._white_list = args.white_list.split(',');
        this._entities = [];
        this._wikidata_path = args.wikidata_path;
        this._wikidata_labels = {};
        this._langPack = new english_1.default('en-US');
    }
    typeToThingTalk(propname, typename, typeHierarchy, manualAnnotation) {
        if (typename in manual_annotations_1.BUILTIN_TYPEMAP)
            return manual_annotations_1.BUILTIN_TYPEMAP[typename];
        if (typeHierarchy[typename].isItemList)
            return new thingtalk_1.Type.Array(this.typeToThingTalk(propname, typeHierarchy[typename].itemType, typeHierarchy, manualAnnotation));
        if (typeHierarchy[typename].isEnum && typeHierarchy[typename].enum.length > 0)
            return new thingtalk_1.Type.Enum(typeHierarchy[typename].enum);
        if (typeHierarchy[typename].representAsStruct)
            return this.makeCompoundType(propname, typename, typeHierarchy[typename], typeHierarchy, manualAnnotation);
        return new thingtalk_1.Type.Entity(this._prefix + typename);
    }
    getBestPropertyType(propname, property, typeHierarchy, manualAnnotation) {
        if (manual_annotations_1.BLACKLISTED_PROPERTIES.has(propname))
            return [undefined, undefined];
        let best = undefined, bestScore = -Infinity;
        // if the property is defined as taking ItemList and something else, we make an array of that something else
        let isArray = property.types.some((type) => typeHierarchy[type] && typeHierarchy[type].isItemList);
        // if the property comment starts with "A " or "An ", we assume there can be multiple values
        // because if it starts with "The ", we assume it can only have one value
        // this is a pretty coarse heuristic, but it works sometimes...
        if (/^an? /i.test(property.comment))
            isArray = true;
        if (manual_annotations_1.PROPERTY_FORCE_ARRAY.has(propname))
            isArray = true;
        if (manual_annotations_1.PROPERTY_FORCE_NOT_ARRAY.has(propname))
            isArray = false;
        // prefer enum if possible
        // then specific data types
        // then fallback to a struct type if one is listed
        // then fallback to text if it's explicitly listed as one of the types
        // then fallback to an entity type
        for (let type of property.types) {
            let score;
            if (typeHierarchy[type] && typeHierarchy[type].isEnum)
                score = 5;
            else if (type === 'Text')
                score = 2;
            else if (type in manual_annotations_1.BUILTIN_TYPEMAP)
                score = 4;
            else if (!typeHierarchy[type])
                score = -1;
            else if (typeHierarchy.isItemList) // ItemList and subclasses are useless
                score = 0;
            else if (typeHierarchy[type].representAsStruct)
                score = 3;
            else
                score = 1;
            if (score > bestScore) {
                best = type;
                bestScore = score;
            }
        }
        // if we didn't find a type we like, return nothing
        if (bestScore < 0)
            return [undefined, undefined];
        if (propname in manual_annotations_1.PROPERTY_TYPE_OVERRIDE)
            return [best, manual_annotations_1.PROPERTY_TYPE_OVERRIDE[propname]];
        // if we chose an item list as the best type, don't wrap into a further array
        if (typeHierarchy[best] && typeHierarchy[best].isItemList)
            isArray = false;
        // HACK
        if (best === 'QuantitativeValue') {
            if (/number/i.test(propname) || /level/i.test(propname) || /quantity/i.test(propname))
                return [best, thingtalk_1.Type.Number];
            if (/duration/i.test(propname))
                return [best, new thingtalk_1.Type.Measure('ms')];
            console.error(`Cannot guess the correct type of ${propname} of type QuantitativeValue, assuming Number`);
            return [best, thingtalk_1.Type.Number];
        }
        // HACK (version 9.0 has Organization over Person for author)
        if (propname === 'author')
            best = 'Person';
        let tttype = this.typeToThingTalk(propname, best, typeHierarchy, manualAnnotation);
        if (!tttype)
            return [undefined, undefined];
        if (tttype.isEntity && tttype.type.startsWith(this._prefix) && !this._entities.includes(tttype.type))
            this._entities.push(tttype.type);
        // an array of booleans or enums does not make much sense
        if (tttype.isBoolean || tttype.isEnum)
            isArray = false;
        if (isArray)
            tttype = new thingtalk_1.Type.Array(tttype);
        return [best, tttype];
    }
    loadPropertyNameOverride(argname) {
        if (manual_annotations_1.PROPERTY_NAME_OVERRIDE_BY_DOMAIN[this._domain]) {
            if (argname in manual_annotations_1.PROPERTY_NAME_OVERRIDE_BY_DOMAIN[this._domain])
                return manual_annotations_1.PROPERTY_NAME_OVERRIDE_BY_DOMAIN[this._domain][argname];
            while (argname.includes('.')) {
                argname = argname.slice(argname.indexOf('.') + 1);
                if (argname in manual_annotations_1.PROPERTY_NAME_OVERRIDE_BY_DOMAIN[this._domain])
                    return manual_annotations_1.PROPERTY_NAME_OVERRIDE_BY_DOMAIN[this._domain][argname];
            }
        }
        return null;
    }
    makeCompoundType(parentPropertyName, startingTypename, typedef, typeHierarchy) {
        const fields = {};
        // collect all properties of this type (incl. inherited ones)
        let allproperties = new Map;
        function recursiveCollectProperties(typename) {
            //console.error(typename);
            const typedef = typeHierarchy[typename];
            if (!typedef)
                return;
            // if something is a subclass of both a struct and non-struct,
            // we ignore the properties coming from the non-struct side
            // (unless the leaf type name we're starting from is explicitly
            // marking as going all the way up)
            if (!manual_annotations_1.STRUCT_INCLUDE_THING_PROPERTIES.has(startingTypename) && !typeHierarchy[typename].isStructSubType)
                return;
            for (let propertyname in typedef.properties) {
                const propertydef = typedef.properties[propertyname];
                if (allproperties.has(propertyname))
                    continue;
                allproperties.set(propertyname, propertydef);
            }
            // stop at the base struct types (so we don't include Thing properties)
            if (!manual_annotations_1.STRUCT_INCLUDE_THING_PROPERTIES.has(startingTypename) && manual_annotations_1.STRUCTURED_HIERARCHIES.indexOf(typename) >= 0)
                return;
            for (let _extends of typeHierarchy[typename].extends)
                recursiveCollectProperties(_extends);
        }
        recursiveCollectProperties(startingTypename);
        let anyfield = false;
        for (const [propertyname, propertydef] of allproperties) {
            const [schemaOrgType, ttType] = this.getBestPropertyType(propertyname, propertydef, typeHierarchy);
            if (!ttType)
                continue;
            const metadata = {};
            const annotation = keepAnnotation ? {
                'org_schema_type': new thingtalk_1.Ast.Value.String(schemaOrgType),
                'org_schema_comment': new thingtalk_1.Ast.Value.String(propertydef.comment)
            } : {
                'org_schema_type': new thingtalk_1.Ast.Value.String(schemaOrgType)
            };
            if (propertyname.endsWith('ratingValue')) {
                annotation['min_number'] = new thingtalk_1.Ast.Value.Number(1);
                annotation['max_number'] = new thingtalk_1.Ast.Value.Number(5);
            }
            if (this._manual && propertyname in manual_annotations_1.MANUAL_COUNTED_OBJECT_OVERRIDE)
                metadata.counted_object = manual_annotations_1.MANUAL_COUNTED_OBJECT_OVERRIDE[propertyname];
            else if (propertyname.startsWith('numberOf'))
                metadata.counted_object = [misc_utils_1.clean(propertyname.slice('numberOf'.length))];
            else if (/num[A-Z].*/.test(propertyname))
                metadata.counted_object = [misc_utils_1.clean(propertyname.slice('num'.length))];
            else if (propertyname.endsWith('Count'))
                metadata.counted_object = [this._langPack.pluralize(misc_utils_1.clean(propertyname.slice(0, -'Count'.length)))];
            if (manual_annotations_1.PROPERTIES_NO_FILTER.includes(propertyname)) {
                annotation['filterable'] = new thingtalk_1.Ast.Value.Boolean(false);
            }
            else if (this._hasGeo && manual_annotations_1.PROPERTIES_DROP_WITH_GEO.includes(propertyname)) {
                annotation['filterable'] = new thingtalk_1.Ast.Value.Boolean(false);
                annotation['drop'] = new thingtalk_1.Ast.Value.Boolean(true);
            }
            const adjustedname = thingtalk_1.Syntax.KEYWORDS.has(propertyname) ? propertyname + '_' : propertyname;
            fields[adjustedname] = new thingtalk_1.Ast.ArgumentDef(null, null, adjustedname, ttType, {
                nl: metadata,
                impl: annotation
            });
            anyfield = true;
        }
        if (!anyfield)
            throw new Error(`Struct type ${startingTypename} has no fields`);
        return new thingtalk_1.Type.Compound(startingTypename, fields);
    }
    loadPropertyCanonicalOverride(name) {
        // 1. check for domain-specific manual property override
        if (this._manual && this._domain && this._domain in manual_annotations_1.MANUAL_PROPERTY_CANONICAL_OVERRIDE_BY_DOMAIN
            && name in manual_annotations_1.MANUAL_PROPERTY_CANONICAL_OVERRIDE_BY_DOMAIN[this._domain])
            return manual_annotations_1.MANUAL_PROPERTY_CANONICAL_OVERRIDE_BY_DOMAIN[this._domain][name];
        // 2. check for global manual property override
        if (this._manual && name in manual_annotations_1.MANUAL_PROPERTY_CANONICAL_OVERRIDE)
            return manual_annotations_1.MANUAL_PROPERTY_CANONICAL_OVERRIDE[name];
        // 3. check default property type override (which is applied even for baseline)
        if (name in manual_annotations_1.PROPERTY_CANONICAL_OVERRIDE)
            return manual_annotations_1.PROPERTY_CANONICAL_OVERRIDE[name];
        // for compound properties, also search by field names
        if (name.includes('.'))
            return this.loadPropertyCanonicalOverride(name.slice(name.indexOf('.') + 1));
        return null;
    }
    addCanonicalAnnotations(classDef) {
        for (let fname in classDef.queries) {
            for (let arg of classDef.queries[fname].iterateArguments()) {
                if (arg.name === 'id')
                    continue;
                arg.metadata.canonical = this.makeArgCanonical(classDef.queries[fname], arg.name, arg.type);
                let elemType = arg.type;
                while (elemType.isArray)
                    elemType = elemType.elem;
                if (elemType.isCompound) {
                    for (let fieldname in elemType.fields) {
                        let field = elemType.fields[fieldname];
                        field.metadata.canonical = this.makeArgCanonical(classDef.queries[fname], `${arg.name}.${field.name}`, field.type);
                    }
                }
            }
        }
    }
    makeArgCanonical(functionDef, argname, ptype) {
        function cleanName(name) {
            name = misc_utils_1.clean(name);
            if (name.endsWith(' value'))
                return name.substring(0, name.length - ' value'.length);
            return name;
        }
        let canonical = this.loadPropertyCanonicalOverride(argname);
        if (canonical)
            return canonical;
        const name = this.loadPropertyNameOverride(argname) || argname.slice(argname.lastIndexOf('.') + 1);
        canonical = {};
        const candidates = name in this._wikidata_labels ? this._wikidata_labels[name].labels : [name];
        for (let candidate of [...new Set(candidates)])
            this.addOneCanonical(canonical, candidate, ptype, functionDef);
        if (!("base" in canonical) && this._always_base_canonical)
            canonical["base"] = [cleanName(name)];
        if (isHumanEntity(ptype)) {
            const singular = (new en_inflectors_1.Inflectors(canonical.base[0])).toSingular();
            const past = (new en_inflectors_1.Inflectors(singular).toPast());
            canonical.reverse_verb = [past];
        }
        return canonical;
    }
    addOneCanonical(canonical, name, ptype, functionDef) {
        // drop all names with char other than letters
        if (!/^[a-zA-Z ]+$/.test(name))
            return;
        base_canonical_generator_1.default(canonical, name, ptype, functionDef);
    }
    async run() {
        let schemajsonld;
        if (await util_1.default.promisify(fs.exists)(this._cache)) {
            schemajsonld = await util_1.default.promisify(fs.readFile)(this._cache, { encoding: 'utf8' });
        }
        else {
            schemajsonld = await Tp.Helpers.Http.get(this._url);
            await util_1.default.promisify(fs.writeFile)(this._cache, schemajsonld);
        }
        if (this._wikidata_path)
            this._wikidata_labels = JSON.parse(await (util_1.default.promisify(fs.readFile))(this._wikidata_path, { encoding: 'utf8' }));
        // type_name -> {
        //    extends: [type_name],
        //    properties: { name -> { types: [type], comment: ... } },
        //    comment: ...
        // }
        const typeHierarchy = {};
        function ensureType(typename) {
            if (typeHierarchy[typename])
                return;
            typeHierarchy[typename] = {
                extends: [],
                properties: {},
                comment: ''
            };
        }
        function isSubClass(typename, subtypeof) {
            for (let _extend of typeHierarchy[typename].extends) {
                if (_extend === subtypeof)
                    return true;
                if (!typeHierarchy[_extend])
                    continue;
                if (isSubClass(_extend, subtypeof))
                    return true;
            }
            return false;
        }
        const enums = {};
        function ensureEnum(enumname) {
            if (enums[enumname])
                return;
            enums[enumname] = [];
        }
        for (let triple of JSON.parse(schemajsonld)['@graph']) {
            try {
                if (getId(triple['@id']) in manual_annotations_1.BUILTIN_TYPEMAP)
                    continue;
                if (manual_annotations_1.BLACKLISTED_TYPES.has(getId(triple['@id'])))
                    continue;
                if (!Array.isArray(triple['@type']))
                    triple['@type'] = [triple['@type']];
                for (let type of triple['@type']) {
                    if (type.startsWith('http://schema.org/')) {
                        // an enum declaration
                        const enumtype = getId(type);
                        const enumvalue = getId(triple['@id']);
                        ensureEnum(enumtype);
                        enums[enumtype].push(enumvalue);
                        continue;
                    }
                    switch (type) {
                        case 'rdf:Property': {
                            // ignore deprecated stuff
                            if (triple['http://schema.org/supersededBy'])
                                continue;
                            const domains = getIncludes(triple['http://schema.org/domainIncludes']);
                            const ranges = getIncludes(triple['http://schema.org/rangeIncludes']);
                            const name = getId(triple['@id']);
                            const comment = triple['rdfs:comment'];
                            if (manual_annotations_1.BLACKLISTED_PROPERTIES.has(name))
                                continue;
                            for (let domain of domains) {
                                if (domain in manual_annotations_1.BUILTIN_TYPEMAP)
                                    continue;
                                if (manual_annotations_1.BLACKLISTED_TYPES.has(domain))
                                    continue;
                                ensureType(domain);
                                typeHierarchy[domain].properties[name] = {
                                    types: ranges,
                                    comment
                                };
                            }
                            break;
                        }
                        case 'rdfs:Class': {
                            const name = getId(triple['@id']);
                            const comment = triple['rdfs:comment'];
                            const _extends = getIncludes(triple['rdfs:subClassOf'] || []);
                            ensureType(name);
                            if (_extends.length > 0 && _extends.every((ex) => ex in manual_annotations_1.BUILTIN_TYPEMAP)) {
                                manual_annotations_1.BLACKLISTED_TYPES.add(name);
                                delete typeHierarchy[name];
                                break;
                            }
                            typeHierarchy[name].extends = _extends.filter((ex) => !manual_annotations_1.BLACKLISTED_TYPES.has(ex));
                            if (typeHierarchy[name].extends.length === 0 && name !== 'Thing')
                                typeHierarchy[name].extends = ['Thing'];
                            typeHierarchy[name].comment = comment;
                            break;
                        }
                        default:
                            throw new Error(`don't know how to handle a triple of type ${type}`); //'
                    }
                }
            }
            catch (e) {
                console.error('Triple failed');
                console.error(triple);
                throw e;
            }
        }
        for (let type in typeHierarchy) {
            typeHierarchy[type].isAction = isSubClass(type, 'Action');
            typeHierarchy[type].isEnum = !!enums[type] || isSubClass(type, 'Enumeration');
            if (typeHierarchy[type].isEnum)
                typeHierarchy[type].enum = enums[type] || [];
            typeHierarchy[type].isItemList = isSubClass(type, 'ItemList');
            if (typeHierarchy[type].isItemList)
                typeHierarchy[type].itemType = getItemType(type, typeHierarchy);
            if (manual_annotations_1.STRUCTURED_HIERARCHIES.indexOf(type) >= 0) {
                typeHierarchy[type].isStructSubType = true;
                typeHierarchy[type].representAsStruct = true;
            }
            else {
                for (let structBase of manual_annotations_1.STRUCTURED_HIERARCHIES) {
                    if (isSubClass(type, structBase)) {
                        typeHierarchy[type].isStructSubType = true;
                        typeHierarchy[type].representAsStruct = true;
                        break;
                    }
                }
            }
            if (manual_annotations_1.NON_STRUCT_TYPES.has(type)) {
                typeHierarchy[type].isStructSubType = false;
                typeHierarchy[type].representAsStruct = false;
            }
        }
        function findCycle(typename, lookfor, visited, cycle = []) {
            if (visited.has(typename)) {
                if (typename === lookfor)
                    console.error('Found cycle for ' + typename, cycle, visited);
                return typename === lookfor;
            }
            visited.add(typename);
            for (let propname in typeHierarchy[typename].properties) {
                let propdef = typeHierarchy[typename].properties[propname];
                for (let type of propdef.types) {
                    if (type in manual_annotations_1.BUILTIN_TYPEMAP)
                        continue;
                    if (!typeHierarchy[type] || !typeHierarchy[type].representAsStruct)
                        continue;
                    cycle.push(propname);
                    if (findCycle(type, lookfor, visited, cycle))
                        return true;
                    cycle.pop();
                }
            }
            return false;
        }
        // check all types - if they form a cycle, we cannot represent them as structs
        for (let typename in typeHierarchy) {
            if (typeHierarchy[typename].isEnum)
                continue;
            if (!typeHierarchy[typename].representAsStruct)
                continue;
            if (findCycle(typename, typename, new Set))
                typeHierarchy[typename].representAsStruct = false;
        }
        // check all types - all parents of non-struct types must also be non-struct types,
        // recursively
        function recursiveMakeNonStruct(typename) {
            typeHierarchy[typename].representAsStruct = false;
            for (let _extend of typeHierarchy[typename].extends) {
                if (!typeHierarchy[_extend])
                    continue;
                recursiveMakeNonStruct(_extend);
            }
        }
        for (let typename in typeHierarchy) {
            if (typeHierarchy[typename].isEnum)
                continue;
            if (typeHierarchy[typename].representAsStruct)
                continue;
            recursiveMakeNonStruct(typename);
        }
        //console.log(JSON.stringify(typeHierarchy, undefined, 2));
        const order = new Set;
        function toposort(typename) {
            if (typeHierarchy[typename].isAction || typeHierarchy[typename].isEnum ||
                typeHierarchy[typename].representAsStruct)
                return;
            for (let _extend of typeHierarchy[typename].extends) {
                if (!typeHierarchy[_extend])
                    continue;
                toposort(_extend);
            }
            order.add(typename);
        }
        for (let type in typeHierarchy) {
            if (order.has(type))
                continue;
            toposort(type);
        }
        const queries = {};
        for (let typename of order) {
            const typedef = typeHierarchy[typename];
            // do not generate a class for ItemList and subclasses
            if (typename === 'ItemList' || typedef.isItemList)
                continue;
            const args = [
                new thingtalk_1.Ast.ArgumentDef(null, thingtalk_1.Ast.ArgDirection.OUT, 'id', new thingtalk_1.Type.Entity(this._prefix + typename), {
                    nl: { canonical: { base: ['name'], passive_verb: ['called', 'named'] } },
                    impl: {
                        'unique': new thingtalk_1.Ast.Value.Boolean(true),
                        'filterable': new thingtalk_1.Ast.Value.Boolean(true)
                    }
                })
            ];
            recursiveAddStringValues(args[0], this._prefix + typename + '_name');
            if (typename !== 'Thing') {
                // override name for each table so we can apply a custom string_values annotation
                // name is preserved to determine if the table has name and id has ner support
                // it will be removed during trimming
                const arg = new thingtalk_1.Ast.ArgumentDef(null, thingtalk_1.Ast.ArgDirection.OUT, 'name', thingtalk_1.Type.String, {
                    nl: {},
                    impl: {
                        'org_schema_type': new thingtalk_1.Ast.Value.String('Text'),
                        'filterable': new thingtalk_1.Ast.Value.Boolean(true)
                    }
                });
                recursiveAddStringValues(arg, this._prefix + typename + '_name');
                args.push(arg);
            }
            this._hasGeo = 'geo' in typedef.properties;
            for (const propertyname in typedef.properties) {
                const propertydef = typedef.properties[propertyname];
                const [schemaOrgType, type] = this.getBestPropertyType(propertyname, propertydef, typeHierarchy);
                if (!type)
                    continue;
                const metadata = {};
                const annotation = keepAnnotation ? {
                    'org_schema_type': new thingtalk_1.Ast.Value.String(schemaOrgType),
                    'org_schema_comment': new thingtalk_1.Ast.Value.String(propertydef.comment)
                } : {
                    'org_schema_type': new thingtalk_1.Ast.Value.String(schemaOrgType)
                };
                if (manual_annotations_1.PROPERTIES_NO_FILTER.includes(propertyname))
                    annotation['filterable'] = new thingtalk_1.Ast.Value.Boolean(false);
                if (this._manual && propertyname in manual_annotations_1.MANUAL_COUNTED_OBJECT_OVERRIDE)
                    metadata.counted_object = manual_annotations_1.MANUAL_COUNTED_OBJECT_OVERRIDE[propertyname];
                else if (propertyname.startsWith('numberOf'))
                    metadata.counted_object = [misc_utils_1.clean(propertyname.slice('numberOf'.length))];
                else if (propertyname.startsWith('num') && propertyname.charAt(3) === propertyname.charAt(3).toUpperCase())
                    metadata.counted_object = [misc_utils_1.clean(propertyname.slice('num'.length))];
                else if (propertyname.endsWith('Count'))
                    metadata.counted_object = [this._langPack.pluralize(misc_utils_1.clean(propertyname.slice(0, -'Count'.length)))];
                const adjustedname = thingtalk_1.Syntax.KEYWORDS.has(propertyname) ? propertyname + '_' : propertyname;
                const arg = new thingtalk_1.Ast.ArgumentDef(null, thingtalk_1.Ast.ArgDirection.OUT, adjustedname, type, {
                    nl: metadata,
                    impl: annotation
                });
                recursiveAddStringValues(arg, this._prefix + typename + '_' + propertyname);
                args.push(arg);
            }
            let query_canonical;
            if (this._manual && typename in manual_annotations_1.MANUAL_TABLE_CANONICAL_OVERRIDE)
                query_canonical = manual_annotations_1.MANUAL_TABLE_CANONICAL_OVERRIDE[typename];
            else if (typename in manual_annotations_1.TABLE_CANONICAL_OVERRIDE)
                query_canonical = manual_annotations_1.TABLE_CANONICAL_OVERRIDE[typename];
            else
                query_canonical = misc_utils_1.clean(typename);
            queries[typename] = new thingtalk_1.Ast.FunctionDef(null, 'query', null /* class */, typename, typedef.extends, {
                is_list: true,
                is_monitorable: false,
            }, args, {
                nl: {
                    'canonical': query_canonical,
                    'confirmation': misc_utils_1.clean(typename),
                },
                impl: keepAnnotation ? {
                    'org_schema_comment': new thingtalk_1.Ast.Value.String(typedef.comment),
                    'confirm': new thingtalk_1.Ast.Value.Boolean(false)
                } : {
                    'confirm': new thingtalk_1.Ast.Value.Boolean(false)
                },
                minimal_projection: new thingtalk_1.Ast.Value.Array([new thingtalk_1.Ast.Value.String('id')])
            });
        }
        const imports = [
            new thingtalk_1.Ast.MixinImportStmt(null, ['loader'], 'org.thingpedia.v2', []),
            new thingtalk_1.Ast.MixinImportStmt(null, ['config'], 'org.thingpedia.config.none', [])
        ];
        const entities = this._entities.map((entityType) => {
            const name = entityType.slice(this._prefix.length);
            return new thingtalk_1.Ast.EntityDef(null, name, null, {});
        });
        const classdef = new thingtalk_1.Ast.ClassDef(null, `${this._className}`, [], { queries, imports, entities }, {
            nl: {
                name: `${this._className.slice(this._className.lastIndexOf('.') + 1)} in Schema.org`,
                description: 'Scraped data from websites that support schema.org'
            },
            impl: {
                whitelist: new thingtalk_1.Ast.Value.Array(this._white_list.map((q) => new thingtalk_1.Ast.Value.String(q.trim())))
            }
        }, {
            is_abstract: false
        });
        this.addCanonicalAnnotations(classdef);
        this._output.end(classdef.prettyprint());
        await StreamUtils.waitFinish(this._output);
    }
}
function initArgparse(subparsers) {
    const parser = subparsers.add_parser('schemaorg-process-schema', {
        add_help: true,
        description: "Process a schema.org JSON+LD definition into a Thingpedia class."
    });
    parser.add_argument('-o', '--output', {
        required: true,
        type: fs.createWriteStream
    });
    parser.add_argument('--cache-file', {
        required: false,
        default: './schema.jsonld',
        help: 'Path to a cache file containing the schema.org definitions.'
    });
    parser.add_argument('--url', {
        required: false,
        default: 'https://raw.githubusercontent.com/schemaorg/schemaorg/main/data/releases/9.0/schemaorg-current-http.jsonld',
        help: 'The schema.org URL to retrieve the definitions from.'
    });
    parser.add_argument('--domain', {
        required: false,
        help: 'The domain of current experiment, used for domain-specific manual overrides.'
    });
    parser.add_argument('--manual', {
        action: 'store_true',
        help: 'Enable manual annotations.',
        default: false
    });
    parser.add_argument('--wikidata-path', {
        required: false,
        help: 'path to the json file with wikidata property labels'
    });
    parser.add_argument('--always-base-canonical', {
        action: 'store_true',
        help: `Always generate base canonical`,
        default: true
    });
    parser.add_argument('--no-always-base-canonical', {
        action: 'store_false',
        help: `Do not always generate base canonical`,
        dest: `always_base_canonical`,
    });
    parser.add_argument('--class-name', {
        required: false,
        help: 'The name of the generated class, this will also affect the entity names',
        default: 'org.schema'
    });
    parser.add_argument('--white-list', {
        required: true,
        help: 'A list of queries allowed to use in the class, split by comma (no space).'
    });
}
exports.initArgparse = initArgparse;
async function execute(args) {
    const schemaProcessor = new SchemaProcessor(args);
    schemaProcessor.run();
}
exports.execute = execute;
//# sourceMappingURL=process-schema.js.map