"use strict";
// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
    if (mod && mod.__esModule) return mod;
    var result = {};
    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
    __setModuleDefault(result, mod);
    return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.execute = exports.initArgparse = void 0;
const assert_1 = __importDefault(require("assert"));
const fs = __importStar(require("fs"));
const util_1 = __importDefault(require("util"));
const stream_1 = __importDefault(require("stream"));
const seedrandom_1 = __importDefault(require("seedrandom"));
const Tp = __importStar(require("thingpedia"));
const thingtalk_1 = require("thingtalk");
const ParserClient = __importStar(require("../lib/prediction/parserclient"));
const parsers_1 = require("../lib/dataset-tools/parsers");
const StreamUtils = __importStar(require("../lib/utils/stream-utils"));
const multi_json_database_1 = __importDefault(require("./lib/multi_json_database"));
const progress_bar_1 = __importDefault(require("./lib/progress_bar"));
const entity_finder_1 = require("../lib/dialogue-agent/entity-linking/entity-finder");
const ThingTalkUtils = __importStar(require("../lib/utils/thingtalk"));
function undoTradePreprocessing(sentence) {
    return sentence.replace(/ -(ly|s)/g, '$1').replace(/\b24:([0-9]{2})\b/g, '00:$1');
}
const POLICY_NAME = 'org.thingpedia.dialogue.transaction';
const SEARCH_SLOTS = new Set([
    'restaurant-name',
    'restaurant-food',
    'restaurant-area',
    'restaurant-price-range',
    'hotel-name',
    'hotel-area',
    'hotel-type',
    'hotel-price-range',
    'hotel-parking',
    'hotel-stars',
    'hotel-internet',
    'attraction-name',
    'attraction-area',
    'attraction-type',
    'train-name',
    'train-day',
    'train-departure',
    'train-destination',
    'train-leave-at',
    'train-leaveat',
    'train-arrive-by',
    'train-arriveby'
]);
const SEARCH_SLOTS_FOR_SYSTEM = new Set([
    // annotations are all over the place....
    'food', 'area',
    'price-range', 'price', 'type', 'parking',
    'internet', 'wifi', 'choice',
    'day',
    'departure', 'depart',
    'destination', 'dest',
    'leave-at', 'leave',
    'arrive-at', 'arrive',
    'stars',
]);
const REQUESTED_SLOT_MAP = {
    price: 'price_range',
    wifi: 'internet',
    leave: 'leave_at',
    arrive: 'arrive_by',
    depart: 'departure',
    dest: 'destination',
    people: 'book_people',
    time: 'book_time',
    stay: 'book_stay',
    day: 'book_day'
};
// copied from trade-dst
// copyright 2019 https://jasonwu0731.github.io/
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
const GENERAL_TYPO = {
    // type
    "guesthouse": "guest house", "guesthouses": "guest house", "guest": "guest house", "mutiple sports": "multiple sports",
    "sports": "multiple sports", "mutliple sports": "multiple sports", "swimmingpool": "swimming pool", "concerthall": "concert hall",
    "concert": "concert hall", "pool": "swimming pool", "night club": "nightclub", "mus": "museum", "ol": "architecture",
    "colleges": "college", "coll": "college", "architectural": "architecture", "musuem": "museum", "churches": "church",
    // area
    "center": "centre", "center of town": "centre", "near city center": "centre", "in the north": "north", "cen": "centre", "east side": "east",
    "east area": "east", "west part of town": "west", "ce": "centre", "town center": "centre", "centre of cambridge": "centre",
    "city center": "centre", "the south": "south", "scentre": "centre", "town centre": "centre", "in town": "centre", "north part of town": "north",
    "centre of town": "centre", "cb30aq": "none",
    // price
    "mode": "moderate", "moderate -ly": "moderate", "mo": "moderate",
    // day
    "next friday": "friday", "monda": "monday",
    // parking
    "free parking": "yes",
    // internet
    "free internet": "yes",
    // star
    "4 star": "4", "4 stars": "4", "0 star rarting": "none",
    // others
    "y": "yes", "any": "dontcare", "n": "no", "does not care": "dontcare",
    "not men": "none", "not": "none", "not mentioned": "none",
    '': "none", "not mendtioned": "none",
    "3 .": "3", "does not": "no", "fun": "none", "art": "none",
    // new typos
    "el shaddia guesthouse": "el shaddai",
    "not given": "none",
    "thur": "thursday",
    "sundaymonday": "sunday|monday",
    "mondaythursday": "monday|thursday",
    "fridaytuesday": "friday|tuesday",
    "cheapmoderate": "cheap|moderate"
};
function fixGeneralLabelError(key, value) {
    if (value in GENERAL_TYPO)
        value = GENERAL_TYPO[value];
    // miss match slot and value
    if ((key === "hotel-type" && ["nigh", "moderate -ly priced", "bed and breakfast", "centre", "venetian", "intern", "a cheap -er hotel"].includes(value) ||
        (key === "hotel-internet" && value === "4") ||
        (key === "hotel-price-range" && value === "2") ||
        (key === "attraction-type" && ["gastropub", "la raza", "galleria", "gallery", "science", "m"].includes(value)) ||
        (/area/.test(key) && value === "moderate") ||
        (/day/.test(key) && value === "t")))
        return 'none';
    if (key === "hotel-type" && ["hotel with free parking and free wifi", "4", "3 star hotel"].includes(value))
        return 'hotel';
    if (key === "hotel-star" && value === "3 star hotel")
        return "3";
    if (key === 'hotel-price-range' && value === '$100')
        return 'none';
    if ((key === 'hotel-book-day' || key === 'restaurant-book-day') && value === 'w')
        return 'wednesday';
    if ((key === 'hotel-book-day' || key === 'restaurant-book-day') && value === 'w')
        return 'wednesday';
    if (/area/.test(key)) {
        if (value === 'no')
            return "north";
        if (value === "we")
            return "west";
        if (value === "cent")
            return "centre";
    }
    return value;
}
function parseTime(v) {
    if (v.indexOf('|') >= 0)
        v = v.substring(0, v.indexOf('|'));
    if (/^[0-9]+:[0-9]+/.test(v)) {
        const [hourstr, minutestr, secondstr] = v.split(':');
        let hour = parseInt(hourstr);
        const minute = parseInt(minutestr);
        let second;
        if (secondstr === undefined)
            second = 0;
        else
            second = parseInt(secondstr);
        if (hour === 24)
            hour = 0;
        return new thingtalk_1.Ast.Value.Time(new thingtalk_1.Ast.Time.Absolute(hour, minute, second));
    }
    let match = /([0-9]{2})([0-9]{2})/.exec(v);
    if (match !== null) {
        let hour = parseInt(match[1]);
        if (hour === 24)
            hour = 0;
        return new thingtalk_1.Ast.Value.Time(new thingtalk_1.Ast.Time.Absolute(hour, parseInt(match[2]), 0));
    }
    match = /([0-9]+)\s*am/.exec(v);
    if (match !== null)
        return new thingtalk_1.Ast.Value.Time(new thingtalk_1.Ast.Time.Absolute(match[1] === '12' ? 0 : parseInt(match[1]), 0, 0));
    match = /([0-9]+)\s*pm/.exec(v);
    if (match !== null)
        return new thingtalk_1.Ast.Value.Time(new thingtalk_1.Ast.Time.Absolute(match[1] === '12' ? 12 : 12 + parseInt(match[1]), 0, 0));
    // oops
    return new thingtalk_1.Ast.Value.Time(new thingtalk_1.Ast.Time.Absolute(0, 0, 0));
}
function getStatementDomain(stmt) {
    return stmt.last.schema.class.name;
}
const USE_MANUAL_AGENT_ANNOTATION = true;
class Converter extends stream_1.default.Readable {
    constructor(args) {
        super({ objectMode: true });
        this._onlyMultidomain = args.only_multidomain;
        this._tpClient = new Tp.FileClient(args);
        this._schemas = new thingtalk_1.SchemaRetriever(this._tpClient, null, true);
        this._userParser = ParserClient.get(args.user_nlu_server, 'en-US');
        this._agentParser = ParserClient.get(args.agent_nlu_server, 'en-US');
        this._useExisting = args.use_existing;
        this._maxTurn = args.max_turn;
        this._simulatorOverrides = new Map;
        const simulatorOptions = {
            rng: seedrandom_1.default.alea('almond is awesome'),
            locale: 'en-US',
            timezone: args.timezone,
            thingpediaClient: this._tpClient,
            schemaRetriever: this._schemas,
            overrides: this._simulatorOverrides,
            interactive: false
        };
        this._database = new multi_json_database_1.default(args.database_file);
        simulatorOptions.database = this._database;
        this._simulator = ThingTalkUtils.createSimulator(simulatorOptions);
        this._n = 0;
        this._N = 0;
    }
    _read() { }
    async start() {
        await this._database.load();
        await this._userParser.start();
        await this._agentParser.start();
    }
    async stop() {
        await this._userParser.start();
        await this._agentParser.start();
    }
    async _parseUtterance(context, parser, utterance, forSide, example_id) {
        let contextCode, contextEntities;
        if (context !== null) {
            context = ThingTalkUtils.prepareContextForPrediction(context, forSide);
            [contextCode, contextEntities] = ThingTalkUtils.serializeNormalized(context);
        }
        else {
            contextCode = ['null'];
            contextEntities = {};
        }
        const parsed = await parser.sendUtterance(utterance, contextCode, contextEntities, {
            tokenized: false,
            skip_typechecking: true,
            example_id
        });
        return ThingTalkUtils.parseAllPredictions(parsed.candidates, parsed.entities, {
            thingpediaClient: this._tpClient,
            schemaRetriever: this._schemas
        });
    }
    _getContextInfo(state) {
        let next = null, current = null;
        for (let idx = 0; idx < state.history.length; idx++) {
            const item = state.history[idx];
            if (item.results === null) {
                if (item.confirm === 'accepted')
                    next = item;
                break;
            }
            current = item;
        }
        return { current, next };
    }
    async _doAgentTurn(context, contextInfo, turn, agentUtterance, exampleId) {
        const parsedAgent = await this._parseUtterance(context, this._agentParser, agentUtterance, 'agent', exampleId);
        let agentTarget;
        if (parsedAgent.length === 0 || !(parsedAgent[0] instanceof thingtalk_1.Ast.DialogueState)) {
            // oops, bad
            agentTarget = new thingtalk_1.Ast.DialogueState(null, POLICY_NAME, 'sys_invalid', null, []);
        }
        else {
            agentTarget = parsedAgent[0];
        }
        if (agentTarget.dialogueAct === 'sys_propose_refined_query') {
            // this is basically never parsed right, so we override it
            agentTarget = new thingtalk_1.Ast.DialogueState(null, POLICY_NAME, 'sys_invalid', null, []);
        }
        if (this._useExisting && USE_MANUAL_AGENT_ANNOTATION) {
            // add some heuristics using the "system_acts" annotation
            const requestedSlots = turn.system_acts.filter((act) => typeof act === 'string');
            if (requestedSlots.length > 0) {
                if (requestedSlots.some((slot) => SEARCH_SLOTS_FOR_SYSTEM.has(slot))) {
                    if (contextInfo.current && contextInfo.current.results.results.length === 0)
                        agentTarget.dialogueAct = 'sys_empty_search_question';
                    else
                        agentTarget.dialogueAct = 'sys_search_question';
                }
                else {
                    if (contextInfo.current && contextInfo.current.results.error)
                        agentTarget.dialogueAct = 'sys_action_error_question';
                    else
                        agentTarget.dialogueAct = 'sys_slot_fill';
                }
                agentTarget.dialogueActParam = requestedSlots.map((slot) => REQUESTED_SLOT_MAP[slot] || slot);
            }
        }
        // adjust to ensure the agent doesn't produce new complete statements
        agentTarget.history = agentTarget.history.filter((item) => {
            if (item.confirm === 'confirmed')
                return false;
            return !item.isExecutable() || item.confirm === 'proposed';
        });
        if (agentTarget.history.length === 0 && contextInfo.next)
            agentTarget.history.push(contextInfo.next.clone());
        return agentTarget;
    }
    _getIDs(type) {
        return this._database.get(type).map((entry) => {
            const id = entry.id;
            return {
                value: id.value,
                name: id.display,
                canonical: id.display
            };
        });
    }
    _resolveEntity(value) {
        const resolved = entity_finder_1.getBestEntityMatch(value.display, value.type, this._getIDs(value.type));
        value.value = resolved.value;
        // do not override the display field, it should match the sentence instead
        // it will be overridden later when round-tripped through the executor
        //value.display = resolved.display;
    }
    async _doUserTurn(context, contextInfo, turn, userUtterance, slotBag, actionDomains, exampleId) {
        if (!this._useExisting) {
            // pure self-training:
            const parsedUser = await this._parseUtterance(context, this._userParser, userUtterance, 'user', exampleId);
            let userTarget;
            if (parsedUser.length === 0) {
                // oops, bad
                userTarget = new thingtalk_1.Ast.DialogueState(null, POLICY_NAME, 'invalid', null, []);
                if (contextInfo.next)
                    userTarget.history.push(contextInfo.next.clone());
            }
            else {
                userTarget = parsedUser[0];
            }
            // ensure that executable statements come first
            userTarget.history.sort((a, b) => {
                const aexec = a.isExecutable();
                const bexec = b.isExecutable();
                if (aexec === bexec)
                    return 0;
                if (aexec)
                    return -1;
                else
                    return 1;
            });
            return userTarget;
        }
        const allSlots = new Map();
        for (const slot of turn.belief_state) {
            assert_1.default(slot.act === 'inform');
            let [key, value] = slot.slots[0];
            assert_1.default(typeof key === 'string');
            assert_1.default(typeof value === 'string');
            key = key.replace(/ /g, '-').replace(/pricerange/, 'price-range');
            value = fixGeneralLabelError(key, value);
            if (value === 'none')
                continue;
            if (/^(hospital|bus|police)-/.test(key))
                continue;
            allSlots.set(key, value);
        }
        const newSearchSlots = new Map();
        const newActionSlots = new Map();
        let domain = 'empty';
        for (const [key, value] of allSlots) {
            if (slotBag.get(key) !== value) {
                slotBag.set(key, value);
                domain = key.split('-')[0];
                if (SEARCH_SLOTS.has(key))
                    newSearchSlots.set(key, value);
                else
                    newActionSlots.set(key, value);
            }
        }
        if (newSearchSlots.size === 0 && newActionSlots.size === 0) {
            // no slot given at this turn
            // parse the utterance and hope for the best...
            const parsedUser = await this._parseUtterance(context, this._userParser, userUtterance, 'user', exampleId);
            let userTarget;
            if (parsedUser.length === 0) {
                // oops, bad
                userTarget = new thingtalk_1.Ast.DialogueState(null, POLICY_NAME, 'invalid', null, []);
            }
            else {
                userTarget = parsedUser[0];
            }
            // remove all new info from it, copy everything over
            userTarget.history.length = 0;
            if (contextInfo.next)
                userTarget.history.push(contextInfo.next.clone());
            return userTarget;
        }
        else {
            const newItems = [];
            const queryname = domain[0].toUpperCase() + domain.substring(1);
            const action = queryname === 'Restaurant' ? 'make_reservation' : 'make_booking';
            const tpClass = 'uk.ac.cam.multiwoz.' + queryname;
            const selector = new thingtalk_1.Ast.DeviceSelector(null, tpClass, null, null);
            // if the only new search slot is name, and we'll be executing the action for this
            // domain, we move the name to an action slot instead
            if (actionDomains.has(domain) && contextInfo.current && getStatementDomain(contextInfo.current.stmt) === domain) {
                const searchKeys = Array.from(newSearchSlots.keys());
                if (searchKeys.length === 1 && searchKeys[0].endsWith('name')) {
                    newActionSlots.set(searchKeys[0], newSearchSlots.get(searchKeys[0]));
                    newSearchSlots.delete(searchKeys[0]);
                }
            }
            if (newSearchSlots.size && domain !== 'taxi') {
                const invocationTable = new thingtalk_1.Ast.InvocationExpression(null, new thingtalk_1.Ast.Invocation(null, selector, queryname, [], null), null);
                const filterClauses = [];
                for (const [key, value] of slotBag) {
                    if (!key.startsWith(domain))
                        continue;
                    if (!SEARCH_SLOTS.has(key))
                        continue;
                    let param = key.split('-').slice(1).join('_');
                    if (param === 'name')
                        param = 'id';
                    if (param === 'arriveby')
                        param = 'arrive_by';
                    if (param === 'leaveat')
                        param = 'leave_at';
                    let ttValue;
                    if (value === 'dontcare') {
                        filterClauses.push(new thingtalk_1.Ast.BooleanExpression.DontCare(null, param));
                    }
                    else if (value === 'hotel|guesthouse' && key === 'hotel-type') {
                        filterClauses.push(new thingtalk_1.Ast.BooleanExpression.Atom(null, 'type', 'in_array', new thingtalk_1.Ast.Value.Array([
                            new thingtalk_1.Ast.Value.Enum('hotel'),
                            new thingtalk_1.Ast.Value.Enum('guest_house')
                        ])));
                    }
                    else if (/^(centre|south|north|east|west)\|(centre|south|north|east|west)$/.test(value) && (key === 'restaurant-area' || key === 'attraction-area' || key === 'hotel-area')) {
                        const [, first, second] = /^(centre|south|north|east|west)\|(centre|south|north|east|west)$/.exec(value);
                        filterClauses.push(new thingtalk_1.Ast.BooleanExpression.Atom(null, 'area', 'in_array', new thingtalk_1.Ast.Value.Array([
                            new thingtalk_1.Ast.Value.Enum(first),
                            new thingtalk_1.Ast.Value.Enum(second)
                        ])));
                    }
                    else if (/^(cheap|moderate|expensive)\|(cheap|moderate|expensive)$/.test(value) && (key === 'restaurant-price-range' || key === 'attraction-price-range' || key === 'hotel-price-range')) {
                        const [, first, second] = /^(cheap|moderate|expensive)\|(cheap|moderate|expensive)$/.exec(value);
                        filterClauses.push(new thingtalk_1.Ast.BooleanExpression.Atom(null, 'price_range', 'in_array', new thingtalk_1.Ast.Value.Array([
                            new thingtalk_1.Ast.Value.Enum(first),
                            new thingtalk_1.Ast.Value.Enum(second)
                        ])));
                    }
                    else {
                        if (param === 'internet' || param === 'parking')
                            ttValue = new thingtalk_1.Ast.Value.Boolean(value !== 'no');
                        else if (param === 'leave_at' || param === 'arrive_by')
                            ttValue = parseTime(value);
                        else if (param === 'id' && contextInfo.current && getStatementDomain(contextInfo.current.stmt) === domain)
                            ttValue = new thingtalk_1.Ast.Value.Entity(null, tpClass + ':' + queryname, value);
                        else if (param === 'stars')
                            ttValue = new thingtalk_1.Ast.Value.Number(parseInt(value) || 0);
                        else if (param === 'area' || param === 'price_range' || param === 'day' || key === 'hotel-type')
                            ttValue = new thingtalk_1.Ast.Value.Enum(value.replace(/\s+/g, '_'));
                        else
                            ttValue = new thingtalk_1.Ast.Value.String(value);
                        if (ttValue instanceof thingtalk_1.Ast.EntityValue)
                            this._resolveEntity(ttValue);
                        let op = '==';
                        if (ttValue.isString)
                            op = '=~';
                        else if (param === 'leave_at')
                            op = '>=';
                        else if (param === 'arrive_by')
                            op = '<=';
                        filterClauses.push(new thingtalk_1.Ast.BooleanExpression.Atom(null, param, op, ttValue));
                    }
                }
                const filterTable = new thingtalk_1.Ast.FilterExpression(null, invocationTable, new thingtalk_1.Ast.BooleanExpression.And(null, filterClauses), null);
                const tableStmt = new thingtalk_1.Ast.ExpressionStatement(null, filterTable);
                newItems.push(new thingtalk_1.Ast.DialogueHistoryItem(null, tableStmt, null, 'accepted'));
            }
            if (newActionSlots.size && domain !== 'attraction') {
                const invocation = new thingtalk_1.Ast.Invocation(null, selector, action, [], null);
                for (const [key, initvalue] of slotBag) {
                    if (!key.startsWith(domain))
                        continue;
                    if (SEARCH_SLOTS.has(key) && !key.endsWith('name'))
                        continue;
                    let param = key.split('-').slice(1).join('_');
                    if (param === 'name')
                        param = domain;
                    if (param === 'arriveby')
                        param = 'arrive_by';
                    if (param === 'leaveat')
                        param = 'leave_at';
                    let value = initvalue;
                    if (value === 'dontcare') {
                        // ???
                        // ignore
                        continue;
                    }
                    if (value.indexOf('|') >= 0)
                        value = value.substring(0, value.indexOf('|'));
                    let ttValue;
                    if (param === 'leave_at' || param === 'arrive_by' || param === 'book_time')
                        ttValue = parseTime(value);
                    else if (param === 'book_people' || param === 'book_stay')
                        ttValue = new thingtalk_1.Ast.Value.Number(parseInt(value) || 0);
                    else if (param === 'book_day' || param === 'day')
                        ttValue = new thingtalk_1.Ast.Value.Enum(value);
                    else if (param === domain)
                        ttValue = new thingtalk_1.Ast.Value.Entity(null, tpClass + ':' + queryname, value);
                    else
                        ttValue = new thingtalk_1.Ast.Value.String(value);
                    if (ttValue instanceof thingtalk_1.Ast.EntityValue)
                        this._resolveEntity(ttValue);
                    invocation.in_params.push(new thingtalk_1.Ast.InputParam(null, param, ttValue));
                }
                const actionStmt = new thingtalk_1.Ast.ExpressionStatement(null, new thingtalk_1.Ast.InvocationExpression(null, invocation, null));
                newItems.push(new thingtalk_1.Ast.DialogueHistoryItem(null, actionStmt, null, 'accepted'));
            }
            else if (contextInfo.next) {
                newItems.push(contextInfo.next.clone());
            }
            const userTarget = new thingtalk_1.Ast.DialogueState(null, POLICY_NAME, 'execute', null, newItems);
            await userTarget.typecheck(this._schemas);
            return userTarget;
        }
    }
    _getActionDomains(dlg) {
        const domains = new Set();
        for (const turn of dlg.dialogue) {
            for (const slot of turn.belief_state) {
                assert_1.default(slot.act === 'inform');
                let [key, value] = slot.slots[0];
                assert_1.default(typeof key === 'string');
                assert_1.default(typeof value === 'string');
                key = key.replace(/ /g, '-').replace(/pricerange/, 'price-range');
                value = fixGeneralLabelError(key, value);
                if (value === 'none' || value === 'dontcare')
                    continue;
                if (SEARCH_SLOTS.has(key))
                    continue;
                const domain = key.split('-')[0];
                domains.add(domain);
            }
        }
        return domains;
    }
    _findTrainName(turn) {
        let name = undefined;
        for (const utterance of [turn.system_transcript, turn.transcript]) {
            for (const token of utterance.split(' ')) {
                if (/^tr[0-9]+$/i.test(token))
                    name = token;
            }
        }
        if (name)
            turn.belief_state.push({ slots: [['train-name', name]], act: 'inform' });
    }
    _extractSimulatorOverrides(utterance) {
        const car = /\b(black|white|red|yellow|blue|grey) (toyota|skoda|bmw|honda|ford|audi|lexus|volvo|volkswagen|tesla)\b/.exec(utterance);
        if (car)
            this._simulatorOverrides.set('car', car[0]);
        for (const token of utterance.split(' ')) {
            // a reference number is an 8 character token containing both letters and numbers
            if (token.length === 8 && /[a-z]/.test(token) && /[0-9]/.test(token))
                this._simulatorOverrides.set('reference_number', token);
        }
    }
    async _doDialogue(dlg) {
        const id = dlg.dialogue_idx;
        const actionDomains = this._getActionDomains(dlg);
        let context = null, contextInfo = { current: null, next: null }, simulatorState = undefined;
        const slotBag = new Map();
        const turns = [];
        for (let idx = 0; idx < dlg.dialogue.length; idx++) {
            const turn = dlg.dialogue[idx];
            this._findTrainName(turn);
            const turnId = id + '/' + idx;
            if (this._maxTurn && idx >= this._maxTurn)
                break;
            try {
                let contextCode = '', agentUtterance = '', agentTargetCode = '';
                if (context !== null) {
                    // use the next turn to find the values of the action output parameters (reference_number and car) if any
                    this._simulatorOverrides.clear();
                    agentUtterance = undoTradePreprocessing(turn.system_transcript);
                    this._extractSimulatorOverrides(agentUtterance);
                    // "execute" the context
                    const { newDialogueState, newExecutorState } = await this._simulator.execute(context, simulatorState);
                    context = newDialogueState;
                    simulatorState = newExecutorState;
                    for (const item of context.history) {
                        if (item.results === null)
                            continue;
                        if (item.results.results.length === 0)
                            continue;
                        const firstResult = item.results.results[0];
                        if (!firstResult.value.id)
                            continue;
                        item.results.results.sort((one, two) => {
                            const idone = one.value.id;
                            const idtwo = two.value.id;
                            if (!(idone instanceof thingtalk_1.Ast.EntityValue) ||
                                !(idtwo instanceof thingtalk_1.Ast.EntityValue))
                                return 0;
                            const onerank = agentUtterance.toLowerCase().indexOf(idone.display.toLowerCase());
                            const tworank = agentUtterance.toLowerCase().indexOf(idtwo.display.toLowerCase());
                            if (onerank === tworank)
                                return 0;
                            if (onerank === -1)
                                return 1;
                            if (tworank === -1)
                                return -1;
                            return onerank - tworank;
                        });
                    }
                    contextInfo = this._getContextInfo(context);
                    contextCode = context.prettyprint();
                    // do the agent
                    const agentTarget = await this._doAgentTurn(context, contextInfo, turn, agentUtterance, turnId);
                    const oldContext = context;
                    context = ThingTalkUtils.computeNewState(context, agentTarget, 'agent');
                    const prediction = ThingTalkUtils.computePrediction(oldContext, context, 'agent');
                    agentTargetCode = prediction.prettyprint();
                }
                const userUtterance = undoTradePreprocessing(turn.transcript);
                const userTarget = await this._doUserTurn(context, contextInfo, turn, userUtterance, slotBag, actionDomains, turnId);
                const oldContext = context;
                context = ThingTalkUtils.computeNewState(context, userTarget, 'user');
                const prediction = ThingTalkUtils.computePrediction(oldContext, context, 'user');
                const userTargetCode = prediction.prettyprint();
                turns.push({
                    context: contextCode,
                    agent: agentUtterance,
                    agent_target: agentTargetCode,
                    user: userUtterance,
                    user_target: userTargetCode,
                });
                // use the next turn to find the values of the action output parameters (reference_number and car) if any
                this._simulatorOverrides.clear();
                if (idx < dlg.dialogue.length - 1)
                    this._extractSimulatorOverrides(dlg.dialogue[idx + 1].system_transcript);
            }
            catch (e) {
                console.error(`Failed in dialogue ${id}`);
                console.error(turn);
                throw e;
            }
        }
        this.push({ id, turns });
        this._n++;
        this.emit('progress', this._n / this._N);
    }
    async run(data) {
        this._n = 0;
        this._N = data.length;
        for (let i = 0; i < data.length;) {
            // run 100 dialogues in parallel
            // Predictor will split the minibatch if necessary
            const promises = [];
            for (; i < data.length && promises.length < 1000; i++) {
                if (this._onlyMultidomain && data[i].domains.length === 1)
                    continue;
                promises.push(this._doDialogue(data[i]));
            }
            await Promise.all(promises);
        }
        this.emit('progress', 1);
        this.push(null);
    }
}
function initArgparse(subparsers) {
    const parser = subparsers.add_parser('auto-annotate-multiwoz', {
        add_help: true,
        description: `Heuristically convert multiwoz annotations to ThingTalk.`
    });
    parser.add_argument('-o', '--output', {
        required: true,
        type: fs.createWriteStream
    });
    parser.add_argument('--timezone', {
        required: false,
        default: undefined,
        help: `Timezone to use to print dates and times (defaults to the current timezone).`
    });
    parser.add_argument('--thingpedia', {
        required: true,
        help: 'Path to ThingTalk file containing class definitions.'
    });
    parser.add_argument('--database-file', {
        required: false,
        help: `Path to a file pointing to JSON databases used to simulate queries.`,
    });
    parser.add_argument('--user-nlu-server', {
        required: false,
        default: 'http://127.0.0.1:8400',
        help: `The URL of the natural language server to parse user utterances. Use a file:// URL pointing to a model directory to use a local instance of genienlp.`
    });
    parser.add_argument('--agent-nlu-server', {
        required: false,
        default: 'http://127.0.0.1:8400',
        help: `The URL of the natural language server to parse agent utterances. Use a file:// URL pointing to a model directory to use a local instance of genienlp.`
    });
    parser.add_argument('--only-multidomain', {
        required: false,
        action: 'store_true',
        help: 'Only translate multi-domain dialogues'
    });
    parser.add_argument('--use-existing', {
        required: false,
        action: 'store_true',
        default: true,
        help: 'Use existing annotations'
    });
    parser.add_argument('--no-use-existing', {
        required: false,
        action: 'store_false',
        dest: 'use_existing',
        help: 'Do not use existing annotations'
    });
    parser.add_argument('--max-turn', {
        required: false,
        help: 'Stop at the given turn when selftraining'
    });
    parser.add_argument('input_file', {
        help: 'Input dialog file'
    });
}
exports.initArgparse = initArgparse;
async function execute(args) {
    const data = JSON.parse(await util_1.default.promisify(fs.readFile)(args.input_file, { encoding: 'utf8' }));
    const converter = new Converter(args);
    const learned = new parsers_1.DialogueSerializer({ annotations: true });
    const promise = StreamUtils.waitFinish(converter.pipe(learned).pipe(args.output));
    const progbar = new progress_bar_1.default(1);
    converter.on('progress', (value) => {
        //console.log(value);
        progbar.update(value);
    });
    // issue an update now to show the progress bar
    progbar.update(0);
    await converter.start();
    await converter.run(data);
    await converter.stop();
    console.log('Finished, waiting for pending writes...');
    await promise;
    console.log('Everything done...');
    // we need this otherwise we hang at exit, due to some open file I cannot find...
    setTimeout(() => process.exit(), 10000);
}
exports.execute = execute;
//# sourceMappingURL=auto-annotate-multiwoz.js.map