"use strict";
// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2019-2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
    if (mod && mod.__esModule) return mod;
    var result = {};
    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
    __setModuleDefault(result, mod);
    return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.execute = exports.initArgparse = void 0;
const assert_1 = __importDefault(require("assert"));
const ThingTalk = __importStar(require("thingtalk"));
const Tp = __importStar(require("thingpedia"));
const stream_1 = __importDefault(require("stream"));
const fs = __importStar(require("fs"));
const parsers_1 = require("../lib/dataset-tools/parsers");
const StreamUtils = __importStar(require("../lib/utils/stream-utils"));
const Utils = __importStar(require("../lib/utils/misc-utils"));
const I18n = __importStar(require("../lib/i18n"));
const ThingTalkUtils = __importStar(require("../lib/utils/thingtalk"));
const progress_bar_1 = __importDefault(require("./lib/progress_bar"));
const argutils_1 = require("./lib/argutils");
class DialogueToTurnStream extends stream_1.default.Transform {
    constructor(options) {
        super({ objectMode: true });
        this._locale = options.locale;
        this._options = options;
        this._side = options.side;
        this._flags = options.flags;
        this._idPrefix = options.idPrefix;
        this._dedupe = options.deduplicate ? new Set : undefined;
        this._tokenized = options.tokenized;
        this._tokenizer = null;
        if (!this._tokenized)
            this._tokenizer = I18n.get(this._locale).getTokenizer();
    }
    _preprocess(sentence, contextEntities) {
        let tokenized;
        if (this._tokenized) {
            const tokens = sentence.split(' ');
            const entities = Utils.makeDummyEntities(sentence);
            tokenized = { tokens, entities };
        }
        else {
            tokenized = this._tokenizer.tokenize(sentence);
        }
        Utils.renumberEntities(tokenized, contextEntities);
        return tokenized;
    }
    _getDedupeKey(context, utterance) {
        return context.join(' ') + '<sep>' + utterance.join(' ');
    }
    async _emitAgentTurn(i, turn, dlg) {
        if (i === 0)
            return;
        const context = await ThingTalkUtils.parse(turn.context, this._options);
        assert_1.default(context instanceof ThingTalk.Ast.DialogueState);
        const agentContext = ThingTalkUtils.prepareContextForPrediction(context, 'agent');
        const [contextCode, contextEntities] = ThingTalkUtils.serializeNormalized(agentContext);
        const agentTarget = await ThingTalkUtils.parse(turn.agent_target, this._options);
        assert_1.default(agentTarget instanceof ThingTalk.Ast.DialogueState);
        const { tokens, entities } = this._preprocess(turn.agent, contextEntities);
        const agentCode = await ThingTalkUtils.serializePrediction(agentTarget, tokens, entities, {
            locale: this._locale
        });
        if (this._dedupe) {
            const key = this._getDedupeKey(contextCode, tokens);
            if (this._dedupe.has(key))
                return;
            this._dedupe.add(key);
        }
        this.push({
            id: this._flags + '' + this._idPrefix + dlg.id + '/' + i,
            context: contextCode.join(' '),
            preprocessed: tokens.join(' '),
            target_code: agentCode.join(' ')
        });
    }
    async _emitUserTurn(i, turn, dlg) {
        let context, contextCode, contextEntities;
        if (i > 0) {
            // if we have an "intermediate context" (C: block after AT:) we ran the execution
            // after the agent spoke, so we don't need to apply the agent turn any more
            //
            // (this occurs only when annotating multiwoz data, when the agent chooses to complete
            // an action with incomplete information, choosing the value spontaneously)
            if (turn.intermediate_context) {
                context = await ThingTalkUtils.parse(turn.intermediate_context, this._options);
                assert_1.default(context instanceof ThingTalk.Ast.DialogueState);
            }
            else {
                context = await ThingTalkUtils.parse(turn.context, this._options);
                assert_1.default(context instanceof ThingTalk.Ast.DialogueState);
                // apply the agent prediction to the context to get the state of the dialogue before
                // the user speaks
                const agentPrediction = await ThingTalkUtils.parse(turn.agent_target, this._options);
                assert_1.default(agentPrediction instanceof ThingTalk.Ast.DialogueState);
                context = ThingTalkUtils.computeNewState(context, agentPrediction, 'agent');
            }
            const userContext = ThingTalkUtils.prepareContextForPrediction(context, 'user');
            [contextCode, contextEntities] = ThingTalkUtils.serializeNormalized(userContext);
        }
        else {
            context = null;
            contextCode = ['null'];
            contextEntities = {};
        }
        const { tokens, entities } = this._preprocess(turn.user, contextEntities);
        const userTarget = await ThingTalkUtils.parse(turn.user_target, this._options);
        assert_1.default(userTarget instanceof ThingTalk.Ast.DialogueState);
        const code = await ThingTalkUtils.serializePrediction(userTarget, tokens, entities, {
            locale: this._locale
        });
        if (this._dedupe) {
            const key = this._getDedupeKey(contextCode, tokens);
            if (this._dedupe.has(key))
                return;
            this._dedupe.add(key);
        }
        this.push({
            id: this._flags + '' + this._idPrefix + dlg.id + '/' + i,
            context: contextCode.join(' '),
            preprocessed: tokens.join(' '),
            target_code: code.join(' ')
        });
    }
    async _doTransform(dlg) {
        for (let i = 0; i < dlg.length; i++) {
            const turn = dlg[i];
            try {
                if (this._side === 'agent')
                    await this._emitAgentTurn(i, turn, dlg);
                else
                    await this._emitUserTurn(i, turn, dlg);
            }
            catch (e) {
                console.error('Failed in dialogue ' + dlg.id);
                console.error(turn);
                throw e;
            }
        }
    }
    _transform(dialog, encoding, callback) {
        this._doTransform(dialog).then(() => callback(null), callback);
    }
    _flush(callback) {
        process.nextTick(callback);
    }
}
function initArgparse(subparsers) {
    const parser = subparsers.add_parser('dialog-to-contextual', {
        add_help: true,
        description: "Transform a dialog input file into a contextual dataset (turn by turn)."
    });
    parser.add_argument('-o', '--output', {
        required: true,
        type: fs.createWriteStream
    });
    parser.add_argument('-l', '--locale', {
        required: false,
        default: 'en-US',
        help: `BGP 47 locale tag of the language to evaluate (defaults to 'en-US', English)`
    });
    parser.add_argument('--tokenized', {
        action: 'store_true',
        default: true,
        help: "The dataset is already tokenized (this is the default)."
    });
    parser.add_argument('--no-tokenized', {
        dest: 'tokenized',
        action: 'store_false',
        help: "The dataset is not already tokenized."
    });
    parser.add_argument('--thingpedia', {
        required: true,
        help: 'Path to ThingTalk file containing class definitions.'
    });
    parser.add_argument('-t', '--target-language', {
        required: false,
        default: 'thingtalk',
        choices: ['thingtalk', 'dlgthingtalk'],
        help: `The programming language to generate`
    });
    parser.add_argument('--side', {
        required: true,
        choices: ['user', 'agent'],
        help: 'Which side of the conversation should be extracted.'
    });
    parser.add_argument('--flags', {
        required: false,
        default: '',
        help: 'Additional flags to add to the generated training examples.'
    });
    parser.add_argument('--id-prefix', {
        required: false,
        default: '',
        help: 'Prefix to add to all sentence IDs (useful to combine multiple datasets).'
    });
    parser.add_argument('--deduplicate', {
        action: 'store_true',
        default: false,
        help: 'Do not output duplicate turns (with the same preprocessed context and utterance)'
    });
    parser.add_argument('--no-deduplicate', {
        action: 'store_false',
        dest: 'deduplicate',
        help: 'Output duplicate turns (with the same preprocessed context and utterance)'
    });
    parser.add_argument('input_file', {
        nargs: '+',
        type: argutils_1.maybeCreateReadStream,
        help: 'Input dialog file; use - for standard input'
    });
    parser.add_argument('--debug', {
        action: 'store_true',
        help: 'Enable debugging.',
        default: true
    });
    parser.add_argument('--no-debug', {
        action: 'store_false',
        dest: 'debug',
        help: 'Disable debugging.',
    });
}
exports.initArgparse = initArgparse;
async function execute(args) {
    let tpClient = null;
    if (args.thingpedia)
        tpClient = new Tp.FileClient(args);
    const counter = new StreamUtils.CountStream();
    argutils_1.readAllLines(args.input_file, '====')
        .pipe(new parsers_1.DialogueParser())
        .pipe(counter)
        .pipe(new DialogueToTurnStream({
        locale: args.locale,
        thingpediaClient: tpClient,
        flags: args.flags,
        idPrefix: args.id_prefix,
        side: args.side,
        tokenized: args.tokenized,
        deduplicate: args.deduplicate,
        debug: args.debug
    }))
        .pipe(new parsers_1.DatasetStringifier())
        .pipe(args.output);
    const progbar = new progress_bar_1.default(1);
    counter.on('progress', (value) => {
        //console.log(value);
        progbar.update(value);
    });
    // issue an update now to show the progress bar
    progbar.update(0);
    await StreamUtils.waitFinish(args.output);
}
exports.execute = execute;
//# sourceMappingURL=dialog-to-contextual.js.map