"use strict";
// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2019-2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
    if (mod && mod.__esModule) return mod;
    var result = {};
    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
    __setModuleDefault(result, mod);
    return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.execute = exports.initArgparse = void 0;
const Tp = __importStar(require("thingpedia"));
const fs = __importStar(require("fs"));
const parsers_1 = require("../lib/dataset-tools/parsers");
const sentence_evaluator_1 = require("../lib/dataset-tools/evaluation/sentence_evaluator");
const ParserClient = __importStar(require("../lib/prediction/parserclient"));
const argutils_1 = require("./lib/argutils");
const evaluate_common_1 = require("./lib/evaluate-common");
function initArgparse(subparsers) {
    const parser = subparsers.add_parser('evaluate-server', {
        add_help: true,
        description: "Evaluate a trained model on a Genie-generated dataset, by contacting a running Genie server."
    });
    parser.add_argument('-o', '--output', {
        required: false,
        default: process.stdout,
        type: fs.createWriteStream,
        help: "Write results to this file instead of stdout"
    });
    parser.add_argument('--url', {
        required: false,
        help: "URL of the server to evaluate. Use a file:// URL pointing to a model directory to evaluate using a local instance of genienlp",
        default: 'http://127.0.0.1:8400',
    });
    parser.add_argument('--tokenized', {
        required: false,
        action: 'store_true',
        default: true,
        help: "The dataset is already tokenized (this is the default)."
    });
    parser.add_argument('--no-tokenized', {
        required: false,
        dest: 'tokenized',
        action: 'store_false',
        help: "The dataset is not already tokenized."
    });
    parser.add_argument('--thingpedia', {
        required: true,
        help: 'Path to ThingTalk file containing class definitions.'
    });
    parser.add_argument('input_file', {
        nargs: '+',
        type: argutils_1.maybeCreateReadStream,
        help: 'Input datasets to evaluate (in TSV format); use - for standard input'
    });
    parser.add_argument('-l', '--locale', {
        required: false,
        default: 'en-US',
        help: `BGP 47 locale tag of the language to evaluate (defaults to 'en-US', English)`
    });
    parser.add_argument('-t', '--target-language', {
        required: false,
        default: 'thingtalk',
        choices: ['thingtalk', 'dlgthingtalk'],
        help: `The programming language to generate`
    });
    parser.add_argument('--contextual', {
        action: 'store_true',
        help: 'Process a contextual dataset.',
        default: false
    });
    parser.add_argument('--split-by-device', {
        action: 'store_true',
        help: 'Compute evaluation statistics separating examples by Thingpedia device',
        default: false
    });
    parser.add_argument('--debug', {
        action: 'store_true',
        help: 'Enable debugging.',
        default: true
    });
    parser.add_argument('--no-debug', {
        action: 'store_false',
        dest: 'debug',
        help: 'Disable debugging.',
    });
    parser.add_argument('--csv', {
        action: 'store_true',
        help: 'Output a single CSV line',
    });
    parser.add_argument('--csv-prefix', {
        required: false,
        default: '',
        help: `Prefix all output lines with this string`
    });
    parser.add_argument('--complexity-metric', {
        choices: ['num_params', 'turn_number'],
        default: 'num_params',
        help: `Complexity metric to use to divide examples by complexity`
    });
    parser.add_argument('--max-complexity', {
        required: false,
        type: Number,
        default: '',
        help: 'Collapse all examples of complexity greater or equal to this',
    });
    parser.add_argument('--min-complexity', {
        required: false,
        type: Number,
        default: 0,
        help: 'Collapse all examples of complexity smaller or equal to this',
    });
    parser.add_argument('--oracle', {
        action: 'store_true',
        help: 'Indicates evaluation of an oracle model where ThingTalk code should be passed to the genienlp server',
        default: false
    });
    parser.add_argument('--offset', {
        required: false,
        type: Number,
        default: 0,
        help: 'Start evaluation from this line of input data',
    });
}
exports.initArgparse = initArgparse;
async function execute(args) {
    let tpClient = null;
    if (args.thingpedia)
        tpClient = new Tp.FileClient(args);
    const parser = ParserClient.get(args.url, args.locale);
    await parser.start();
    const output = argutils_1.readAllLines(args.input_file)
        .pipe(new parsers_1.DatasetParser({ contextual: args.contextual, preserveId: true, parseMultiplePrograms: true, offset: args.offset }))
        .pipe(new sentence_evaluator_1.SentenceEvaluatorStream(parser, {
        locale: args.locale,
        targetLanguage: args.target_language,
        thingpediaClient: tpClient,
        tokenized: args.tokenized,
        debug: args.debug,
        complexityMetric: args.complexity_metric,
        oracle: args.oracle
    }))
        .pipe(new sentence_evaluator_1.CollectSentenceStatistics({
        minComplexity: args.min_complexity,
        maxComplexity: args.max_complexity,
        splitByDevice: args.split_by_device
    }));
    const result = await output.read();
    evaluate_common_1.outputResult(args, result);
    await parser.stop();
}
exports.execute = execute;
//# sourceMappingURL=evaluate-server.js.map