"use strict";
// -*- mode: js; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2019-2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Giovanni Campagna <gcampagn@cs.stanford.edu>
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
    if (mod && mod.__esModule) return mod;
    var result = {};
    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
    __setModuleDefault(result, mod);
    return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.execute = exports.initArgparse = void 0;
const fs = __importStar(require("fs"));
const stream_1 = __importDefault(require("stream"));
const csv_parse_1 = __importDefault(require("csv-parse"));
const csv_stringify_1 = __importDefault(require("csv-stringify"));
const StreamUtils = __importStar(require("../lib/utils/stream-utils"));
const I18n = __importStar(require("../lib/i18n"));
const argutils_1 = require("./lib/argutils");
class TokenizerStream extends stream_1.default.Transform {
    constructor(options) {
        super({ objectMode: true });
        this._tokenizer = I18n.get(options.locale).getTokenizer();
    }
    _transform(row, encoding, callback) {
        if (row.length < 1 || !row[0]) {
            callback();
            return;
        }
        let value, preprocessed, weight;
        if (row.length === 1) {
            value = row[0];
            weight = 1.0;
        }
        else if (row.length === 2) {
            if (isFinite(+row[1])) {
                value = row[0];
                weight = row[1];
            }
            else {
                value = row[0];
                preprocessed = row[1];
                weight = 1.0;
            }
        }
        else {
            value = row[0];
            preprocessed = row[1];
            weight = parseFloat(row[2]) || 1.0;
        }
        if (!(weight > 0.0))
            weight = 1.0;
        if (preprocessed !== undefined) {
            callback(null, [value, preprocessed, weight]);
        }
        else {
            const result = this._tokenizer.tokenize(value);
            // ignore lines with uppercase (entity) tokens
            if (result.tokens.some((t) => /[A-Z]/.test(t)))
                callback(null);
            else
                callback(null, [value, result.tokens.join(' '), weight]);
        }
    }
    _flush(callback) {
        process.nextTick(callback);
    }
}
function initArgparse(subparsers) {
    const parser = subparsers.add_parser('preprocess-string-dataset', {
        add_help: true,
        description: "Preprocess (tokenize) a string value dataset."
    });
    parser.add_argument('-l', '--locale', {
        required: false,
        default: 'en-US',
        help: `BGP 47 locale tag of the language to use for tokenization (defaults to 'en-US', English)`
    });
    parser.add_argument('-o', '--output', {
        required: true,
        type: fs.createWriteStream
    });
    parser.add_argument('input_file', {
        nargs: '+',
        type: argutils_1.maybeCreateReadStream,
        help: 'Input string datasets to tokenize (in TSV format); use - for standard input'
    });
}
exports.initArgparse = initArgparse;
async function execute(args) {
    await StreamUtils.waitFinish(StreamUtils.chain(args.input_file, {})
        .pipe(csv_parse_1.default({ delimiter: '\t', relax: true }))
        .pipe(new TokenizerStream(args))
        .pipe(csv_stringify_1.default({ delimiter: '\t' }))
        .pipe(args.output));
}
exports.execute = execute;
//# sourceMappingURL=preprocess-string-dataset.js.map