"use strict";
// -*- mode: typescript; indent-tabs-mode: nil; js-basic-offset: 4 -*-
//
// This file is part of Genie
//
// Copyright 2020 The Board of Trustees of the Leland Stanford Junior University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Silei Xu <silei@cs.stanford.edu>
//         Giovanni Campagna <gcampagn@cs.stanford.edu>
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
    if (mod && mod.__esModule) return mod;
    var result = {};
    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
    __setModuleDefault(result, mod);
    return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.ParaphraseValidatorFilter = exports.ParaphraseValidator = void 0;
const stream_1 = __importDefault(require("stream"));
const I18n = __importStar(require("../../i18n"));
const Utils = __importStar(require("../../utils/misc-utils"));
const ThingTalkUtils = __importStar(require("../../utils/thingtalk"));
class ParaphraseValidator {
    constructor(schemaRetriever, langPack, tokenizer, locale, row, counter, debug) {
        this.target_preprocessed = [];
        this.context_preprocessed = [];
        this.preprocessed = [];
        this.entities = {};
        this._locale = locale;
        this._schemas = schemaRetriever;
        this._tokenizer = tokenizer;
        this._noIdea = langPack.NO_IDEA;
        this._counter = counter;
        this.id = row.id;
        this.target_code = row.target_code;
        this.paraphrase = row.paraphrase;
        this.synthetic = row.synthetic;
        this.context = row.context;
        this._debug = debug;
    }
    async clean() {
        this.ast = await ThingTalkUtils.parse(this.target_code, this._schemas);
        this.paraphrase = this.paraphrase.replace(/([.?])(["”])/g, '$2$1');
        const tokenized = this._tokenizer.tokenize(this.paraphrase);
        if (this.context && this.context !== 'null') {
            const context = await ThingTalkUtils.parse(this.context, this._schemas);
            const contextEntities = {};
            [this.context_preprocessed,] = ThingTalkUtils.serializeNormalized(context, contextEntities);
            Utils.renumberEntities(tokenized, contextEntities);
        }
        else {
            this.context_preprocessed = ['null'];
        }
        this.preprocessed = tokenized.tokens;
        this.entities = tokenized.entities;
    }
    isValid() {
        if (this.isNoIdea())
            return false;
        else if (!this.checkValues())
            return false;
        return true;
    }
    isNoIdea() {
        if (this.paraphrase.length < 5) {
            this._counter['no_idea'] += 1;
            return true;
        }
        for (const noidea of this._noIdea) {
            if (this.paraphrase === noidea || this.paraphrase.indexOf(noidea) > -1) {
                this._counter['no_idea'] += 1;
                return true;
            }
        }
        return false;
    }
    checkValues() {
        // try to conver to NN syntax
        // this will automatically trigger entity assignment in ThingTalk
        try {
            const target_code = ThingTalkUtils.serializePrediction(this.ast, this.preprocessed, this.entities, {
                locale: this._locale
            });
            this.target_preprocessed = target_code;
            let inString = false;
            let prevToken = null;
            for (const token of target_code) {
                if (token === '"') {
                    inString = !inString;
                    prevToken = token;
                    continue;
                }
                if (inString)
                    continue;
                if (prevToken === '"') {
                    // we just closed a literal string in the thingtalk code
                    // if the string was of String, Entity(tt:hashtag) or Entity(tt:username) type,
                    // we reject the program, as those should be QUOTED_STRING/HASHTAG/USERNAME entities
                    // instead, so we can replace them later
                    if (!token.startsWith('^^') || token === '^^tt:hashtag' || token === '^^tt:username') {
                        this._counter['quoting']++;
                        return false;
                    }
                }
                prevToken = token;
            }
            if (prevToken === '"') {
                // the last token was a '"', closing a literal string (unquoted string)
                // reject the program with bad quoting
                this._counter['quoting']++;
                return false;
            }
            return true;
        }
        catch (e) {
            if (this._debug)
                console.log(String(this.id), e.message, '|||', this.paraphrase);
            // HACK should be an error subclass in ThingTalk
            if (!e.message.startsWith('Cannot find entity'))
                throw e;
            this._counter['values']++;
            return false;
        }
    }
}
exports.ParaphraseValidator = ParaphraseValidator;
class ParaphraseValidatorFilter extends stream_1.default.Transform {
    constructor(schemaRetriever, options) {
        super({
            readableObjectMode: true,
            writableObjectMode: true
        });
        this._schemas = schemaRetriever;
        this._locale = options.locale;
        this._langPack = I18n.get(options.locale);
        this._tokenizer = this._langPack.getTokenizer();
        this._counter = {
            'good': 0,
            'no_idea': 0,
            'values': 0,
            'quoting': 0,
            'manual': 0
        };
        this._debug = options.debug;
        this._validationCounts = options.validationCounts;
        this._validationThreshold = options.validationThreshold || 0;
    }
    async _validate(row) {
        const paraphrase = new ParaphraseValidator(this._schemas, this._langPack, this._tokenizer, this._locale, row, this._counter, this._debug);
        try {
            await paraphrase.clean();
            if (!paraphrase.isValid()) {
                //if (this._debug)
                //    console.log(`Rejected paraphrase ${row.id} (of ${row.synthetic_id}): ${row.paraphrase}`);
                return null;
            }
            if (this._validationCounts) {
                const count = this._validationCounts.get(String(row.id));
                // `count` could be undefined if this paraphrase was not voted on at all
                // (either it was excluded from the mturk validation task because its
                // corresponding synthetic had not enough paraphrases, or all validators
                // working on this paraphrase were rejected
                if (!count || !(count.same_count >= this._validationThreshold)) {
                    this._counter['manual']++;
                    return null;
                }
            }
            row.preprocessed = paraphrase.preprocessed.join(' ');
            row.target_preprocessed = paraphrase.target_preprocessed.join(' ');
            if (row.context)
                row.context_preprocessed = paraphrase.context_preprocessed.join(' ');
            this._counter['good']++;
            return row;
        }
        catch (e) {
            console.error(`Failed paraphrase ${row.id} (${row.synthetic_id}): ${e.message}`);
            return null;
        }
    }
    _transform(row, encoding, callback) {
        this._validate(row).then((validated) => callback(null, validated), (err) => callback(err));
    }
    _flush(callback) {
        if (this._debug) {
            console.log('Validation Statistics:');
            for (const key in this._counter)
                console.log(`  ${key} = ${this._counter[key]}`);
        }
        process.nextTick(callback);
    }
}
exports.ParaphraseValidatorFilter = ParaphraseValidatorFilter;
//# sourceMappingURL=validator.js.map