"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.Preprocessor = void 0;
/* eslint-disable @typescript-eslint/unbound-method */
var constants_1 = require("./constants");
var normalizer_1 = require("./preprocessing-tools/normalizer");
var stemmer_1 = require("./preprocessing-tools/stemmer");
var tokenizer_1 = require("./preprocessing-tools/tokenizer");
var Preprocessor = /** @class */ (function () {
    function Preprocessor(language, maxSeqLen, vocabulary, normalizer, tokenizer, stemmer) {
        if (normalizer === void 0) { normalizer = new normalizer_1.DefaultNormalizer(); }
        if (tokenizer === void 0) { tokenizer = new tokenizer_1.DefaultTokenizer(); }
        if (stemmer === void 0) { stemmer = new stemmer_1.DefaultStemmer(); }
        this.language = language;
        this.maxSeqLen = maxSeqLen;
        this.vocabulary = vocabulary;
        this.normalizer = normalizer;
        this.tokenizer = tokenizer;
        this.stemmer = stemmer;
    }
    Preprocessor.fromModelData = function (modelData, engines) {
        return new Preprocessor(modelData.language, modelData.maxSeqLen, modelData.vocabulary, engines.normalizer, engines.tokenizer, engines.stemmer);
    };
    Preprocessor.fromData = function (data, language, maxSeqLen, engines) {
        var id = 0;
        var vocabulary = {};
        vocabulary[constants_1.UNKNOWN_TOKEN] = id;
        id++;
        data.forEach(function (sample) {
            var normalizedSentence = engines.normalizer.normalize(sample.feature);
            var tokens = engines.tokenizer.tokenize(normalizedSentence);
            var stemmedTokens = tokens.map(function (token) {
                return engines.stemmer.stem(token, language);
            });
            stemmedTokens.forEach(function (token) {
                if (!(token in vocabulary)) {
                    vocabulary[token] = id;
                    id++;
                }
            });
        });
        return new Preprocessor(language, maxSeqLen, vocabulary, engines.normalizer, engines.tokenizer, engines.stemmer);
    };
    Preprocessor.prototype.preprocess = function (sentence) {
        var _this = this;
        var normalizedSentence = this.normalizer.normalize(sentence);
        var tokens = this.tokenizer.tokenize(normalizedSentence);
        var stemmedTokens = tokens.map(function (token) {
            return _this.stemmer.stem(token, _this.language);
        });
        var sequence = this.computeSequence(stemmedTokens);
        return sequence;
    };
    Preprocessor.prototype.computeSequence = function (tokens) {
        var _this = this;
        var sequence = tokens.map(function (token) { var _a; return (_a = _this.vocabulary[token]) !== null && _a !== void 0 ? _a : 0; });
        sequence = sequence.slice(0, this.maxSeqLen);
        return Array(this.maxSeqLen - sequence.length)
            .fill(0)
            .concat(sequence);
    };
    return Preprocessor;
}());
exports.Preprocessor = Preprocessor;
//# sourceMappingURL=preprocessor.js.map