"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.Processor = void 0;
var tfjs_node_1 = require("@tensorflow/tfjs-node");
var constants_1 = require("../../../preprocess/constants");
var constants_2 = require("./constants");
var Processor = /** @class */ (function () {
    function Processor(preprocessor, tokenEncoder, entityEncoder) {
        this.preprocessor = preprocessor;
        this.tokenEncoder = tokenEncoder;
        this.entityEncoder = entityEncoder;
    }
    // Processes samples and generates the Input and Output data.
    Processor.prototype.process = function (samples) {
        var _this = this;
        var processedSamples = samples.map(function (s) { return _this.processSample(s); });
        return {
            x: tfjs_node_1.tensor(processedSamples.map(function (s) { return s.x; })),
            y: tfjs_node_1.tensor(processedSamples.map(function (s) { return s.y; })),
        };
    };
    // Generates the Input data and the unmasked sequence.
    Processor.prototype.generateInput = function (text) {
        var sequence = this.preprocessor.truncate(this.preprocessor.pad(this.processText(text), constants_1.PADDING_TOKEN));
        var input = tfjs_node_1.tensor([this.processTokens(sequence)]);
        return { sequence: sequence, input: input };
    };
    Processor.prototype.processSample = function (sample) {
        var _this = this;
        var tokens = [];
        var entities = [];
        var start = 0;
        var partialSequence;
        var partialEntities;
        sample.entities.forEach(function (e) {
            var text = sample.text.slice(start, e.start);
            partialSequence = _this.processText(text);
            partialEntities = Array(partialSequence.length).fill(constants_2.NEUTRAL_ENTITY);
            tokens = tokens.concat(partialSequence);
            entities = entities.concat(partialEntities);
            text = e.text;
            partialSequence = _this.processText(text);
            partialEntities = Array(partialSequence.length).fill(e.label);
            tokens = tokens.concat(partialSequence);
            entities = entities.concat(partialEntities);
            start = e.end;
        });
        var text = sample.text.slice(start);
        partialSequence = this.processText(text);
        partialEntities = Array(partialSequence.length).fill(constants_2.NEUTRAL_ENTITY);
        tokens = tokens.concat(partialSequence);
        entities = entities.concat(partialEntities);
        return {
            x: this.processTokens(tokens),
            y: this.processEntities(entities),
        };
    };
    Processor.prototype.processText = function (text) {
        return this.preprocessor.stem(this.preprocessor.removeStopwords(this.preprocessor.tokenize(this.preprocessor.normalize(text))));
    };
    Processor.prototype.maskUnknownTokens = function (tokens) {
        var _this = this;
        return tokens.map(function (t) {
            return _this.tokenEncoder.items.includes(t) ? t : constants_1.UNKNOWN_TOKEN;
        });
    };
    Processor.prototype.processTokens = function (sequence) {
        return this.tokenEncoder.encode(this.preprocessor.truncate(this.preprocessor.pad(this.maskUnknownTokens(sequence), constants_1.PADDING_TOKEN)));
    };
    Processor.prototype.processEntities = function (sequence) {
        return this.entityEncoder.encode(this.preprocessor.truncate(this.preprocessor.pad(sequence, constants_2.NEUTRAL_ENTITY)));
    };
    return Processor;
}());
exports.Processor = Processor;
//# sourceMappingURL=processor.js.map