"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.Dataset = void 0;
var constants_1 = require("../preprocess/constants");
var array_utils_1 = require("../utils/array-utils");
var loader_1 = require("./loader");
var Dataset = /** @class */ (function () {
    function Dataset(classes, entities, samples) {
        this.classes = classes;
        this.entities = entities;
        this.samples = samples;
    }
    Dataset.load = function (path) {
        return loader_1.DatasetLoader.load(path);
    };
    Dataset.prototype.split = function (testProportion, shuffle) {
        if (testProportion === void 0) { testProportion = 0.2; }
        if (shuffle === void 0) { shuffle = true; }
        if (1 < testProportion || testProportion < 0) {
            throw new RangeError("testProportion must be a number between 0 and 1.");
        }
        var samples = shuffle ? array_utils_1.shuffle(this.samples) : this.samples;
        var trainSamples = samples.slice(testProportion * samples.length);
        var testSamples = samples.slice(0, testProportion * samples.length);
        return {
            trainSet: new Dataset(this.classes, this.entities, trainSamples),
            testSet: new Dataset(this.classes, this.entities, testSamples),
        };
    };
    Dataset.prototype.extractVocabulary = function (preprocessor) {
        var sequences = this.samples.map(function (sample) {
            return preprocessor.preprocess(sample.text, constants_1.PADDING_TOKEN);
        });
        var datasetTokens = array_utils_1.flatten(sequences);
        var tokens = [constants_1.PADDING_TOKEN, constants_1.UNKNOWN_TOKEN].concat(datasetTokens);
        return array_utils_1.unique(tokens);
    };
    Object.defineProperty(Dataset.prototype, "length", {
        get: function () {
            return this.samples.length;
        },
        enumerable: false,
        configurable: true
    });
    return Dataset;
}());
exports.Dataset = Dataset;
//# sourceMappingURL=dataset.js.map