"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.getTokenLengthByInput = void 0;
const js_tiktoken_1 = require("js-tiktoken");
let encoderCache = {};
const getTokenLengthByInput = (text, modelName = "gpt-3.5-turbo") => __awaiter(void 0, void 0, void 0, function* () {
    let encoder = encoderCache[modelName];
    if (!encoder) {
        encoder = yield (0, js_tiktoken_1.encodingForModel)(modelName);
        encoderCache[modelName] = encoder;
        if (!(0, js_tiktoken_1.encodingForModel)(modelName)) {
            throw new Error(`Unknown model ${modelName}`);
        }
    }
    // automatically free
    const numTokens = encoder.encode(text).length;
    return numTokens;
});
exports.getTokenLengthByInput = getTokenLengthByInput;
