"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.countChatTokens = exports.countTokens = void 0;
var js_tiktoken_1 = require("js-tiktoken");
var DEFAULT_ENCODING = 'o200k_base';
var resolveEncoding = function (model) {
    var normalized = String(model || '').trim();
    if (normalized) {
        try {
            return (0, js_tiktoken_1.encodingForModel)(normalized);
        }
        catch (_a) { }
    }
    return (0, js_tiktoken_1.getEncoding)(DEFAULT_ENCODING);
};
var countTokens = function (text, model) {
    var encoding = resolveEncoding(model);
    try {
        var tokens = encoding.encode(text || '');
        return tokens.length;
    }
    finally {
        var maybeEncoding = encoding;
        if (typeof (maybeEncoding === null || maybeEncoding === void 0 ? void 0 : maybeEncoding.free) === 'function') {
            maybeEncoding.free();
        }
        else if (typeof (maybeEncoding === null || maybeEncoding === void 0 ? void 0 : maybeEncoding.dispose) === 'function') {
            maybeEncoding.dispose();
        }
    }
};
exports.countTokens = countTokens;
var countChatTokens = function (messages, model) {
    var flattened = (messages || [])
        .map(function (msg) { return "".concat((msg === null || msg === void 0 ? void 0 : msg.role) || 'user', ":\n").concat((msg === null || msg === void 0 ? void 0 : msg.content) || ''); })
        .join('\n\n');
    return (0, exports.countTokens)(flattened, model);
};
exports.countChatTokens = countChatTokens;

//# sourceMappingURL=tokenizer.js.map
