"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.TiktokenTokenizer = void 0;
const js_tiktoken_1 = require("js-tiktoken");
class TiktokenTokenizer {
    constructor() {
        this.enc = (0, js_tiktoken_1.getEncoding)('cl100k_base');
    }
    tokenize(content) {
        const tokens = [];
        const encoded = this.enc.encode(content);
        let currentIndex = 0;
        for (const tokenId of encoded) {
            const text = this.enc.decode([tokenId]);
            const length = text.length;
            tokens.push({
                text,
                type: 'bpe',
                startIndex: currentIndex,
                endIndex: currentIndex + length,
            });
            currentIndex += length;
        }
        return tokens;
    }
}
exports.TiktokenTokenizer = TiktokenTokenizer;
//# sourceMappingURL=tiktoken-tokenizer.js.map