"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const globals_1 = require("@jest/globals");
const tokenizer_1 = require("./tokenizer");
globals_1.jest.setTimeout(20000);
const model = "openai:gpt-5";
(0, globals_1.beforeAll)(async () => {
    // warm up the worker_thread and tokenizer before running tests
    await (0, globals_1.expect)((0, tokenizer_1.loadTokenizerModules)([model])).resolves.toHaveLength(1);
});
(0, globals_1.beforeEach)(() => {
    (0, tokenizer_1.__resetTokenizerForTests)();
});
(0, globals_1.describe)("tokenizer", () => {
    (0, globals_1.test)("loadTokenizerModules warms known encodings", async () => {
        const tokenizer = await (0, tokenizer_1.getTokenizerForModel)(model);
        (0, globals_1.expect)(typeof tokenizer.encoding).toBe("string");
        (0, globals_1.expect)(tokenizer.encoding.length).toBeGreaterThan(0);
    });
    (0, globals_1.test)("countTokens returns stable values", async () => {
        const text = "cmux-tokenizer-smoke-test";
        const first = await (0, tokenizer_1.countTokens)(model, text);
        const second = await (0, tokenizer_1.countTokens)(model, text);
        (0, globals_1.expect)(first).toBeGreaterThan(0);
        (0, globals_1.expect)(second).toBe(first);
    });
    (0, globals_1.test)("countTokensBatch matches individual calls", async () => {
        const texts = ["alpha", "beta", "gamma"];
        const batch = await (0, tokenizer_1.countTokensBatch)(model, texts);
        (0, globals_1.expect)(batch).toHaveLength(texts.length);
        const individual = await Promise.all(texts.map((text) => (0, tokenizer_1.countTokens)(model, text)));
        (0, globals_1.expect)(batch).toEqual(individual);
    });
});
//# sourceMappingURL=tokenizer.test.js.map