/**
 * Token Counter
 *
 * Provides utilities for counting tokens in strings and messages.
 * Uses tiktoken for accuracy, with character-based fallback for robustness.
 */
import { encoding_for_model, get_encoding } from 'tiktoken';
import { logger } from '../utils/logger.js';
/**
 * Creates a token counter instance for a specific model.
 * Automatically handles fallbacks if tiktoken or specific model encodings are unavailable.
 *
 * @param model - The model name (e.g., 'gpt-4', 'gpt-3.5-turbo').
 * @returns A TokenCounter implementation.
 */
export function createTokenCounter(model = 'gpt-4') {
    let encoder;
    try {
        // Try to get encoding for specific model
        encoder = encoding_for_model(model);
    }
    catch (_error) {
        // Fallback to cl100k_base (used by GPT-4, GPT-3.5-turbo)
        logger.debug(`Could not load encoding for model ${model}, falling back to cl100k_base`);
        try {
            encoder = get_encoding('cl100k_base');
        }
        catch (_e) {
            // Ultimate fallback to estimation if tiktoken fails completely (e.g. wasm issues)
            logger.warn('Failed to initialize tiktoken, using character-based estimation');
            return new EstimatingTokenCounter();
        }
    }
    return new TiktokenCounter(encoder);
}
/**
 * Token counter implementation using the tiktoken library.
 */
class TiktokenCounter {
    encoder;
    constructor(encoder) {
        this.encoder = encoder;
    }
    countTokens(text) {
        if (!text)
            return 0;
        try {
            return this.encoder.encode(text).length;
        }
        catch (error) {
            logger.warn('Token counting failed, falling back to estimation', { error });
            return Math.ceil(text.length / 4);
        }
    }
    countMessageTokens(messages) {
        let numTokens = 0;
        // Per-message overhead (format dependent, simplified for approximation)
        // <|start|>{role}\n{content}<|end|>
        const tokensPerMessage = 3;
        for (const message of messages) {
            numTokens += tokensPerMessage;
            // Role
            numTokens += this.countTokens(message.role);
            // Content - handle both string and array formats
            if (message.content) {
                if (typeof message.content === 'string') {
                    numTokens += this.countTokens(message.content);
                }
                else if (Array.isArray(message.content)) {
                    // Handle content parts array (OpenAI format)
                    for (const part of message.content) {
                        if (typeof part === 'object' && part !== null && 'text' in part) {
                            numTokens += this.countTokens(part.text);
                        }
                    }
                }
            }
            // Tool calls overhead
            if (message.tool_calls && Array.isArray(message.tool_calls)) {
                numTokens += this.countTokens(JSON.stringify(message.tool_calls));
            }
        }
        numTokens += 3; // Priming tokens for next response
        return numTokens;
    }
    estimateStreamingTokens(chunk) {
        return this.countTokens(chunk);
    }
    dispose() {
        try {
            this.encoder.free();
        }
        catch {
            // Ignore cleanup errors
        }
    }
}
/**
 * Fallback estimator when tiktoken is unavailable (e.g., in environments
 * where WASM or the library cannot load).
 */
class EstimatingTokenCounter {
    countTokens(text) {
        if (!text)
            return 0;
        // Average English token is ~4 characters
        // Code can be denser, so we use 3.5 as a safer estimate
        return Math.ceil(text.length / 3.5);
    }
    countMessageTokens(messages) {
        let chars = 0;
        for (const msg of messages) {
            chars += msg.role.length;
            if (msg.content) {
                if (typeof msg.content === 'string') {
                    chars += msg.content.length;
                }
                else if (Array.isArray(msg.content)) {
                    for (const part of msg.content) {
                        if (typeof part === 'object' && part !== null && 'text' in part) {
                            chars += part.text.length;
                        }
                    }
                }
            }
            if (msg.tool_calls)
                chars += JSON.stringify(msg.tool_calls).length;
        }
        return this.countTokens('a'.repeat(chars)); // Dummy string for calc
    }
    estimateStreamingTokens(chunk) {
        return this.countTokens(chunk);
    }
    dispose() { }
}
// ============================================================================
// Singleton and Helper Functions
// ============================================================================
/** Cache of token counters by model */
const counterCache = new Map();
/**
 * Get or create a cached token counter for a model
 */
function getCounter(model = 'gpt-4') {
    let counter = counterCache.get(model);
    if (!counter) {
        counter = createTokenCounter(model);
        counterCache.set(model, counter);
    }
    return counter;
}
/**
 * Count tokens in a text string
 * @param text - The text to count tokens in
 * @param model - The model to use for token counting (default: gpt-4)
 */
export function countTokens(text, model = 'gpt-4') {
    return getCounter(model).countTokens(text);
}
/**
 * Count tokens in a single message
 * @param message - The message to count tokens in
 * @param model - The model to use for token counting (default: gpt-4)
 */
export function countMessageTokens(message, model = 'gpt-4') {
    return getCounter(model).countMessageTokens([message]);
}
/**
 * Count tokens in multiple messages
 * @param messages - The messages to count tokens in
 * @param model - The model to use for token counting (default: gpt-4)
 */
export function countMessagesTokens(messages, model = 'gpt-4') {
    return getCounter(model).countMessageTokens(messages);
}
//# sourceMappingURL=token-counter.js.map