/**
 * Generic tokenizer for models without a provider-specific encoder.
 *
 * Uses tiktoken's `o200k_base` BPE, which tracks arbitrary modern models — and
 * especially code/JSON density — far more closely than a flat chars-per-token
 * ratio. Degrades to the char/4 FallbackTokenizer when tiktoken can't load
 * (constrained runtimes, WASM unavailable) or an encode throws, so this
 * tokenizer can never itself fail and stays a safe default for any provider.
 */
import type { Message } from '../../types/core.js';
import type { Tokenizer } from '../../types/tokenization.js';
export declare class GenericTokenizer implements Tokenizer {
    private encoding;
    private readonly fallback;
    constructor();
    encode(text: string): number;
    countTokens(message: Message): number;
    getName(): string;
    free(): void;
}
//# sourceMappingURL=generic-tokenizer.d.ts.map