import { Token, Tokenizer } from "../types.js";
export type LlamaTextClass = {
    <const V extends LlamaTextValue = LlamaTextValue, const V2 extends LlamaTextValue = LlamaTextValue>(strings: TemplateStringsArray | V | LlamaText<V> | readonly (LlamaText<V> | V)[], ...values: readonly (V | LlamaText<V> | V2 | LlamaText<V2> | number | boolean | readonly (LlamaText<V> | V | LlamaText<V2> | V2)[])[]): LlamaText<V | V2>;
    fromJSON(json: LlamaTextJSON): LlamaText;
};
export type LlamaText<T extends LlamaTextValue = LlamaTextValue> = {
    <const V extends LlamaTextValue = LlamaTextValue, const V2 extends LlamaTextValue = LlamaTextValue>(strings: TemplateStringsArray | V | LlamaText | readonly (LlamaText | V)[], ...values: readonly (V | V2 | LlamaText | number | boolean | readonly (LlamaText | V | V2)[])[]): LlamaText<T | V>;
    readonly type: "LlamaText";
    readonly values: readonly T[];
    mapValues<V extends LlamaTextValue = LlamaTextValue>(mapper: (value: T) => V): LlamaText<V>;
    joinValues<V extends LlamaTextValue = LlamaTextValue>(separator: LlamaText<V> | V): LlamaText<T | V>;
    toString(): string;
    toJSON(): LlamaTextJSON;
    tokenize(tokenizer: Tokenizer): Token[];
};
export type LlamaTextValue = string | SpecialToken;
export type LlamaTextJSON = Array<LlamaTextJSONValue>;
export type LlamaTextJSONValue = string | LlamaTextSpecialTokenJSON;
export type LlamaTextSpecialTokenJSON = {
    type: "specialToken";
    value: string;
    builtin?: true;
};
export declare const LlamaText: LlamaTextClass;
export declare class SpecialToken {
    readonly value: string;
    constructor(value: string);
    toString(): string;
    tokenize(tokenizer: Tokenizer): Token[];
    toJSON(): LlamaTextSpecialTokenJSON;
    static fromJSON(json: LlamaTextSpecialTokenJSON): SpecialToken;
    static isSpecialTokenJSON(value: LlamaTextJSONValue): value is LlamaTextSpecialTokenJSON;
}
export type BuiltinSpecialTokenValue = "BOS" | "EOS" | "NL";
export declare class BuiltinSpecialToken extends SpecialToken {
    readonly value: BuiltinSpecialTokenValue;
    constructor(value: BuiltinSpecialTokenValue);
    tokenize(tokenizer: Tokenizer): Token[];
    toJSON(): LlamaTextSpecialTokenJSON;
}
export declare function isLlamaText(value: unknown): value is LlamaText;
export declare function tokenizeText(text: string | LlamaText, tokenizer: Tokenizer): Token[];
