import { default as BaseTokeniser } from './BaseTokeniser';
import { Conversation } from './type';
export default class CharTokeniser extends BaseTokeniser {
    vocabSize: number;
    eosToken: number;
    bosToken: number;
    unkToken: number;
    vocab: string[];
    private cache;
    private _trained;
    constructor(vocabSizeOrVocab: number | string[]);
    addToken(token: string, index?: number): number;
    get trained(): boolean;
    destroy(): void;
    train(text: Conversation[][], cb?: (vocab: number) => void, datasetID?: string): Promise<number>;
    tokenise(text: string[], numeric: true): number[][];
    tokenise(text: string[]): string[][];
    detokenise(tokens: (number[] | Uint16Array)[]): string[];
    encode(text: string): number[];
    decode(tokens: number[] | Uint16Array): string;
    getVocab(): string[];
    getMerges(): [string, string][];
    createTrainingData(text: string[], windowSize?: number): Promise<[number[], number[]]>;
}
