import { default as BaseTokeniser } from './BaseTokeniser';
import { Conversation } from './type';
export default class BPETokeniser extends BaseTokeniser {
    private targetSize;
    private vocab;
    private vocabIndex;
    private merges;
    private pretokenMap;
    constructor(vocabSize: number);
    constructor(vocab: string[], merges?: [string, string][]);
    addToken(token: string, index?: number): number;
    destroy(): void;
    get trained(): boolean;
    get vocabSize(): number;
    get eosToken(): number;
    get bosToken(): number;
    get unkToken(): number;
    train(text?: Conversation[][], cb?: (vocab: number) => void, datasetID?: string): Promise<number>;
    getVocab(): string[];
    getMerges(): [string, string][];
    private tokeniseWord;
    private tokeniseStrings;
    tokenise(text: string[], numeric: true): number[][];
    tokenise(text: string[]): string[][];
    detokenise(tokens: number[][]): string[];
    encode(text: string): number[];
    decode(tokens: number[]): string;
}
