import { Conversation, ITokeniser } from './type';
import { default as EE } from 'eventemitter3';
export declare const SPECIALS: string[];
export default abstract class BaseTokeniser extends EE<'trainStatus'> implements ITokeniser {
    id: string;
    datasetID?: string;
    protected specialTokens: Map<string, number>;
    protected specialTokenSet: Set<number>;
    abstract vocabSize: number;
    abstract eosToken: number;
    abstract bosToken: number;
    abstract trained: boolean;
    abstract addToken(token: string, index?: number): number;
    isSpecialToken(index: number): boolean;
    protected addSpecialTokens(): void;
    protected addSpecialToken(token: string, index: number): void;
    generateID(): void;
    abstract train(text: Conversation[][], cb?: (vocab: number) => void, datasetID?: string): Promise<number>;
    abstract getVocab(): string[];
    abstract getMerges(): [string, string][];
    abstract destroy(): void;
    abstract encode(text: string): number[];
    encodeSequence(text: string): number[];
    encodeAsSequence(conversation: Conversation[], completion?: boolean): number[];
    encodeConversation(conversation: Conversation[], completion?: boolean): number[];
    abstract decode(tokens: number[]): string;
    decodeConversation(tokens: number[] | Uint16Array): Conversation[];
    getSpecialTokenIndex(token: string): number | undefined;
}
