import { getEncoding } from 'js-tiktoken';
import { Token } from '../types';
import { TokenizerStrategy } from '../tokenizer.interface';

export class TiktokenTokenizer implements TokenizerStrategy {
  // Use cl100k_base (GPT-4) as the standard encoding
  private enc = getEncoding('cl100k_base');

  tokenize(content: string): Token[] {
    const tokens: Token[] = [];
    const encoded = this.enc.encode(content);
    
    let currentIndex = 0;
    
    // Iterate through token IDs, decode them individually to get text and length.
    // This allows us to reconstruct the offsets (startIndex/endIndex).
    for (const tokenId of encoded) {
        // decoding a single token is the only way to get its exact text representation
        // to map back to the source string indices.
        const text = this.enc.decode([tokenId]);
        const length = text.length;
        
        tokens.push({
            text,
            type: 'bpe',
            startIndex: currentIndex,
            endIndex: currentIndex + length,
            // startPosition is not calculated for Tiktoken strategy as it's computationally expensive 
            // and not strictly required for the patching algorithm which relies on text matching.
        });
        currentIndex += length;
    }
    return tokens;
  }
}