/**
 * Blended 4-chars-per-token heuristic with CJK / emoji adjustment.
 *
 * Rationale: Anthropic and OpenAI both document ~4 chars/token for
 * English / Latin / punctuation. CJK characters, Hangul, kana, fullwidth
 * forms, and emoji tokenize roughly 1:1 in practice. We use a simple
 * two-bucket counter rather than a full BPE tokenizer to keep the
 * estimate cheap and synchronous — the result is shown with a leading
 * `~` so the user is never misled about precision.
 */

/** A single Unicode code-point character from `text`. */
function forEachCodePoint(text: string, visit: (ch: string) => void): void {
  // `Array.from` iterates by code point, so surrogate pairs and
  // combining marks stay together.
  for (const ch of Array.from(text)) {
    visit(ch);
  }
}

const CJK_REGEX = /[\u3000-\u9FFF\uAC00-\uD7FF\uF900-\uFAFF\uFF00-\uFFEF\u{1F300}-\u{1FAFF}]/u;

/**
 * Estimate the token count of a piece of text.
 *
 * - CJK / Hangul / kana / fullwidth / emoji → 1 token per character
 * - Everything else → 1 token per 4 characters, rounded up
 *
 * Returns 0 for the empty string.
 */
export function estimateTokens(text: string): number {
  if (text.length === 0) return 0;
  let cjk = 0;
  let other = 0;
  forEachCodePoint(text, (ch) => {
    if (CJK_REGEX.test(ch)) {
      cjk += 1;
    } else {
      other += 1;
    }
  });
  return Math.ceil(cjk * 1) + Math.ceil(other / 4);
}

/** Count Unicode code points in a string (graphemes ≈ characters for our purposes). */
export function countCharacters(text: string): number {
  if (text.length === 0) return 0;
  let n = 0;
  forEachCodePoint(text, () => {
    n += 1;
  });
  return n;
}
