#!/usr/bin/env bun
/**
 * Eval runner CLI. Parses flags into ZIDANE_EVAL_* env vars and spawns the
 * real-inference test file. Keeps the flag surface stable regardless of how
 * `bun test` treats unknown argv.
 *
 * Usage:
 *   bun run test:eval --model claude-opus-4-8 --scorer claude-opus-4-8
 *   bun run test:eval --provider openai --model gpt-5 --filter vite --repeat 3
 *   bun run test:eval --provider all                       # every provider with an API key in env/.env
 *   bun run test:eval --provider baseten --models zai-org/GLM-5.1,moonshotai/Kimi-K2.6
 *
 * Flags:
 *   --provider <name>        agent provider (anthropic|openai|openrouter|cerebras|arcee|baseten|local).
 *                            `*` or `all` runs every provider with a configured API key, on default models.
 *   --model <id>             agent model override
 *   --models <id1>,<id2>     run every eval on each listed model (requires a specific --provider)
 *   --judge-provider <name>  judge provider (defaults to agent provider)
 *   --judge-model | --scorer judge model override
 *   --filter | -g <text>     run only evals whose suite/id/tag contains <text>
 *   --repeat <n>             repeat each eval n times
 *   --concurrency | --jobs <n>  max eval cases to run in parallel (default 4)
 *   --report-dir <dir>       write per-case + run-summary JSON here
 *   --artifact-dir <dir>     write per-case result/events/transcript/workspace here
 *   --timeout <ms>           per-test timeout (default 180000)
 */

import { existsSync } from 'node:fs'
import { resolve } from 'node:path'

const DEFAULT_TIMEOUT_MS = '180000'
const EVAL_TEST_FILE = 'test/eval-real.ts'

const FLAG_TO_ENV: Record<string, string> = {
  'provider': 'ZIDANE_EVAL_PROVIDER',
  'model': 'ZIDANE_EVAL_MODEL',
  'models': 'ZIDANE_EVAL_MODELS',
  'judge-provider': 'ZIDANE_EVAL_JUDGE_PROVIDER',
  'judge-model': 'ZIDANE_EVAL_JUDGE_MODEL',
  'scorer': 'ZIDANE_EVAL_JUDGE_MODEL',
  'filter': 'ZIDANE_EVAL_FILTER',
  'g': 'ZIDANE_EVAL_FILTER',
  'repeat': 'ZIDANE_EVAL_REPEAT',
  'concurrency': 'ZIDANE_EVAL_CONCURRENCY',
  'jobs': 'ZIDANE_EVAL_CONCURRENCY',
  'report-dir': 'ZIDANE_EVAL_REPORT_DIR',
  'artifact-dir': 'ZIDANE_EVAL_ARTIFACT_DIR',
}

const NUMERIC_FLAGS = new Set(['repeat', 'concurrency', 'jobs', 'timeout'])

export interface EvalCliParseOptions {
  cwd?: string
  now?: Date
  existingEnv?: Record<string, string | undefined>
}

export interface EvalCliParseResult {
  env: Record<string, string>
  timeout: string
  testFile: string
}

export class EvalCliError extends Error {
  constructor(message: string, readonly exitCode = 2) {
    super(message)
    this.name = 'EvalCliError'
  }
}

export function parseEvalCliArgs(argv: string[], options: EvalCliParseOptions = {}): EvalCliParseResult {
  const env: Record<string, string> = {}
  let timeout = DEFAULT_TIMEOUT_MS
  for (let i = 0; i < argv.length; i++) {
    const token = argv[i]
    if (!token.startsWith('-'))
      continue
    const key = token.replace(/^-+/, '')
    const eq = key.indexOf('=')
    const rawKey = eq === -1 ? key : key.slice(0, eq)
    const inlineVal = eq === -1 ? undefined : key.slice(eq + 1)
    const value = inlineVal ?? (argv[i + 1] && !argv[i + 1].startsWith('-') ? argv[++i] : undefined)
    if (!value)
      throw new EvalCliError(`Missing value for eval flag: --${rawKey}`)
    if (NUMERIC_FLAGS.has(rawKey))
      assertPositiveInteger(rawKey, value)
    if (rawKey === 'timeout') {
      timeout = value
      continue
    }
    const envKey = FLAG_TO_ENV[rawKey]
    if (!envKey)
      throw new EvalCliError(`Unknown eval flag: --${rawKey}`)
    env[envKey] = value
  }

  // Cross-flag validation. `_config.ts` re-checks at run time (env vars can
  // come from elsewhere), but failing here gives a clean exit-2 before any
  // test process spawns.
  const provider = env.ZIDANE_EVAL_PROVIDER
  const wildcard = provider === '*' || provider === 'all'
  if (env.ZIDANE_EVAL_MODELS && env.ZIDANE_EVAL_MODEL)
    throw new EvalCliError('Use either --model or --models, not both.')
  if (env.ZIDANE_EVAL_MODELS && (!provider || wildcard))
    throw new EvalCliError('--models requires a specific --provider.')
  if (wildcard && env.ZIDANE_EVAL_MODEL)
    throw new EvalCliError('--model cannot be combined with --provider=* — each detected provider runs its default model.')

  const cwd = options.cwd ?? process.cwd()
  const existingEnv = options.existingEnv ?? process.env
  // Default a run-scoped artifact dir so the summary can render clickable links
  // to each case's output. Overridable via --artifact-dir / ZIDANE_EVAL_ARTIFACT_DIR.
  if (!env.ZIDANE_EVAL_ARTIFACT_DIR && !existingEnv.ZIDANE_EVAL_ARTIFACT_DIR) {
    const stamp = (options.now ?? new Date()).toISOString().replace(/[:.]/g, '-')
    env.ZIDANE_EVAL_ARTIFACT_DIR = `${cwd}/.zidane/evals/${stamp}`
  }

  return { env, timeout, testFile: resolve(cwd, EVAL_TEST_FILE) }
}

export async function runEvalCli(argv: string[]): Promise<number> {
  let parsed: EvalCliParseResult
  try {
    parsed = parseEvalCliArgs(argv)
  }
  catch (err) {
    if (err instanceof EvalCliError) {
      console.error(err.message)
      return err.exitCode
    }
    throw err
  }

  if (!existsSync(parsed.testFile)) {
    console.error('`bun run test:eval` is only available from a zidane source checkout.')
    console.error(`Missing ${parsed.testFile}.`)
    return 1
  }

  const proc = Bun.spawn(['bun', 'test', `--timeout=${parsed.timeout}`, parsed.testFile], {
    env: { ...process.env, ...parsed.env },
    stdout: 'inherit',
    stderr: 'inherit',
    stdin: 'inherit',
  })

  return await proc.exited
}

function assertPositiveInteger(flag: string, value: string): void {
  const parsed = Number(value)
  if (!Number.isInteger(parsed) || parsed <= 0)
    throw new EvalCliError(`Eval flag --${flag} must be a positive integer, got ${JSON.stringify(value)}`)
}

if (import.meta.main)
  process.exit(await runEvalCli(process.argv.slice(2)))
