/**
 * models.ts — Central model registry for AILF evaluations.
 *
 * Define all models to test here. Each model declares which evaluation
 * modes it participates in (e.g., "literacy", "mcp-server") and
 * optionally which variants within those modes.
 *
 * When a model enrolls in a mode without specifying variants, all
 * variants defined by the mode base are included by default.
 *
 * @see docs/archive/exec-plans/architecture-overhaul/phase-1-ts-config-loading.md
 */

import { defineModels } from "@sanity/ailf-core"
import { modelDisplayName } from "@sanity/ailf-shared"

export default defineModels({
  models: [
    // ── Anthropic ──────────────────────────────────────────────
    {
      id: "anthropic:messages:claude-opus-4-6",
      label: modelDisplayName("anthropic:messages:claude-opus-4-6"),
      config: { temperature: 0.2, max_tokens: 4096 },
      modes: ["literacy", "mcp-server", "knowledge-probe"],
      // All literacy variants included by default (baseline, observed,
      // agentic-naive, agentic-optimized)
    },
    {
      // Phase 5 LLM card routing (D-07). AI-SPEC §4 routes 3 routine cards
      // (top-recommendations, weakest-area, regression-vs-baseline) here.
      // Pricing already in AnthropicLLMClient; baseline literacy variant only.
      id: "anthropic:messages:claude-sonnet-4-6",
      label: modelDisplayName("anthropic:messages:claude-sonnet-4-6"),
      config: { temperature: 0.2, max_tokens: 4096 },
      modes: ["literacy"],
      variants: {
        literacy: ["baseline"],
      },
    },

    // ── Google ─────────────────────────────────────────────────
    // {
    //   id: "google:gemini-2.5-pro",
    //   label: "Gemini 2.5 Pro",
    //   config: { temperature: 0.2, max_tokens: 4096 },
    //   modes: ["literacy"],
    // },

    // ── OpenAI ─────────────────────────────────────────────────
    {
      // gpt-5.2 routes through chat completions (and through the in-house
      // agentic provider for naive/optimized variants). `verbosity` is a
      // Responses-API-only field — it would be silently dropped here, so
      // it isn't configured. See W0131.
      id: "openai:chat:gpt-5.2",
      label: modelDisplayName("openai:chat:gpt-5.2"),
      config: {
        max_completion_tokens: 8192,
      },
      modes: ["literacy", "knowledge-probe"],
      // All literacy variants included by default
    },
    {
      // GPT 5.4 evaluated only on the baseline literacy variant. Promptfoo's
      // native handling of `openai:responses:` honors reasoning / verbosity /
      // summary; the in-house agentic provider does not (W0131). MCP-server
      // and knowledge-probe routes go through Promptfoo native too.
      id: "openai:responses:gpt-5.4",
      label: modelDisplayName("openai:responses:gpt-5.4"),
      config: {
        reasoning: { effort: "medium", summary: "auto" },
        verbosity: "medium",
        max_output_tokens: 32_000, // reasoning tokens share this budget — 4096 was too low
        maxRetries: 1,
      },
      timeoutMs: 600_000, // 10 min — reasoning model needs more headroom
      modes: ["literacy", "mcp-server", "knowledge-probe"],
      variants: {
        literacy: ["baseline"],
      },
    },

    // ── Disabled models (uncomment to enable) ──────────────────
    // { id: "anthropic:claude-sonnet-4-20250514", label: "Claude Sonnet 4",
    //   config: { temperature: 0.2, max_tokens: 4096 },
    //   modes: ["literacy"],
    //   variants: { literacy: ["baseline"] } },
    // { id: "anthropic:claude-3.5-sonnet-20241022", label: "Claude 3.5 Sonnet",
    //   config: { temperature: 0.2, max_tokens: 4096 },
    //   modes: ["literacy"],
    //   variants: { literacy: ["baseline", "agentic-naive", "agentic-optimized"] } },
    // { id: "google:gemini-2.0-flash", label: "Gemini 2.0 Flash",
    //   config: { temperature: 0.2, max_tokens: 4096 },
    //   modes: ["literacy"],
    //   variants: { literacy: ["baseline"] } },
    // { id: "openrouter:deepseek/deepseek-r1", label: "DeepSeek R1",
    //   config: { temperature: 0.2, max_tokens: 4096 },
    //   modes: ["literacy"],
    //   variants: { literacy: ["baseline"] } },
  ],

  // ── Grading Model ──────────────────────────────────────────
  // Which model scores the responses. Separate from the models being tested.
  grader: {
    // Role context ("grader") lives at the display site, not in the label —
    // the same model reads identically wherever it appears (D-model-registry).
    id: "anthropic:messages:claude-opus-4-5-20251101",
    label: modelDisplayName("anthropic:messages:claude-opus-4-5-20251101"),
  },

  // ── Evaluation Options ─────────────────────────────────────
  evalBudgetMs: 1_200_000, // 20 min per eval mode — outer kill switch
  maxConcurrency: 32, // max parallel API calls — benchmarked in DOC-1896

  // ── Default Config ─────────────────────────────────────────
  // Applied to all models unless overridden per-model.
  defaults: {
    temperature: 0.2,
    max_tokens: 4096,
    // Global default round budget for agentic modes. Per-mode overrides
    // below give naive more headroom (W0134) since it spends rounds on
    // retries when fetches fail. Per-model `config.maxToolRounds` still
    // wins over both values.
    maxToolRounds: 5,
    modeMaxToolRounds: {
      "agentic-naive": 8,
      "agentic-optimized": 5,
    },
    observerOptions: {
      // Per-class preview caps (W0133): default 4 KB, but search responses
      // get 16 KB and llms.txt gets 128 KB so trace audits can resolve
      // which result the model actually saw.
      maxPreviewBytes: 4096,
      previewLimits: {
        default: 4096,
        llmsTxt: 131072,
        search: 16384,
      },
      captureResponsePreview: true,
      includePatterns: ["sanity.io", "sanity.dev", "cdn.sanity.io"],
      sensitiveHeaders: ["authorization", "cookie", "x-api-key"],
      // statusOnlyForUnmatched defaults to true (W0132) — model-side
      // traffic to api.openai.com / api.anthropic.com / googleapis.com
      // surfaces in run artifacts as slim status-only entries.
    },
  },
})
