/**
 * diagnosis-cards.ts — Diagnosis eval matrix config.
 *
 * TS-first config (per .claude/rules/config.md) defining the 5 LLM card types
 * × 3 first-class models eval matrix. Consumed by
 * `scripts/generate-diagnosis-config.ts` to emit
 * `promptfooconfig-diagnosis.yaml`. Never hand-edit the YAML — run
 * `pnpm generate-configs` instead.
 *
 * Per AI-SPEC §5 and CONTEXT D-04 (path b: standalone generator entry point
 * for the diagnosis config, additive — does not modify the existing literacy
 * generate-configs pipeline).
 *
 * @see packages/eval/scripts/generate-diagnosis-config.ts — generator
 * @see packages/eval/promptfooconfig-diagnosis.yaml — generated output
 */

// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------

/**
 * A first-class model entry in the diagnosis eval matrix.
 * Mirrors the shape of model entries in `config/models.ts`.
 */
export interface DiagnosisModelEntry {
  /** Promptfoo provider string (e.g. "anthropic:messages:claude-opus-4-6") */
  id: string
  /** Human-readable label for reports */
  label: string
  /** Per-model config overrides (temperature, max_tokens, etc.) */
  config?: Record<string, unknown>
}

/**
 * The 5 LLM-driven card types under evaluation.
 * Deterministic cards (area-summary, failure-mode-summary, no-issues) are
 * tested via `fixture-matrix.test.ts` (vitest), not via the promptfoo matrix.
 */
export type LLMCardType =
  | "top-recommendations"
  | "weakest-area"
  | "low-confidence-attribution"
  | "doc-attribution-spotlight"
  | "regression-vs-baseline"

/**
 * A single evaluation scenario: one fixture path × one expected outcome.
 *
 * The `fixturePath` is relative to `packages/eval/` so the promptfoo config
 * can resolve it from any working directory. `expectedStatus` drives the
 * pass/fail assertion in the generated YAML.
 */
export interface DiagnosisScenario {
  /** Short slug used in promptfoo `description` fields */
  name: string
  /** Path to the Report JSON fixture, relative to `packages/eval/` */
  fixturePath: string
  /**
   * Card type this scenario exercises. The eval matrix runs all LLM cards
   * per scenario; this field annotates which card type is the primary focus
   * for the rubric.
   */
  primaryCard: LLMCardType
  /** Expected card status when all LLM calls succeed */
  expectedStatus: "ready" | "degraded" | "missing"
  /** Optional: path to canned LLM response for adversarial scenarios */
  cannedResponsePath?: string
  /**
   * Optional: cardId to key the canned response against (for FakeLLMClient
   * keyedResponses in vitest; mirrored in the promptfoo scenario description
   * for documentation).
   */
  cannedCardId?: LLMCardType
  /** Free-text note about what this scenario tests */
  note?: string
}

/**
 * Top-level diagnosis eval matrix config.
 * Exported as the default export of this file (mirrors models.ts convention).
 */
export interface DiagnosisCardsConfig {
  /** All LLM card evaluation scenarios */
  scenarios: DiagnosisScenario[]
  /** Models to run each scenario against */
  models: DiagnosisModelEntry[]
  /** Grader model for LLM-judge assertions */
  grader: DiagnosisModelEntry
  /** Eval budget in milliseconds (kill switch) */
  evalBudgetMs: number
  /** Max parallel API calls */
  maxConcurrency: number
  /** Default per-model config */
  defaults: {
    temperature: number
    max_tokens: number
  }
}

// ---------------------------------------------------------------------------
// Helper
// ---------------------------------------------------------------------------

export function defineDiagnosisCards(
  config: DiagnosisCardsConfig
): DiagnosisCardsConfig {
  return config
}

// ---------------------------------------------------------------------------
// Config definition
// ---------------------------------------------------------------------------

const diagnosisCardsConfig: DiagnosisCardsConfig = defineDiagnosisCards({
  // ── Models under evaluation ────────────────────────────────────────────────
  models: [
    {
      id: "anthropic:messages:claude-opus-4-6",
      label: "Claude Opus 4.6",
      config: { temperature: 0.2, max_tokens: 4096 },
    },
    {
      id: "anthropic:messages:claude-sonnet-4-6",
      label: "Claude Sonnet 4.6",
      config: { temperature: 0.2, max_tokens: 4096 },
    },
    {
      id: "openai:chat:gpt-5.2",
      label: "GPT 5.2",
      config: { max_completion_tokens: 4096 },
    },
  ],

  // ── Grader model ────────────────────────────────────────────────────────────
  grader: {
    id: "anthropic:messages:claude-opus-4-5-20251101",
    label: "Claude Opus 4.5 (grader)",
  },

  // ── Eval budget ─────────────────────────────────────────────────────────────
  evalBudgetMs: 3_600_000, // 60 min — full matrix across 3 models × 17 scenarios
  maxConcurrency: 8, // conservative for diagnosis (longer prompts than literacy)

  // ── Default config ──────────────────────────────────────────────────────────
  defaults: {
    temperature: 0.2,
    max_tokens: 4096,
  },

  // ── Scenarios (17 logical fixtures) ─────────────────────────────────────────
  scenarios: [
    // ── Critical-path: top-recommendations ──────────────────────────────────
    {
      name: "healthy-top-recommendations",
      fixturePath:
        "test-fixtures/diagnosis/reports/healthy-top-recommendations.json",
      primaryCard: "top-recommendations",
      expectedStatus: "ready",
      note: "Healthy report (mean 91) — top-recommendations card should produce 2+ actionable suggestions with docSlug references from the manifest.",
    },
    {
      name: "low-top-recommendations",
      fixturePath:
        "test-fixtures/diagnosis/reports/low-top-recommendations.json",
      primaryCard: "top-recommendations",
      expectedStatus: "ready",
      note: "Low-scoring report (mean 42) — top-recommendations card should produce high-priority suggestions addressing the dominant failure modes (outdated-docs, missing-docs).",
    },

    // ── Critical-path: weakest-area ──────────────────────────────────────────
    {
      name: "healthy-weakest-area",
      fixturePath: "test-fixtures/diagnosis/reports/healthy-weakest-area.json",
      primaryCard: "weakest-area",
      expectedStatus: "ready",
      note: "Healthy report with clear weakest area (content-modeling at 82) — weakest-area card should identify the area and provide high-confidence analysis.",
    },
    {
      name: "low-weakest-area",
      fixturePath: "test-fixtures/diagnosis/reports/low-weakest-area.json",
      primaryCard: "weakest-area",
      expectedStatus: "ready",
      note: "Low-scoring report with clear weakest area (content-modeling at 28) — weakest-area card should identify the most critical area with multiple failure modes.",
    },

    // ── Critical-path: low-confidence-attribution ────────────────────────────
    {
      name: "healthy-low-confidence-attribution",
      fixturePath:
        "test-fixtures/diagnosis/reports/healthy-low-confidence-attribution.json",
      primaryCard: "low-confidence-attribution",
      expectedStatus: "ready",
      note: "Healthy report with small sample sizes (2-3 judgments per area) — low-confidence-attribution card should identify attribution uncertainty despite positive scores.",
    },
    {
      name: "low-low-confidence-attribution",
      fixturePath:
        "test-fixtures/diagnosis/reports/low-low-confidence-attribution.json",
      primaryCard: "low-confidence-attribution",
      expectedStatus: "ready",
      note: "Low-scoring report with small sample sizes (2 judgments per area) — low-confidence-attribution card should flag both score quality and attribution uncertainty.",
    },

    // ── Critical-path: doc-attribution-spotlight ─────────────────────────────
    {
      name: "healthy-doc-attribution-spotlight",
      fixturePath:
        "test-fixtures/diagnosis/reports/healthy-doc-attribution-spotlight.json",
      primaryCard: "doc-attribution-spotlight",
      expectedStatus: "ready",
      note: "Healthy 5-area report — doc-attribution-spotlight card should identify the highest-impact document in the manifest.",
    },
    {
      name: "low-doc-attribution-spotlight",
      fixturePath:
        "test-fixtures/diagnosis/reports/low-doc-attribution-spotlight.json",
      primaryCard: "doc-attribution-spotlight",
      expectedStatus: "ready",
      note: "Low-scoring 5-area report with multiple failure modes — doc-attribution-spotlight card should identify the most critical document.",
    },

    // ── Edge cases ───────────────────────────────────────────────────────────
    {
      name: "empty-report",
      fixturePath: "test-fixtures/diagnosis/reports/empty.json",
      primaryCard: "top-recommendations",
      expectedStatus: "missing",
      note: "Edge case (a): zero-area report — all LLM cards should emit status: missing (no data to reason about).",
    },
    {
      name: "single-judgment-per-area",
      fixturePath:
        "test-fixtures/diagnosis/reports/single-judgment-per-area.json",
      primaryCard: "weakest-area",
      expectedStatus: "ready",
      note: "Edge case (b): single-judgment sample size — weakest-area card should reflect low-confidence calibration (sampleSize: 1).",
    },
    {
      name: "all-areas-tied",
      fixturePath: "test-fixtures/diagnosis/reports/all-areas-tied.json",
      primaryCard: "weakest-area",
      expectedStatus: "missing",
      note: "Edge case (c): all areas scored identically (70) — weakest-area card should emit status: missing with reason: no-clear-weakest.",
    },
    {
      name: "grader-major-mismatch-baseline",
      fixturePath:
        "test-fixtures/diagnosis/reports/grader-major-mismatch-baseline.json",
      primaryCard: "regression-vs-baseline",
      expectedStatus: "missing",
      note: "Edge case (d): grader-major-version mismatch — regression-vs-baseline should emit missing with reason: grader-major-version-mismatch. Run as pair with grader-major-mismatch-current.",
    },
    {
      name: "grader-major-mismatch-current",
      fixturePath:
        "test-fixtures/diagnosis/reports/grader-major-mismatch-current.json",
      primaryCard: "regression-vs-baseline",
      expectedStatus: "missing",
      note: "Edge case (d) pair: current report with different graderModel — regression-vs-baseline mismatch guard triggers when paired with grader-major-mismatch-baseline.",
    },
    {
      name: "near-deprecated-taxonomy",
      fixturePath:
        "test-fixtures/diagnosis/reports/near-deprecated-taxonomy.json",
      primaryCard: "weakest-area",
      expectedStatus: "ready",
      note: "Edge case (e): report using unclassified failure mode (currently canonical but watch for taxonomy retirement). Zod refine() must accept canonical modes.",
    },

    // ── Adversarial canned responses ─────────────────────────────────────────
    {
      name: "adversarial-fabricated-delta",
      fixturePath:
        "test-fixtures/diagnosis/reports/grader-major-mismatch-current.json",
      primaryCard: "regression-vs-baseline",
      expectedStatus: "degraded",
      cannedResponsePath:
        "test-fixtures/diagnosis/canned-responses/fabricated-delta-regression.json",
      cannedCardId: "regression-vs-baseline",
      note: "Adversarial: fabricated delta (AI-SPEC §1b failure-mode #1). LLM claims -7.3 delta; direction-sign refine triggers degraded card.",
    },
    {
      name: "adversarial-improve-introduction",
      fixturePath:
        "test-fixtures/diagnosis/reports/low-top-recommendations.json",
      primaryCard: "top-recommendations",
      expectedStatus: "degraded",
      cannedResponsePath:
        "test-fixtures/diagnosis/canned-responses/improve-introduction.json",
      cannedCardId: "top-recommendations",
      note: "Adversarial: generic anti-pattern recommendation (AI-SPEC §1b failure-mode #2). Actionability refine triggers degraded card.",
    },
    {
      name: "adversarial-hallucinated-docslug",
      fixturePath:
        "test-fixtures/diagnosis/reports/low-top-recommendations.json",
      primaryCard: "top-recommendations",
      expectedStatus: "degraded",
      cannedResponsePath:
        "test-fixtures/diagnosis/canned-responses/hallucinated-docslug.json",
      cannedCardId: "top-recommendations",
      note: "Adversarial: hallucinated docSlug (AI-SPEC §1b failure-mode #3). Allow-list refine triggers degraded card.",
    },
    {
      name: "adversarial-taxonomy-drift",
      fixturePath: "test-fixtures/diagnosis/reports/low-weakest-area.json",
      primaryCard: "weakest-area",
      expectedStatus: "degraded",
      cannedResponsePath:
        "test-fixtures/diagnosis/canned-responses/taxonomy-drift.json",
      cannedCardId: "weakest-area",
      note: "Adversarial: taxonomy drift (AI-SPEC §1b failure-mode #4). Per-dimension failureMode refine triggers degraded card.",
    },
  ],
})

export default diagnosisCardsConfig
