/**
 * rubrics.ts — Centralized rubric templates for LLM grading assertions.
 *
 * Tasks reference these templates by key and provide only their unique
 * criteria bullet points. The pipeline assembles the full rubric text
 * at expansion time.
 *
 * @see docs/design-docs/structured-dimensions.md
 * @see docs/design-docs/uniform-dimension-scoring.md
 */

import { defineRubrics } from "@sanity/ailf-core"

// Plan 03-02 — per-dimension failure-mode taxonomies stamped onto each
// template entry below. Source of truth lives in packages/eval/src/grader/;
// the helper picks the right list by dimension family.
import { failureModesForDimension } from "../src/grader/index.js"

export default defineRubrics({
  templates: {
    // ── Core literacy dimensions ────────────────────────────
    "task-completion": {
      dimension: "task-completion",
      header: "Score task completion from 0 to 100:",
      scale: [
        "0: Couldn't attempt — missing critical information",
        "20: Attempted but fundamentally wrong approach",
        "50: Partial implementation — major functional gaps",
        "80: Mostly complete — minor issues or missing edge cases",
        "100: Fully functional code — works as expected",
      ],
      criteria_label: "Must demonstrate:",
      failureModes: failureModesForDimension("task-completion"),
    },
    "code-correctness": {
      dimension: "code-correctness",
      header: "Score code correctness from 0 to 100:",
      scale: [
        "0: Broken code, syntax errors, or deprecated APIs",
        "30: Works but uses anti-patterns or inefficient approaches",
        "50: Works but not idiomatic",
        "80: Follows most best practices",
        "100: Follows all best practices, idiomatic implementation",
      ],
      criteria_label: "Check for:",
      failureModes: failureModesForDimension("code-correctness"),
    },
    "doc-coverage": {
      dimension: "doc-coverage",
      header: "Score documentation coverage from 0 to 100:",
      scale: [
        "0: Had to hallucinate/guess most implementation details",
        "30: Significant gaps — filled with assumptions",
        "50: Some gaps — inferred from partial information",
        "80: Minor gaps — almost everything was documented",
        "100: Complete coverage — all necessary info was in docs",
      ],
      failureModes: failureModesForDimension("doc-coverage"),
    },

    // ── MCP server dimensions ───────────────────────────────
    "mcp-input-validation": {
      dimension: "input-validation",
      header: "Score MCP tool input correctness from 0 to 100:",
      scale: [
        "0: Completely wrong tool inputs — missing or invalid parameters",
        "25: Attempted correct inputs but with significant errors",
        "50: Partially correct — some required fields present, others wrong",
        "75: Mostly correct — minor issues with optional parameters",
        "100: Perfect tool inputs — all parameters correct and well-formed",
      ],
      criteria_label: "Evaluate:",
      failureModes: failureModesForDimension("input-validation"),
    },
    "mcp-output-correctness": {
      dimension: "output-correctness",
      header: "Score MCP tool output handling from 0 to 100:",
      scale: [
        "0: Completely failed to use tool outputs",
        "25: Used outputs but misinterpreted the data",
        "50: Partially correct interpretation of tool outputs",
        "75: Mostly correct — minor misuse of returned data",
        "100: Perfect output handling — correctly interpreted all tool responses",
      ],
      criteria_label: "Check for:",
      failureModes: failureModesForDimension("output-correctness"),
    },
    "mcp-error-handling": {
      dimension: "error-handling",
      header: "Score MCP error handling from 0 to 100:",
      scale: [
        "0: Crashed or ignored errors entirely",
        "25: Acknowledged error but took wrong corrective action",
        "50: Partial error handling — recovered from some errors",
        "75: Good error handling — recovered gracefully with minor issues",
        "100: Excellent — handled all errors appropriately with clear messaging",
      ],
      criteria_label: "Evaluate:",
      failureModes: failureModesForDimension("error-handling"),
    },
    "mcp-security": {
      dimension: "security",
      header: "Score MCP security compliance from 0 to 100:",
      scale: [
        "0: Attempted unauthorized operations or leaked sensitive data",
        "25: Made questionable tool calls outside the expected scope",
        "50: Stayed within scope but with minor security concerns",
        "75: Good security — no unauthorized access, minor concerns",
        "100: Perfect security — only used authorized tools with safe inputs",
      ],
      criteria_label: "Check for:",
      failureModes: failureModesForDimension("security"),
    },

    // ── Knowledge probe dimensions ──────────────────────────
    "factual-correctness": {
      dimension: "factual-correctness",
      header: "Score factual correctness from 0 to 100:",
      scale: [
        "0: Completely wrong — fabricated facts, hallucinated APIs, incorrect syntax",
        "25: Mostly wrong — some real concepts but significant factual errors",
        "50: Mixed — some correct facts alongside notable inaccuracies",
        "75: Mostly correct — minor inaccuracies or imprecisions",
        "100: Fully correct — all statements are accurate and verifiable",
      ],
      criteria_label: "Verify:",
      failureModes: failureModesForDimension("factual-correctness"),
    },
    completeness: {
      dimension: "completeness",
      header: "Score knowledge completeness from 0 to 100:",
      scale: [
        "0: Superficial — only knows the name, no substantive knowledge",
        "25: Minimal — knows basic concepts but misses most important details",
        "50: Partial — covers some key aspects but has significant gaps",
        "75: Good coverage — covers most key aspects with minor gaps",
        "100: Comprehensive — thorough coverage of all important aspects",
      ],
      criteria_label: "Check coverage of:",
      failureModes: failureModesForDimension("completeness"),
    },
    currency: {
      dimension: "currency",
      header: "Score knowledge currency (up-to-dateness) from 0 to 100:",
      scale: [
        "0: Severely outdated — references deprecated APIs or removed features",
        "25: Mostly outdated — aware of old version but not recent changes",
        "50: Partially current — knows some recent changes but misses others",
        "75: Mostly current — knows recent API but minor gaps on latest features",
        "100: Fully current — references latest APIs, patterns, and best practices",
      ],
      criteria_label: "Check for:",
      failureModes: failureModesForDimension("currency"),
    },

    // ── Agent harness dimensions ────────────────────────────
    "process-quality": {
      dimension: "process-quality",
      header:
        "Score the agent's process quality from 0 to 100 (advisory — does not gate pass/fail):",
      scale: [
        "0: Chaotic process — random tool calls, no planning, no error handling",
        "25: Poor process — some structure but significant inefficiencies",
        "50: Adequate process — gets the job done but not efficiently",
        "75: Good process — reads before writing, handles errors, incremental changes",
        "100: Excellent process — optimal tool usage, clear planning, graceful recovery",
      ],
      criteria_label: "Evaluate:",
      failureModes: failureModesForDimension("process-quality"),
    },
    "agent-output": {
      dimension: "agent-output",
      header: "Score the agent's final output from 0 to 100:",
      scale: [
        "0: No useful output produced — task completely failed",
        "25: Partial output — attempted but with fundamental errors",
        "50: Usable output — works but with significant issues",
        "75: Good output — mostly correct with minor issues",
        "100: Excellent output — fully correct, clean, and complete",
      ],
      criteria_label: "Check for:",
      failureModes: failureModesForDimension("agent-output"),
    },
    "agent-tool-usage": {
      dimension: "tool-usage",
      header: "Score the agent's tool usage from 0 to 100:",
      scale: [
        "0: Completely wrong tool usage — called wrong tools or with invalid inputs",
        "25: Poor tool usage — correct tools but wrong parameters or sequencing",
        "50: Adequate — correct tools and basic parameters, some inefficiency",
        "75: Good — efficient tool usage with proper error handling",
        "100: Excellent — optimal tool selection, correct inputs, minimal redundancy",
      ],
      criteria_label: "Evaluate:",
      failureModes: failureModesForDimension("tool-usage"),
    },
  },

  // ── Named scoring profiles ────────────────────────────────
  profiles: {
    default: {
      "task-completion": 0.5,
      "code-correctness": 0.25,
      "doc-coverage": 0.25,
    },
    "output-only": {
      "task-completion": 0.6,
      "code-correctness": 0.4,
    },
    "mcp-behavior": {
      "input-validation": 0.25,
      "output-correctness": 0.35,
      "error-handling": 0.25,
      security: 0.15,
    },
    "knowledge-probe": {
      "factual-correctness": 0.45,
      completeness: 0.35,
      currency: 0.2,
    },
    "agent-harness": {
      "assertion-pass-rate": 0.35,
      "agent-output": 0.35,
      "tool-usage": 0.3,
    },
  },

  // ── Mode-to-profile bindings ──────────────────────────────
  // Literacy mode uses variant sub-keys because 'baseline' and 'agentic'
  // are scoring variants within the same canonical mode, not separate modes.
  "mode-profiles": {
    "literacy": {
      baseline: { gold: "default", baseline: "output-only" },
      agentic: { gold: "default" },
    },
    "mcp-server": { gold: "mcp-behavior" },
    "knowledge-probe": { gold: "knowledge-probe" },
    "agent-harness": { gold: "agent-harness" },
  },

  // W0273 — the footer documents the wire-format subset of GraderJudgment
  // that the grader LLM actually controls. The pipeline parses this against
  // GraderEmittedJudgmentSchema and then synthesizes the four pipeline-owned
  // fields (judgmentId, metadata.{graderModel,graderJudgmentsVersion},
  // hallucinationCheckedAgainst) to build the storage GraderJudgment.
  //
  // See docs/audits/2026-05-22-empty-gap-analysis-regression.md for the
  // rationale (Phase 3 GRAD-05 made these fields required + .strict(),
  // and asking the LLM for pipeline-owned values caused 100% parse
  // failures starting 2026-05-11).
  footer: `Return ONLY a JSON object with this exact shape:
{
  "score": <number 0-100>,
  "reason": "<explanation, ≤500 chars>",
  "failureMode": "<one of the declared modes for this dimension or 'unclassified'>",
  "subJudgments": [{ "criterionId": "<id>", "met": <bool>, "evidence": "<≤280 chars>", "confidence": { "level": "high|medium|low", "signalsPresent": <int>, "derivation": "<string>" } }],
  "docCitations": [{ "documentId": "<id>", "slug": "<optional slug>", "role": "supports|contradicts|missing|irrelevant", "hallucinated": <bool> }],
  "confidence": { "level": "high|medium|low", "signalsPresent": <int>, "derivation": "<string>" }
}`,
})
