/**
 * Knowledge probe: Sanity defineType API.
 *
 * Tests whether models know the current typed schema API
 * vs the legacy untyped format.
 *
 * Knowledge probes test the model's built-in knowledge WITHOUT providing documentation
 * context (no `context.docs`). Contrast with "literacy" tasks which inject docs.
 */
import { defineTask } from "../../_vendor/ailf-core/index.js"

export default defineTask({
  // "knowledge-probe" mode: no docs injected — measures what the model already knows
  mode: "knowledge-probe",
  id: "kp-define-type-api",
  title: "What is Sanity's defineType API?",
  description:
    "Explain how to define document schemas in Sanity using the defineType, " +
    "defineField, and defineArrayMember helper functions.",
  // Used for score aggregation in reports and --area CLI filtering
  area: "studio",
  // Metadata for reporting; does not affect evaluation behavior
  difficulty: "basic",
  // Freeform labels for --tag CLI filtering
  tags: ["knowledge-probe", "studio", "schema"],
  // Controls how the probe explores knowledge: "breadth-first" covers many topics, "depth-first" drills deep
  probeStrategy: "breadth-first",
  prompt: {
    // Direct prompt text sent to the model. The compiler treats `prompt.text`
    // as the canonical prompt body across every mode.
    text:
      "Explain Sanity's schema definition API:\n\n" +
      "1. What is `defineType` and how do you use it?\n" +
      "2. What are `defineField` and `defineArrayMember`?\n" +
      "3. Why were these typed helpers introduced? What did they replace?\n" +
      "4. Show a complete example of a document schema with various field types\n" +
      "5. How do you add validation rules using the typed API?",
  },
  assertions: [
    { type: "contains", value: "defineType" },
    { type: "contains", value: "defineField" },
    // Inline llm-rubric (value is the rubric text itself, unlike template+criteria in literacy tasks)
    {
      type: "llm-rubric",
      value:
        "The response should accurately explain the typed schema helpers. " +
        "Check that the code examples use the current API, not the legacy " +
        "untyped format. Penalize if the response uses the old pattern " +
        "without mentioning defineType.",
    },
    {
      type: "llm-rubric",
      value:
        "Evaluate currency: does the response know about defineType " +
        "(introduced in Sanity v3)? Does it mention TypeScript type " +
        "inference benefits?",
    },
  ],
})
