/**
 * GeneralizedTaskDefinition — Mode-discriminated task definitions.
 *
 * This is the canonical task type for the architecture overhaul. It supports
 * all five evaluation modes via a discriminated union on the `mode`
 * field. Common fields (id, title, description, area, etc.) are shared;
 * mode-specific fields live only on the relevant variant.
 *
 * @see docs/design-docs/architecture-overhaul/domain-model.md (canonical)
 * @see docs/design-docs/architecture-overhaul/test-definition.md (authoring surfaces)
 * @see docs/archive/exec-plans/architecture-overhaul/phase-0-foundation-types.md (task 0h)
 */
/** Difficulty level for a task */
export type TaskDifficulty = "basic" | "intermediate" | "advanced";
/** Lifecycle status for a task */
export type TaskStatus = "active" | "draft" | "paused" | "archived";
/**
 * A canonical documentation reference — polymorphic resolution strategy.
 *
 * Each entry resolves docs through one of four strategies, discriminated
 * by key presence (no explicit `type` field). All strategies carry an
 * optional `reason` for context.
 *
 * @see docs/design-docs/canonical-doc-resolution.md
 */
export type GeneralizedDocRef = SlugDocRef | PathDocRef | IdDocRef | PerspectiveDocRef;
/** Resolve by article slug field. Legacy — prefer `path` for uniqueness. */
export interface SlugDocRef {
    slug: string;
    reason?: string;
}
/** Resolve by URL path (after /docs/). Unique across sections. */
export interface PathDocRef {
    path: string;
    reason?: string;
}
/** Resolve by Sanity document `_id`. The primary resolution strategy.
 *
 * Optional `slug` and `path` provide human-readable context — they are
 * NOT used for resolution (the `_id` is authoritative) but help YAML
 * authors understand which document is being referenced.
 */
export interface IdDocRef {
    id: string;
    reason?: string;
    /** Human-readable slug (informational only — not used for resolution) */
    slug?: string;
    /** Human-readable path (informational only — not used for resolution) */
    path?: string;
}
/** Resolve all articles in a content release. One-to-many. */
export interface PerspectiveDocRef {
    perspective: string;
    reason?: string;
}
/**
 * A single criterion within a templated llm-rubric assertion.
 *
 * The `id` is a stable, slug-formatted identifier — auto-derived from
 * `text` in Studio (via slug `options.source: "text"`), or backfilled
 * from Sanity's `_key` for pre-migration documents. Survives criterion
 * text edits; downstream judgments and diagnosis cards reference by
 * `id` per D0052 (judgment-ref granularity).
 */
export interface CriterionRef {
    /** Stable per-criterion identifier — slug-format (`[a-z0-9][a-z0-9-]*`). */
    id: string;
    /** Author-facing criterion text (the original bullet). */
    text: string;
}
/** A templated assertion referencing a rubric template */
export interface GeneralizedTemplatedAssertion {
    type: "llm-rubric";
    template: string;
    criteria: CriterionRef[];
    weight?: number;
}
/** A value-based assertion (contains, javascript, cost, latency, etc.) */
export interface GeneralizedValueAssertion {
    type: string;
    value?: unknown;
    weight?: number;
    [key: string]: unknown;
}
/** Any assertion definition */
export type GeneralizedAssertionDefinition = GeneralizedTemplatedAssertion | GeneralizedValueAssertion;
/** Rubric reference — either a named template or an inline rubric */
export type RubricRef = {
    ref: string;
} | {
    inline: string;
    dimensions?: {
        key: string;
        weight: number;
    }[];
};
/** Provider override for a specific task */
export interface TaskProviderConfig {
    /** Provider ID (e.g., "openai:chat:gpt-4o", "anthropic:messages:claude-sonnet-4-6") */
    id: string;
    /** Provider-specific configuration overrides */
    config?: Record<string, unknown>;
}
/** Task-level options */
export interface TaskOptions {
    /** Execution timeout in milliseconds */
    timeout?: number;
    /** Whether to cache results for this task */
    cache?: boolean;
    /** Output transform expression */
    transformOutput?: string;
    /** Arbitrary Promptfoo overrides (escape hatch) */
    promptfooOverrides?: Record<string, unknown>;
}
/**
 * Variable keys reserved by the AILF compilers. Populated automatically
 * from canonical task fields; forbidden in `prompt.vars` to prevent silent
 * override of canonical values.
 */
export type ReservedPromptVarKey = "task" | "docs" | "__featureArea";
/**
 * Variables for prompt-template interpolation. Freeform extras are
 * allowed; reserved keys (see {@link ReservedPromptVarKey}) are forbidden.
 */
export type PromptVars = Record<string, unknown> & {
    [K in ReservedPromptVarKey]?: never;
};
/** Fields shared by all task modes */
export interface TaskCommonFields {
    /** Unique task identifier */
    id: string;
    /** Human-readable title */
    title: string;
    /** Detailed description of what this task evaluates */
    description?: string;
    /** Feature area this task belongs to (e.g., "groq", "studio", "mutations") */
    area?: string;
    /** Difficulty level */
    difficulty?: TaskDifficulty;
    /** Freeform labels for filtering and organization */
    tags?: string[];
    /** Lifecycle status — controls pipeline inclusion */
    status?: TaskStatus;
    /** Assertion definitions (rubric templates + value assertions) */
    assertions?: GeneralizedAssertionDefinition[];
    /** Rubric template or inline rubric for grading */
    rubric?: RubricRef;
    /** Provider overrides for this task */
    providers?: TaskProviderConfig[];
    /** Task-level execution options */
    options?: TaskOptions;
    /**
     * Prompt configuration. Reserved `vars` keys (see
     * {@link ReservedPromptVarKey}) are forbidden — use `prompt.text` for
     * the prompt body and `context.docs` for documentation references.
     */
    prompt?: {
        /** Named prompt template */
        template?: string;
        /** Inline prompt text (mutually exclusive with template) */
        text?: string;
        /** System message override */
        systemMessage?: string;
        /** Variables for template interpolation (reserved keys forbidden) */
        vars?: PromptVars;
    };
    /** Arbitrary metadata */
    metadata?: Record<string, unknown>;
}
/**
 * Literacy mode — documentation quality evaluation.
 *
 * The original AILF evaluation mode. Tests whether AI coding tools
 * can find the right docs and produce correct code.
 */
export interface LiteracyTaskDefinition extends TaskCommonFields {
    mode: "literacy";
    /** Documentation context configuration */
    context?: {
        /** Canonical doc references for this task */
        docs?: GeneralizedDocRef[];
        /** Fixture references */
        fixtures?: string[];
    };
    /** Path to the reference solution (relative to eval package root) */
    referenceSolution?: string;
    /** Whether doc coverage rubric should be auto-generated */
    docCoverage?: boolean;
    /** Baseline variant configuration */
    baseline?: {
        enabled?: boolean;
        rubric?: "abbreviated" | "full" | "none";
    };
}
/**
 * MCP server mode — evaluates an MCP server's capabilities.
 *
 * Tests whether an MCP server correctly implements tools, resources,
 * and prompts according to the MCP specification.
 */
export interface MCPServerTaskDefinition extends TaskCommonFields {
    mode: "mcp-server";
    /** Server under test */
    serverConfig?: {
        /** Transport protocol */
        transport: "stdio" | "sse" | "streamable-http";
        /** Command to start the server (stdio) */
        command?: string;
        /** Server URL (sse/streamable-http) */
        url?: string;
        /** Environment variables for the server process */
        env?: Record<string, string>;
        /**
         * HTTP headers for remote transports (sse / streamable-http).
         * Merged on top of any auth-derived headers, so explicit values
         * here take precedence over `auth`-generated headers.
         *
         * Values support `{{env.VAR}}` template syntax for secrets.
         *
         * @example
         * headers: { Authorization: "Bearer {{env.SANITY_API_TOKEN}}" }
         */
        headers?: Record<string, string>;
        /** Startup timeout in milliseconds */
        startupTimeoutMs?: number;
        /**
         * Authentication config for the MCP server.
         * Maps directly to Promptfoo's auth config.
         * @see https://www.promptfoo.dev/docs/providers/mcp/#authentication
         */
        auth?: {
            type: "bearer" | "basic" | "api_key" | "oauth";
            /** Bearer token (for type: "bearer") */
            token?: string;
            /** Username (for type: "basic" or "oauth" password grant) */
            username?: string;
            /** Password (for type: "basic" or "oauth" password grant) */
            password?: string;
            /** API key value (for type: "api_key") */
            value?: string;
            /** Header/query param name for API key (default: "X-API-Key") */
            keyName?: string;
            /** "header" or "query" (for type: "api_key", default: "header") */
            placement?: "header" | "query";
            /** OAuth grant type */
            grantType?: "client_credentials" | "password";
            /** OAuth token endpoint URL */
            tokenUrl?: string;
            /** OAuth client ID */
            clientId?: string;
            /** OAuth client secret */
            clientSecret?: string;
            /** OAuth scopes */
            scopes?: string[];
        };
    };
    /** Expected capabilities to verify */
    capabilities?: string[];
    /** Documentation context (MCP servers can also be evaluated against docs) */
    context?: {
        docs?: GeneralizedDocRef[];
        tools?: string[];
        fixtures?: string[];
    };
    /** Multi-turn conversation definition */
    multiTurn?: {
        turns: {
            role: "user" | "assistant";
            content: string;
        }[];
    };
    /**
     * Override model list (provider IDs). When set, only these models are
     * used instead of models filtered from the registry by `mcp-server` mode.
     *
     * @example ["anthropic:messages:claude-opus-4-6"]
     */
    models?: string[];
    /**
     * Maximum number of tool call rounds before forcing a final text response.
     * Each round may include multiple parallel tool calls.
     * @default 5
     */
    maxToolRounds?: number;
}
/**
 * Agent harness mode — evaluates autonomous agents in a sandbox.
 *
 * Tests whether an AI agent can complete tasks correctly using a
 * defined set of tools in an isolated environment.
 */
export interface AgentHarnessTaskDefinition extends TaskCommonFields {
    mode: "agent-harness";
    /** Sandbox configuration for isolated execution */
    sandbox?: {
        type: "docker" | "git-worktree" | "none" | "nsjail" | "tempdir";
        image?: string;
        limits?: {
            cpus?: number;
            memoryBytes?: number;
            diskBytes?: number;
            networkAccess?: boolean;
        };
    };
    /** Tools available to the agent */
    tools?: string[];
    /** Fixture references for test data */
    fixtures?: string[];
    /** Documentation context */
    context?: {
        docs?: GeneralizedDocRef[];
        fixtures?: string[];
    };
    /** Multi-turn conversation definition */
    multiTurn?: {
        turns: {
            role: "user" | "assistant";
            content: string;
        }[];
    };
}
/**
 * Knowledge probe mode — measures knowledge coverage across a corpus.
 *
 * Tests breadth and depth of an LLM's knowledge about a specific
 * domain by systematically probing different aspects.
 */
export interface KnowledgeProbeTaskDefinition extends TaskCommonFields {
    mode: "knowledge-probe";
    /** How to traverse the knowledge space */
    probeStrategy?: "breadth-first" | "depth-first" | "random-sample" | "coverage-guided";
    /** Knowledge base reference */
    knowledgeBase?: {
        type: "sanity-dataset" | "embeddings-index" | "file-corpus";
        name: string;
        config?: Record<string, unknown>;
    };
    /** Documentation context */
    context?: {
        docs?: GeneralizedDocRef[];
        fixtures?: string[];
    };
}
/**
 * Custom mode — user-provided evaluation handler.
 *
 * The escape hatch for evaluation types not covered by the built-in
 * modes. Users provide a handler module implementing `EvalModeHandler`.
 */
export interface CustomTaskDefinition extends TaskCommonFields {
    mode: "custom";
    /** Module path to the custom handler */
    handler: string;
    /** Freeform schema for mode-specific configuration */
    schema?: Record<string, unknown>;
    /** Documentation context */
    context?: {
        docs?: GeneralizedDocRef[];
        fixtures?: string[];
    };
}
/**
 * Generalized task definition — a mode-discriminated union.
 *
 * The `mode` field determines which fields are available:
 * - `"literacy"` → context.docs, referenceSolution, docCoverage, baseline
 * - `"mcp-server"` → serverConfig, capabilities, multiTurn
 * - `"agent-harness"` → sandbox, tools, fixtures, multiTurn
 * - `"knowledge-probe"` → probeStrategy, knowledgeBase
 * - `"custom"` → handler, schema
 *
 * Use `defineTask()` from `@sanity/ailf` for full IDE autocomplete
 * when authoring tasks.
 */
export type GeneralizedTaskDefinition = LiteracyTaskDefinition | MCPServerTaskDefinition | AgentHarnessTaskDefinition | KnowledgeProbeTaskDefinition | CustomTaskDefinition;
/**
 * The subset of task modes that can be authored as `ailf.task` documents in
 * the Content Lake (Sanity Studio). Today exactly `"literacy"`.
 *
 * Expanding this set is a deliberate decision: execution-bound fields
 * (filesystem handles, local commands, sandbox config, module paths) cannot
 * round-trip through Content Lake, so not every mode belongs here. Adding a
 * mode requires a new or superseding ADR and a coordinated schema update
 * across the domain type, Studio schema, and `ContentLakeTaskSource` adapter
 * per the `ailf-schema-sync` skill.
 *
 * @see docs/decisions/D0038-content-lake-authorable-task-modes.md
 */
export type ContentLakeAuthorableMode = "literacy";
/**
 * The slice of `GeneralizedTaskDefinition` authorable in the Content Lake,
 * derived mechanically from `ContentLakeAuthorableMode`. Used as the return
 * type of `ContentLakeTaskSource` so the adapter's mode literal is
 * type-checked against the boundary rather than a loose cast.
 */
export type ContentLakeAuthorableTask = Extract<GeneralizedTaskDefinition, {
    mode: ContentLakeAuthorableMode;
}>;
