import type { LanguageModel } from "ai";
import type { HandleMessageStreamEvent, RuntimeIdentity } from "#protocol/message.js";
import type { SendTurnInput, SessionState } from "#client/types.js";
import type { InputRequest, InputResponse } from "#runtime/input/types.js";
import type { JsonObject } from "#shared/json.js";
import type { AgentModelOptionsDefinition } from "#shared/agent-definition.js";
import type { EvalReporter } from "#evals/runner/reporters/types.js";
/**
 * Assumptions an eval case needs the runner to verify against the live target
 * or eval process environment before executing the case.
 */
export type EveEvalRequirement = "mockModels" | "devRoutes" | `env:${string}`;
/**
 * Shared fields for data and scripted eval cases.
 */
interface EveEvalCaseBase {
    /** Uniquely identifies the case within its eval. */
    readonly id: string;
    /**
     * Reference value scorers compare against. The runner coerces this to a
     * string for autoevals-compatible scorers; `args.case.expected` exposes it
     * unmodified.
     */
    readonly expected?: unknown;
    /**
     * Hard assertions for this case, appended to the eval-level `checks`.
     * Any failure marks the case failed and flips the CLI exit code.
     */
    readonly checks?: readonly EveEvalCheck[];
    /** Additional scorers for this case, appended to the eval-level `scores`. */
    readonly scores?: readonly EveEvalScorer[];
    /** Case-local target/process assumptions, appended to eval-level `requires`. */
    readonly requires?: readonly EveEvalRequirement[];
    /** Used by `--tag` filtering, and passed through for reporting. */
    readonly tags?: readonly string[];
    /** Passed through for reporting and Braintrust span logging. */
    readonly metadata?: Readonly<Record<string, unknown>>;
}
/**
 * Dataset-shaped eval case routed through the eval-level task.
 */
export interface EveEvalDataCase extends EveEvalCaseBase {
    /**
     * The case prompt, or a structured record the task derives messages from.
     * A string is sent verbatim and a record is `JSON.stringify`d, unless the
     * task's `messages`/`prompt` reads specific fields.
     */
    readonly input: string | Record<string, unknown>;
    readonly run?: never;
}
/**
 * Scripted eval case with self-contained interaction control flow.
 */
export interface EveEvalScriptedCase extends EveEvalCaseBase {
    readonly input?: never;
    run(ctx: EveEvalRunContext): unknown | Promise<unknown | void> | void;
}
/**
 * One normalized eval case. Evals produce these from raw data sources.
 */
export type EveEvalCase = EveEvalDataCase | EveEvalScriptedCase;
/**
 * One tool call extracted from the captured stream, pairing the
 * `actions.requested` request with its matching `action.result`.
 */
export interface EveEvalToolCall {
    /** Authored tool name (e.g. `"get_weather"`). */
    readonly name: string;
    /** Tool input as requested by the model. */
    readonly input: JsonObject;
    /** Tool output from the matching `action.result`; `undefined` when the call never resolved. */
    readonly output: unknown;
    /** True when the matching `action.result` reported a failure. */
    readonly isError: boolean;
    /** Zero-based index of the turn the call happened in. */
    readonly turnIndex: number;
    /** Owning session id, when the runner knows it. */
    readonly sessionId?: string;
}
/**
 * One subagent delegation extracted from the captured stream
 * (`subagent.called` / `subagent.started`, joined with `subagent.completed`).
 */
export interface EveEvalSubagentCall {
    /** Subagent name. */
    readonly name: string;
    /** Remote agent URL for remote delegations (`subagent.called` remote metadata). */
    readonly remoteUrl?: string;
    /** Output from the matching `subagent.completed` event; `undefined` when the call never completed. */
    readonly output?: unknown;
    /** True when the matching subagent action result reported a failure. */
    readonly isError: boolean;
    /** Zero-based index of the turn the delegation happened in. */
    readonly turnIndex: number;
    /** Owning session id, when the runner knows it. */
    readonly sessionId?: string;
}
/**
 * Execution facts the runner extracts from a completed session's stream events.
 */
export interface EveEvalDerivedFacts {
    readonly toolCalls: readonly EveEvalToolCall[];
    readonly toolCallCount: number;
    readonly subagentCalls: readonly EveEvalSubagentCall[];
    readonly subagentCallCount: number;
    /** Every HITL input request raised during the run (`input.requested`). */
    readonly inputRequests: readonly InputRequest[];
    /** True when the run ended parked on unanswered HITL input requests. */
    readonly parked: boolean;
    readonly messageCount: number;
    readonly reasoningBlockCount: number;
    readonly failureCode?: string;
}
/**
 * Captured event stream and facts for one session involved in a case.
 */
export interface EveEvalSessionResult {
    readonly derived: EveEvalDerivedFacts;
    readonly events: readonly HandleMessageStreamEvent[];
    readonly primary: boolean;
    readonly sessionId?: string;
    readonly state: SessionState;
}
/**
 * Full result of executing one eval case against an Eve agent.
 */
export interface EveEvalTaskResult {
    /**
     * The scored value. Defaults to `finalMessage` unless `task.parseOutput`
     * overrides it. Mutable because the runner assigns it after parsing.
     */
    output: unknown;
    /** The agent's last assistant message, or null when none was produced. */
    readonly finalMessage: string | null;
    readonly sessionId?: string;
    /**
     * How the run's final turn ended: `"completed"` (session finished),
     * `"failed"` (terminal failure), or `"waiting"` (parked for the next
     * user message).
     */
    readonly status: "completed" | "failed" | "waiting";
    /** The captured stream events from the run. */
    readonly events: readonly HandleMessageStreamEvent[];
    /** Lines written through `ctx.log` while the case ran. */
    readonly logs?: readonly string[];
    /** Facts extracted from the stream (tool calls, message counts, etc.). */
    readonly derived: EveEvalDerivedFacts;
    /** Per-session event streams captured while executing this case. */
    readonly sessions?: readonly EveEvalSessionResult[];
    /**
     * Runtime identity metadata captured from the `session.started` stream event.
     * Present when the Eve server populates the event with its runtime metadata.
     */
    readonly runtimeIdentity?: RuntimeIdentity;
}
/**
 * Result returned by a single scorer invocation.
 */
export interface EveEvalScorerResult {
    /** Scorer name. Used as the key in Braintrust score maps. */
    readonly name: string;
    /** Score between 0 and 1, or null if the scorer could not produce a score. */
    readonly score: number | null;
    /** Optional metadata for debugging or Braintrust span logging. */
    readonly metadata?: Readonly<Record<string, unknown>>;
}
/**
 * Arguments passed to every scorer invocation.
 *
 * The flat `input`, `output`, and `expected` fields are coerced to `string`
 * for compatibility with Braintrust autoevals scorers (e.g. `Factuality`,
 * `Levenshtein`). For the original values, use `"input" in case ? case.input
 * : undefined`, `case.expected`, and `result.output`.
 */
export interface EveEvalScorerArgs {
    /** The eval case input, coerced to string. Autoevals-compatible. */
    readonly input: string;
    /** The task output, coerced to string. Autoevals-compatible. */
    readonly output: string;
    /** The expected value, coerced to string. Autoevals-compatible. */
    readonly expected?: string | undefined;
    /**
     * Eval-level scorer model, when the eval provides one. This does not
     * change the target Eve agent model. `undefined` when the eval omits
     * `model`; model-backed scorers throw a descriptive error in that case.
     */
    readonly model: LanguageModel | undefined;
    /** Eval-level provider options for model-backed scorers. */
    readonly modelOptions?: AgentModelOptionsDefinition;
    /** Full eval case. */
    readonly case: EveEvalCase;
    /** Full task result with events and derived facts. */
    readonly result: EveEvalTaskResult;
}
/**
 * Receives flattened input/output/expected fields (autoevals-compatible) plus
 * the full Eve case and result. Return `null` to skip scoring a case (e.g. when
 * expected is absent).
 */
export type EveEvalScorer = (args: EveEvalScorerArgs) => EveEvalScorerResult | Promise<EveEvalScorerResult | null> | null;
/**
 * Result returned by a single check invocation.
 *
 * Unlike scores, checks are hard assertions: any `passed: false` marks the
 * case failed and produces a non-zero `eve eval` exit code.
 */
export interface EveEvalCheckResult {
    readonly name: string;
    readonly passed: boolean;
    /** Human-readable failure detail, shown in console output and artifacts. */
    readonly message?: string;
    readonly metadata?: Readonly<Record<string, unknown>>;
}
/**
 * Arguments passed to every check invocation. Checks see the same result data
 * as scorers but never receive a judge model — they are deterministic
 * assertions over the captured run.
 */
export interface EveEvalCheckArgs {
    readonly case: EveEvalCase;
    readonly result: EveEvalTaskResult;
    /** The target under test, so checks can reference runner-assigned values like its URL. */
    readonly target: EveEvalTargetHandle;
}
/**
 * One hard assertion over a completed eval case. Built-ins live in
 * `eve/evals/checks`; custom checks are plain functions with this shape.
 */
export type EveEvalCheck = (args: EveEvalCheckArgs) => EveEvalCheckResult | Promise<EveEvalCheckResult>;
/**
 * Driver exposed to `task.run` and scripted cases.
 */
export interface EveEvalSession {
    /** All events observed on this session so far. */
    readonly events: readonly HandleMessageStreamEvent[];
    /** Input requests left pending by the last parked turn. */
    readonly pendingInputRequests: readonly InputRequest[];
    /** Serializable cursor for resuming this session. */
    readonly state: SessionState;
    /** Eve session id after the first successful send. */
    readonly sessionId: string | undefined;
    /** Assert the last turn parked on HITL input and return matching requests. */
    expectInputRequests(filter?: {
        readonly display?: InputRequest["display"];
        readonly toolName?: string;
    }): readonly InputRequest[];
    /** Resolve specific pending requests and run the resumed turn. */
    respond(...responses: InputResponse[]): Promise<EveEvalTurn>;
    /** Resolve every pending request with the same option id. */
    respondAll(optionId: string): Promise<EveEvalTurn>;
    /** Send one turn through this session. */
    send(input: SendTurnInput): Promise<EveEvalTurn>;
    /** Send one text turn with a local file attached as a data URL. */
    sendFile(text: string, filePath: string, mediaType?: string): Promise<EveEvalTurn>;
}
/**
 * One completed eval-driver turn.
 */
export interface EveEvalTurn {
    readonly data: unknown;
    readonly events: readonly HandleMessageStreamEvent[];
    readonly inputRequests: readonly InputRequest[];
    readonly message: string | undefined;
    readonly status: "completed" | "failed" | "waiting";
    readonly toolCalls: readonly EveEvalToolCall[];
    expectOk(): this;
}
/**
 * Context passed to imperative eval runs.
 */
export interface EveEvalRunContext {
    /** The case under execution. */
    readonly case: EveEvalCase;
    /** Case timeout signal. */
    readonly signal: AbortSignal;
    /** Primary session, fresh per case. */
    readonly session: EveEvalSession;
    /** Current target under test. */
    readonly target: EveEvalTargetHandle;
    /** Structured eval log hook. */
    readonly log: (message: string) => void;
    /** Create an additional independent session against the same target. */
    newSession(): EveEvalSession;
}
interface EveEvalTaskBase {
    /**
     * Transform the raw task result into the scored output value. Available on
     * every task variant. When omitted, `result.output` defaults to
     * `result.finalMessage`.
     */
    readonly parseOutput?: (result: EveEvalTaskResult) => unknown;
}
export interface EveEvalTaskFields extends EveEvalTaskBase {
    readonly messages?: (testCase: EveEvalCase) => string[];
    readonly prompt?: (testCase: EveEvalCase) => string;
    readonly run?: (ctx: EveEvalRunContext) => unknown | Promise<unknown | void> | void;
}
/**
 * Declarative task configuration for an eval. The runner owns session
 * lifecycle, stream capture, and derived metadata; evals declare either a
 * scripted run, derived messages, a prompt, or just parse output.
 */
export type EveEvalTask = (EveEvalTaskBase & {
    /**
     * Imperative interaction script for one eval case.
     * Mutually exclusive with `prompt` and `messages`.
     */
    readonly run: (ctx: EveEvalRunContext) => unknown | Promise<unknown | void> | void;
    readonly messages?: never;
    readonly prompt?: never;
}) | (EveEvalTaskBase & {
    /**
     * Derive an ordered list of messages for a multi-turn eval.
     * Mutually exclusive with `prompt` and `run`.
     */
    readonly messages: (testCase: EveEvalCase) => string[];
    readonly prompt?: never;
    readonly run?: never;
}) | (EveEvalTaskBase & {
    readonly messages?: never;
    /**
     * Derive a single prompt string from one eval case.
     * Mutually exclusive with `messages` and `run`.
     */
    readonly prompt: (testCase: EveEvalCase) => string;
    readonly run?: never;
}) | (EveEvalTaskBase & {
    readonly run?: never;
    readonly messages?: never;
    readonly prompt?: never;
});
/**
 * Describes the Eve server an eval runs against.
 */
export interface EveEvalTarget {
    /**
     * `"local"` for a dev server the runner starts in-process, `"remote"` for
     * a deployed instance addressed by `--url`.
     */
    readonly kind: "local" | "remote";
    /** Base HTTP URL the eval client connects to and sends message requests. */
    readonly url: string;
    /** Capabilities discovered from the live target's info route. */
    readonly capabilities: EveEvalTargetCapabilities;
}
export interface EveEvalTargetCapabilities {
    readonly devRoutes: boolean;
    readonly mockModels: boolean;
}
export interface EveEvalScheduleDispatchResult {
    readonly scheduleId: string;
    readonly sessionIds: readonly string[];
}
/**
 * Live target handle exposed to eval cases.
 */
export interface EveEvalTargetHandle extends EveEvalTarget {
    /** Dispatch a dev-only authored schedule. Requires declaring `"devRoutes"`. */
    dispatchSchedule(scheduleId: string): Promise<EveEvalScheduleDispatchResult>;
    /** Authenticated fetch against the target base URL. */
    fetch(path: string, init?: RequestInit): Promise<Response>;
    /** Attach to a pre-existing session and consume one turn boundary. */
    attachSession(sessionId: string, opts?: {
        readonly startIndex?: number;
    }): Promise<EveEvalSession>;
}
/**
 * Shared fields between the user-facing input and the validated eval.
 *
 * Eval identity (`id`) is derived from the `evals/<path>.eval.ts` file
 * path by the discovery layer; it is not authored on the input.
 */
interface EveEvalBase {
    readonly description?: string;
    readonly task?: EveEvalTask;
    /**
     * Hard assertions applied to every case in the eval. Case-level `checks`
     * append to these. Any failed check marks the case failed and produces a
     * non-zero `eve eval` exit code, unlike scores which stay soft data.
     */
    readonly checks?: readonly EveEvalCheck[];
    readonly scores: readonly EveEvalScorer[];
    /**
     * Target/process assumptions applied to every case. Case-level
     * requirements append to these and are verified before execution.
     */
    readonly requires?: readonly EveEvalRequirement[];
    /**
     * Model used by model-backed scorers in this eval. Required only when a
     * model-backed scorer (e.g. the `Autoevals` wrappers) is present without
     * its own per-scorer model override.
     *
     * String model IDs route through the Vercel AI Gateway; the runner uses
     * provider model instances directly. This model is only for scoring and
     * never changes the Eve agent under test.
     */
    readonly model?: LanguageModel;
    /**
     * Provider-specific options passed to model-backed scorers.
     */
    readonly modelOptions?: AgentModelOptionsDefinition;
    /**
     * Maximum number of cases the runner executes at once.
     * Defaults to 8 when omitted.
     */
    readonly maxConcurrency?: number;
    readonly timeoutMs?: number;
    /** Used by `--tag` filtering: an eval carrying a requested tag runs all its cases. */
    readonly tags?: readonly string[];
    readonly metadata?: Readonly<Record<string, unknown>>;
    readonly reporters?: readonly EvalReporter[];
    /**
     * Minimum score thresholds per scorer name. A case "passes" when every
     * scorer meets or exceeds its threshold. Scorers not listed here
     * default to a threshold of 1.0 (exact match).
     *
     * @example
     * ```ts
     * thresholds: {
     *   "Factuality": 0.5,
     *   "run.didNotFail": 1.0,
     * }
     * ```
     */
    readonly thresholds?: Readonly<Record<string, number>>;
}
/**
 * Complete top-level key set accepted by {@link defineEval}, used to reject
 * unknown authored keys. The stricter {@link EveEvalInput} union enforces
 * `load`/`cases` exclusivity.
 */
export interface EveEvalInputFields extends Omit<EveEvalBase, "task"> {
    readonly cases?: readonly EveEvalCase[];
    readonly task?: EveEvalTaskFields;
    load?(): Promise<EveEvalCase[]>;
}
/**
 * Full eval input passed to `defineEval()`.
 *
 * Provide either `load` (async function) or `cases` (static array), not both;
 * `cases` causes the runner to synthesize a `load` internally. Eval identity is
 * derived from the file path, so authors do not specify an `id` or `name`.
 */
export type EveEvalInput = (EveEvalBase & {
    readonly cases?: never;
    /** Load cases dynamically. Mutually exclusive with `cases`. */
    load(): Promise<EveEvalCase[]>;
}) | (EveEvalBase & {
    /** Static inline cases. Mutually exclusive with `load`. */
    readonly cases: readonly EveEvalCase[];
    load?: never;
});
/**
 * Eval returned by `defineEval()`. Carries no `id` yet: discovery stamps
 * the path-derived id at import time to produce a full {@link EveEval}. The
 * `_tag` literal (`"EveEval"`) brands the value so discovery and the runner
 * can recognize a defined eval.
 */
export interface EveEvalDefinition extends EveEvalBase {
    readonly _tag: "EveEval";
    /** Always present. Synthesized from `cases` when the input uses static cases. */
    load(): Promise<EveEvalCase[]>;
}
/**
 * Validated eval consumed by the runner and reporters. The `id` is the
 * path-derived slug attached by discovery
 * (e.g. `evals/weather.eval.ts` → `"weather"`).
 */
export interface EveEval extends EveEvalDefinition {
    readonly id: string;
}
/**
 * Per-case outcome computed by the runner:
 *
 * - `"passed"`  — no execution error, every check passed, every score met its threshold
 * - `"failed"`  — a check failed or execution errored (timeout, transport, thrown task)
 * - `"scored"`  — passed checks but at least one score fell below its threshold
 * - `"skipped"` — the case was not executed (reserved for unmet `requires` entries)
 */
export type EveEvalCaseVerdict = "passed" | "failed" | "scored" | "skipped";
/**
 * Result of scoring one eval case.
 */
export interface EveEvalCaseResult {
    readonly case: EveEvalCase;
    readonly result: EveEvalTaskResult;
    /** Hard-assertion results (eval-level checks first, then case-level). */
    readonly checks: readonly EveEvalCheckResult[];
    readonly scores: readonly EveEvalScorerResult[];
    /** Per-case verdict; see {@link EveEvalCaseVerdict}. */
    readonly verdict: EveEvalCaseVerdict;
    readonly error?: string;
    /** Why the case was skipped, when `verdict` is `"skipped"`. */
    readonly skipReason?: string;
}
/**
 * Aggregated result for one eval run.
 *
 * `id` is the path-derived eval id
 * (e.g. `evals/weather.eval.ts` → `"weather"`).
 */
export interface EveEvalResult {
    readonly id: string;
    readonly target: EveEvalTarget;
    readonly cases: readonly EveEvalCaseResult[];
    readonly startedAt: string;
    readonly completedAt: string;
    /** Cases with verdict `"passed"`. */
    readonly passed: number;
    /** Cases with verdict `"failed"` (check failures and execution errors). */
    readonly failed: number;
    /** Cases with verdict `"scored"` (below-threshold scores only). */
    readonly scored: number;
    /** Cases with verdict `"skipped"`. */
    readonly skipped: number;
    /** The execution-error subset of `failed` (timeouts, connection failures, exceptions). */
    readonly errored: number;
}
export {};
