/**
 * EvalCampaign — opinionated matrix runner that wires the four
 * capture-integrity directives by construction.
 *
 * The canonical benchmark shape — matrix runner → for each
 * (variant, scenario, seed) → start a TraceEmitter → call LLMs → end the
 * run → analyze — has a bug class at the integration boundary: raw
 * events not captured, route silently wrong, integrity not asserted,
 * analyst never run. The directives in `SKILL.md § Capture integrity`
 * are the mitigations.
 *
 * `EvalCampaign` is the structural fix — consumers don't wire the
 * integrity surface themselves; the campaign owns it. Specifically:
 *
 *   - calls `assertLlmRoute` once at preflight before any work runs
 *   - constructs a per-run `TraceStore` and `RawProviderSink` via factories
 *   - constructs the `TraceEmitter` with `onRunComplete: [analyst hook]`
 *   - hands the runner an `LlmClientOptions` pre-wired with the sink and
 *     trace context — the runner can't accidentally call an LLM without
 *     capturing the raw HTTP envelope
 *   - calls `assertRunCaptured` after every `endRun` and routes failures
 *     through a configurable policy (`throw` / `mark_failed` / `log`)
 *   - assembles per-run `RunRecord`s and runs `researchReport` at the end
 *     so the campaign artifact is launch-decision-grade by default
 *   - embeds the campaign fingerprint (a SHA-256 over the canonicalised
 *     run set) and optional `preregistrationHash` in the report
 *
 * The runner contract is intentionally narrow: produce a `CampaignRunOutcome`
 * given a fully-wired `CampaignRunContext`. Everything orchestration-shaped
 * lives in the campaign. This is the inversion-of-control point — consumers
 * stop writing matrix runners and start writing scenario-runners.
 *
 * Out of scope for v1 (tracked in `docs/research-report-methodology.md`):
 *
 *   - Distributed/cluster execution (concurrency is local async)
 *   - Adaptive sampling / sequential interim looks
 *   - Resume from partial state across crashes
 *   - LLM-call retry beyond what `LlmClient` already does
 */
import { type AgentProfileCell, type AgentProfileCellInput } from './agent-profile-cell';
import { type LlmClientOptions, type LlmRouteRequirements } from './llm-client';
import type { JudgeScoresRecord, RunJudgeMetadata, RunRecord, RunSplitTag, RunTokenUsage } from './run-record';
import { type ResearchReport, type ResearchReportOptions } from './summary-report';
import type { RunCompleteHook } from './trace/emitter';
import { TraceEmitter } from './trace/emitter';
import { type RunIntegrityExpectations, type RunIntegrityReport } from './trace/integrity';
import { type RawProviderSink } from './trace/raw-provider-sink';
import type { FailureClass } from './trace/schema';
import type { TraceStore } from './trace/store';
export interface CampaignVariant<V> {
    id: string;
    payload: V;
}
export interface CampaignScenario {
    scenarioId: string;
    /** Free-form metadata propagated to runs and reports. */
    tags?: Record<string, string>;
}
export interface CampaignRunContext<V> {
    /** Stable run id. The campaign generates this; the runner does not. */
    runId: string;
    /** Logical experiment id (campaignId by default; overridable per-run via opts). */
    experimentId: string;
    variant: V;
    variantId: string;
    scenarioId: string;
    scenarioTags: Record<string, string>;
    seed: number;
    splitTag: RunSplitTag;
    /**
     * The TraceEmitter for this run, with `onRunComplete` hooks pre-wired
     * (analyst auto-execution if configured, plus integrity check). The
     * runner MUST call `emitter.startRun` before doing any work and either
     * `emitter.endRun` or `emitter.abortRun` before returning.
     */
    emitter: TraceEmitter;
    store: TraceStore;
    rawSink: RawProviderSink;
    /**
     * Pre-wired LLM client options — `rawSink` and `traceContext` are populated
     * so any `callLlm(req, ctx.llmOpts)` automatically captures raw HTTP. The
     * runner can spread additional fields if needed.
     */
    llmOpts: LlmClientOptions;
}
export interface CampaignRunOutcome {
    /** Did the run pass? Mirrors `RunOutcome.pass` semantics. */
    pass: boolean;
    /** Score for the run on its split. Maps to `searchScore` or `holdoutScore`. */
    score: number;
    /** Mandatory cost in USD. Use 0 + raw.cost_unknown=1 only if truly unknown. */
    costUsd: number;
    tokenUsage: RunTokenUsage;
    /** Snapshot model id (e.g. `claude-sonnet-4-6@2025-04-15`). */
    model: string;
    /** sha256 of the effective prompt sent to the model. */
    promptHash: string;
    /** sha256 of the effective config (model, temperature, tools, judges, splits). */
    configHash: string;
    /** Optional extra numeric metrics to land in `outcome.raw`. */
    raw?: Record<string, number>;
    /** Canonical cross-agent failure class from the shared `FAILURE_CLASSES`
     *  taxonomy. Propagated to `RunRecord.failureClass` so campaign runs
     *  aggregate failures in the same vocabulary as every other producer. */
    failureClass?: FailureClass;
    /** Optional free-form failure detail, scoped under `failureClass`. */
    failureMode?: string;
    /** Optional judge metadata when a judge was used. */
    judgeMetadata?: RunJudgeMetadata;
    /**
     * Optional per-judge / per-dim breakdown for ensemble-judged runs.
     * Propagated to `outcome.judgeScores` on the resulting `RunRecord`.
     * Single-judge or scalar-only runs leave this unset.
     */
    judgeScores?: JudgeScoresRecord;
    /**
     * Agent profile cell observed by the runner. When supplied, it overrides
     * `EvalCampaignOptions.agentProfile` for this run and must match the
     * outcome's `model` and `promptHash`.
     */
    agentProfile?: AgentProfileCell | AgentProfileCellInput;
}
export type CampaignRunner<V> = (ctx: CampaignRunContext<V>) => Promise<CampaignRunOutcome>;
export type CampaignIntegrityPolicy = 'throw' | 'mark_failed' | 'log';
export interface EvalCampaignOptions<V> {
    /**
     * Stable id for the campaign. Used as the default `experimentId` on
     * every run, and folded into the campaign fingerprint.
     */
    campaignId: string;
    variants: CampaignVariant<V>[];
    scenarios: CampaignScenario[];
    /** Default `[0, 1, 2]`. */
    seeds?: number[];
    /** Default `'holdout'` — the split that anchors a launch decision. */
    splitTag?: RunSplitTag;
    /** Git SHA the campaign is run against. Mandatory; `RunRecord` rejects unset. */
    commitSha: string;
    /**
     * LLM client config. Augmented per-run with `rawSink` and `traceContext`
     * before being passed to the runner. The campaign asserts this config
     * matches `routeRequirements` once at preflight.
     */
    llmOpts: LlmClientOptions;
    /**
     * Default `{ requireExplicitBaseUrl: true, requireAuth: true }` — fail
     * loud if the campaign would silently fall back to the public router or
     * run unauthenticated. Override with an empty object to disable.
     */
    routeRequirements?: LlmRouteRequirements;
    /**
     * Per-run TraceStore factory. Common shape: a fresh store per run keyed
     * on `runId`. Implementations that share a store across the campaign
     * are valid — the campaign only writes through `emitter`.
     */
    storeFactory: (params: CampaignFactoryParams) => TraceStore;
    /**
     * Per-run RawProviderSink factory. Defaults to `FileSystemRawProviderSink`
     * rooted at `${workDir}/raw-events/${runId}` if `workDir` is supplied;
     * otherwise required. Forensic capture is non-negotiable in a campaign
     * run — pass `NoopRawProviderSink` explicitly if you want to opt out.
     */
    rawSinkFactory?: (params: CampaignFactoryParams) => RawProviderSink;
    /**
     * Filesystem root for default `rawSinkFactory`. Ignored if
     * `rawSinkFactory` is supplied.
     */
    workDir?: string;
    /**
     * Extra `onRunComplete` hooks the campaign appends (after its own
     * integrity-check hook). Pass `traceAnalystOnRunComplete(...)` here.
     */
    onRunComplete?: RunCompleteHook[];
    /**
     * Per-run integrity expectations. Defaults to:
     *   `{ llmSpansMin: 1, requireRawCoverageOfLlmSpans: true, requireOutcome: true }`.
     * Override (e.g. `{ llmSpansMin: 0 }`) for runs that don't call LLMs.
     */
    integrity?: RunIntegrityExpectations;
    /** Behaviour when integrity fails. Default `'mark_failed'`. */
    onIntegrityFailure?: CampaignIntegrityPolicy;
    /**
     * Per-run runner. Receives a fully-wired context; produces an outcome
     * the campaign converts into a `RunRecord`.
     */
    runner: CampaignRunner<V>;
    /**
     * If set, the campaign computes `researchReport` at the end. `comparator`
     * is a `variantId`. Other fields are forwarded verbatim.
     */
    report?: {
        comparator?: string;
    } & Omit<ResearchReportOptions, 'comparator' | 'preregistrationHash' | 'generatedAt'>;
    /**
     * Hash of a signed `HypothesisManifest` (see `pre-registration.ts`).
     * Embedded in the campaign fingerprint and the research report.
     */
    preregistrationHash?: string;
    /** Local concurrency. Default `1` (sequential). */
    concurrency?: number;
    /**
     * Override the time source. Tests pass a mock to make wallMs deterministic.
     */
    now?: () => number;
    /** Override the runId generator. Tests pin this. */
    runId?: (params: CampaignFactoryParams) => string;
    /**
     * Agent profile cell for campaign runs. Static profiles can pass an object;
     * routers or variant-specific harnesses can pass a factory. The campaign
     * stamps the built cell onto every `RunRecord` and rejects profile/model or
     * profile/prompt contradictions.
     */
    agentProfile?: AgentProfileCell | AgentProfileCellInput | ((params: CampaignFactoryParams & {
        variant: V;
        scenarioTags: Record<string, string>;
    }) => AgentProfileCell | AgentProfileCellInput | Promise<AgentProfileCell | AgentProfileCellInput>);
}
export interface CampaignFactoryParams {
    campaignId: string;
    runId: string;
    variantId: string;
    scenarioId: string;
    seed: number;
}
export interface FailedRun {
    runId: string;
    variantId: string;
    scenarioId: string;
    seed: number;
    reason: string;
    error?: string;
}
export interface EvalCampaignResult {
    campaignId: string;
    /** SHA-256 over canonicalised `(variantIds, scenarioIds, seeds, comparator, splitTag, baseUrl, provider, preregistrationHash)`. */
    campaignFingerprint: string;
    preregistrationHash: string | null;
    /** Successful runs only. Failed runs land in `failedRuns`. */
    runs: RunRecord[];
    /** Integrity reports for every successful run. */
    integrityReports: RunIntegrityReport[];
    failedRuns: FailedRun[];
    /** Computed when `report` is set on options. */
    report?: ResearchReport;
    startedAt: string;
    endedAt: string;
}
export declare function runEvalCampaign<V>(opts: EvalCampaignOptions<V>): Promise<EvalCampaignResult>;
/**
 * Abort a run whose owning work threw or was orphaned by a sibling's genuine
 * error, finalizing the emitter so hooks fire and the store records a terminal
 * status. Safe to call unconditionally: it only aborts runs that are still
 * `running`. Two no-op cases are intentional and benign:
 *
 *   - the run was never started (absent from the store) — nothing to finalize
 *   - the run is already terminal (`completed` / `failed` / `aborted`) —
 *     finalizing again would overwrite the real outcome (e.g. flip a passed run
 *     to `aborted` with `{ pass: false, notes: reason }`), so we leave it alone
 *
 * Any OTHER failure of the store read or the abort write (disk full, FS fault,
 * backend down) is a genuine diagnostic and propagates rather than being
 * swallowed.
 */
export declare function finalizeAbort(emitter: TraceEmitter, runId: string, reason: string): Promise<void>;
//# sourceMappingURL=eval-campaign.d.ts.map