/**
 * HeldOutGate — first-class held-out paired-delta promotion gate.
 *
 * Encodes the "honesty override" pattern that lived inline in
 * `~/webb/redteam/scripts/agent-eval-autoresearch.ts:138–171`.
 * The optimizer's best-guess is one thing; what we should actually
 * ship is another. The gate is the line between them.
 *
 * A candidate is promoted iff ALL three pass:
 *
 *   1. **Productive runs**: the candidate has at least
 *      `minProductiveRuns` paired observations on items where BOTH
 *      candidate and baseline produced a real (non-silent) score.
 *   2. **Paired delta**: the lower bound of the bootstrap CI on the
 *      median per-item delta (candidate − baseline) on the HOLDOUT
 *      split is strictly greater than `pairedDeltaThreshold`.
 *   3. **Overfit gap**: the candidate's gap between search-split
 *      score and holdout-split score is no worse (more positive)
 *      than the baseline's gap by more than `overfitGapThreshold`.
 *      "Better on search, worse on holdout" is the canonical
 *      overfit pattern; this catches it.
 *
 * The decision carries a machine-readable `rejectionCode` plus an
 * `evidence` block with every number the gate looked at, so the
 * downstream researcher / paper / dashboard can re-derive the
 * verdict without re-running.
 *
 * See also:
 *   - `src/statistics.ts` for `pairedBootstrap` + `wilcoxonSignedRank`
 *   - `src/run-record.ts` for the input row schema
 *   - `src/reference-replay.ts` for the older, reference-replay-
 *     specific promotion path (still useful for replay-style evals).
 */
import type { RunRecord } from './run-record';
export type HeldOutGateRejectionCode = 'few_runs' | 'negative_delta' | 'overfit_gap' | 'cost_ceiling';
export interface HeldOutGateConfig {
    /** Minimum number of paired (candidate, baseline) holdout observations
     *  required before the gate will even consider promoting. Default 3. */
    minProductiveRuns?: number;
    /** The bootstrap-CI lower bound on the median paired holdout delta
     *  must exceed this to promote. Default 0. */
    pairedDeltaThreshold?: number;
    /** Maximum allowed worsening of (search − holdout) gap relative to
     *  baseline. Default 0.15 (i.e. candidate may overfit by up to 15
     *  absolute score points more than baseline before rejection). */
    overfitGapThreshold?: number;
    /** Stable label of the baseline candidate. Required — paper-grade
     *  evaluation never compares two unlabelled candidates. */
    baselineKey: string;
    /** Confidence level for the bootstrap CI. Default 0.95. */
    confidence?: number;
    /** Bootstrap resamples. Default 2000. */
    bootstrapResamples?: number;
    /** Optional deterministic seed for the bootstrap. Default undefined
     *  (Math.random). */
    seed?: number;
    /**
     * Hard ceiling on the candidate's median per-task USD cost. When the
     * candidate clears the quality gates (paired-delta + overfit-gap) but
     * its median cost exceeds this number, the gate rejects with
     * `cost_ceiling`. Default `undefined` = no cost ceiling, behaving
     * exactly like the pre-cost gate.
     *
     * This exists because "we ship the better prompt" is only an honest
     * pitch when the better prompt also fits a customer-stated budget.
     * Cost is read from `RunRecord.costUsd` (already mandatory on every
     * run) so no new schema is required.
     */
    costPerTaskCeiling?: number;
}
export interface GateEvidence {
    /** Number of paired (candidate, baseline) holdout observations used. */
    productiveRuns: number;
    /** Median of (candidate − baseline) paired holdout deltas. */
    medianPairedDelta: number;
    /** Bootstrap CI on the median paired holdout delta. */
    pairedCI: {
        low: number;
        high: number;
    };
    /** Wilcoxon signed-rank p-value on the paired holdout deltas. */
    pairedPValue: number;
    /** Mean candidate score on the search split (NaN if none). */
    searchScore: number;
    /** Mean candidate score on the holdout split (NaN if none). */
    holdoutScore: number;
    /** Candidate (search − holdout) gap. */
    overfitGap: number;
    /** Baseline (search − holdout) gap. */
    baselineOverfitGap: number;
    /** Median per-task USD cost across the candidate's runs. Recorded
     *  even when no `costPerTaskCeiling` is configured so downstream
     *  dashboards (intelligence.tangle.tools) can render \$/task per
     *  generation regardless of gating policy. */
    medianCandidateCost: number;
    /** Median per-task USD cost across the baseline runs, for
     *  symmetric reporting. */
    medianBaselineCost: number;
}
export interface GateDecision {
    /** Final promote/no-promote verdict. */
    promote: boolean;
    /** The candidate that was evaluated. */
    candidateId: string;
    /** The baseline it was compared against. */
    baselineId: string;
    /** Every number the gate looked at, for audit + paper export. */
    evidence: GateEvidence;
    /** Human-readable reason. */
    reason: string;
    /** Machine-readable rejection code, or null on promote. */
    rejectionCode: HeldOutGateRejectionCode | null;
}
/**
 * Held-out paired-delta promotion gate. Construct once with config,
 * call `evaluate(candidateRuns, baselineRuns)` per (candidate,
 * baseline) pair. Stateless across calls.
 */
export declare class HeldOutGate {
    private readonly minProductiveRuns;
    private readonly pairedDeltaThreshold;
    private readonly overfitGapThreshold;
    private readonly baselineKey;
    private readonly confidence;
    private readonly resamples;
    private readonly seed?;
    private readonly costPerTaskCeiling?;
    constructor(config: HeldOutGateConfig);
    /** Decide whether `candidate` should replace `baseline`. Pairing
     *  is by (experimentId, seed) — identical experiment + seed pairs
     *  the candidate run with the matching baseline run. Pairs without
     *  a holdout score on both sides are dropped. */
    evaluate(candidate: RunRecord[], baseline: RunRecord[]): GateDecision;
}
//# sourceMappingURL=held-out-gate.d.ts.map