/**
 * Sample-efficient adaptation evaluation.
 *
 * For foundation-model-based agents, the load-bearing capability isn't
 * raw end-state performance — it's *how fast the agent reaches that
 * performance from cold start*. The same model with a worse prompt that
 * adapts in 5 demonstrations beats the same model with a better prompt
 * that needs 50. Standard meta-learning eval (Finn et al., MAML, RL² lit)
 * reports an *adaptation curve*: score after k=0, 1, 2, 4, 8, 16, …
 * in-context examples or fine-tune steps.
 *
 * This module ships:
 *
 *   1. `runAdaptationCurve` — given a runner that takes k demonstrations
 *      and returns a score, produce the (k, score) curve.
 *   2. `compareAdaptationCurves` — paired comparison across two policies.
 *      Returns per-k delta with bootstrap CIs and an "area-under-curve"
 *      summary statistic.
 *   3. `firstPassK` — for pass/fail evaluation, the minimum k at which
 *      the policy reliably passes (≥ pass-rate threshold over reps).
 *
 * Use cases:
 *   - Compare two prompt designs that have similar end-state performance
 *     but different in-context efficiency.
 *   - Decide between fine-tuning and prompting based on adaptation cost.
 *   - Detect when a policy "memorizes" k=0 inputs vs. genuinely adapts.
 */
export interface AdaptationRunner<S> {
    /**
     * Runs the policy on `scenario` with `k` demonstrations. Returns a
     * scalar score in [0, 1]. The runner is responsible for any caching;
     * the harness calls it once per (scenario, k, rep) cell.
     */
    run(args: {
        scenario: S;
        k: number;
        rep: number;
    }): Promise<number>;
}
export interface RunAdaptationCurveOptions<S> {
    scenarios: S[];
    /** Number-of-shots to evaluate at. Default `[0, 1, 2, 4, 8, 16]`. */
    ks?: number[];
    /** Reps per (scenario, k) cell. Default 3. */
    reps?: number;
    runner: AdaptationRunner<S>;
    /** Pass-rate threshold for `firstPassK` reporting. Default 0.5. */
    passThreshold?: number;
}
export interface AdaptationPoint {
    k: number;
    meanScore: number;
    passRate: number;
    std: number;
    n: number;
    /** Per-scenario means at this k. */
    perScenario: Array<{
        scenarioId: string;
        meanScore: number;
        passes: number;
        total: number;
    }>;
}
export interface AdaptationCurve {
    points: AdaptationPoint[];
    /**
     * Smallest `k` at which `passRate ≥ passThreshold`. `null` if no `k`
     * tested reaches it.
     */
    firstPassK: number | null;
    /**
     * Area under the (k, meanScore) curve, normalized by max-k. A
     * single-number summary of "how well does this policy adapt from
     * cold-start to fully-conditioned." Higher = better adapter.
     */
    adaptationArea: number;
}
export declare function runAdaptationCurve<S extends {
    scenarioId?: string;
}>(opts: RunAdaptationCurveOptions<S>): Promise<AdaptationCurve>;
export interface CompareCurvesResult {
    perK: Array<{
        k: number;
        deltaMean: number;
        aLow: number;
        aHigh: number;
        bLow: number;
        bHigh: number;
    }>;
    areaDelta: number;
    firstPassKDelta: number | null;
    /** Verdict: 'a_better' | 'b_better' | 'similar'. */
    verdict: 'a_better' | 'b_better' | 'similar';
    /** Rationale, ready to render. */
    rationale: string;
}
/**
 * Paired comparison of two adaptation curves. Per-k deltas with 95%
 * bootstrap CIs (constructed from each curve's `perScenario` per-k means
 * — the bootstrap unit is the scenario, not the rep).
 */
export declare function compareAdaptationCurves(a: AdaptationCurve, b: AdaptationCurve, opts?: {
    confidence?: number;
    bootstrapResamples?: number;
    seed?: number;
}): CompareCurvesResult;
/** First k at which the curve's per-scenario pass rate reliably hits the threshold. */
export declare function firstPassK(curve: AdaptationCurve, threshold?: number): number | null;
//# sourceMappingURL=adaptation-eval.d.ts.map