/**
 * `compareProposers` — a head-to-head lift benchmark across surface proposers
 * on ONE corpus. This is the forcing function: optimizer quality (GEPA
 * reflection vs GEPA+Pareto vs SkillOpt) becomes a NUMBER with a confidence
 * interval, so a proposer regression — or shipping a simplified proposer and
 * calling it the real one — turns a build red instead of going
 * measurement-invisible.
 *
 * Every entrant is scored the SAME way: each proposer returns the surface it
 * promoted, then the benchmark scores the baseline + every winner on the
 * SAME held-out scenarios with the SAME judges. Apples-to-apples by
 * construction — the comparison never depends on how a proposer measured itself.
 * The per-scenario held-out composites feed a paired bootstrap (`statistics.ts`)
 * for each proposer's lift CI and for the pairwise "which proposer wins" CI.
 */
import type { LlmClientOptions } from '../../llm-client';
import { type FapoProposerOptions, type ParameterCandidate } from '../proposers/fapo';
import { type RunCampaignOptions } from '../run-campaign';
import type { DispatchContext, JudgeConfig, MutableSurface, Scenario, SurfaceProposer } from '../types';
import { type RunImprovementLoopOptions } from './run-improvement-loop';
/** What an optimizer produced: the surface it promoted + what it cost to get
 *  there. The comparison does the held-out scoring itself, so an entry only
 *  needs to run its loop and hand back the winner. */
export interface ProposerEntry {
    name: string;
    optimize: () => Promise<{
        winnerSurface: MutableSurface;
        costUsd: number;
        durationMs?: number;
    }>;
}
export interface ProposerScore {
    name: string;
    /** Mean held-out composite of the baseline (identical across proposers). */
    baselineComposite: number;
    /** Mean held-out composite of this proposer's promoted surface. */
    winnerComposite: number;
    /** Mean per-scenario held-out lift (winner − baseline). */
    lift: number;
    /** Paired-bootstrap CI of the per-scenario lift. `low > 0` ⇒ a real gain. */
    liftCi: {
        low: number;
        high: number;
    };
    costUsd: number;
    durationMs?: number;
    winnerSurface: MutableSurface;
    /** 1-based, by descending lift. */
    rank: number;
}
export interface ProposerPairwise {
    /** Higher-ranked proposer. */
    a: string;
    b: string;
    /** Mean per-scenario held-out delta (a − b). */
    deltaMean: number;
    low: number;
    high: number;
    /** `a` if the CI clears 0, `b` if it is entirely negative, else `'tie'`. */
    favored: string;
}
export interface ProposerComparison {
    /** Sorted by descending lift; `rank` set accordingly. */
    scores: ProposerScore[];
    best: ProposerScore;
    /** Best vs each other proposer, paired-bootstrap on the held-out winners. */
    pairwise: ProposerPairwise[];
    holdoutScenarioIds: string[];
}
export interface CompareProposersOptions<TScenario extends Scenario, TArtifact> extends Omit<RunCampaignOptions<TScenario, TArtifact>, 'dispatch' | 'scenarios'> {
    proposers: ProposerEntry[];
    baselineSurface: MutableSurface;
    /** The held-out scenarios every winner is scored on. */
    holdoutScenarios: TScenario[];
    /** Scores a surface on a scenario — the same dispatcher the proposers used. */
    dispatchWithSurface: (surface: MutableSurface, scenario: TScenario, ctx: DispatchContext) => Promise<TArtifact>;
    /** Bootstrap resamples for the lift CIs. Default 2000. */
    resamples?: number;
    /** CI confidence. Default 0.95. */
    confidence?: number;
}
export declare function compareProposers<TScenario extends Scenario, TArtifact>(opts: CompareProposersOptions<TScenario, TArtifact>): Promise<ProposerComparison>;
/** Shared corpus + transport for the three built-in optimizer entries. */
export interface OptimizerEntryConfig<TScenario extends Scenario, TArtifact> {
    baselineSurface: string;
    /** Training scenarios the proposers reflect on. */
    trainScenarios: TScenario[];
    /** Held-out scenarios (the gate axis + the benchmark scoring axis). */
    holdoutScenarios: TScenario[];
    dispatchWithSurface: (surface: MutableSurface, scenario: TScenario, ctx: DispatchContext) => Promise<TArtifact>;
    judges: JudgeConfig<TArtifact, TScenario>[];
    llm: LlmClientOptions;
    model: string;
    target: string;
    runDir: string;
    seed?: number;
    /** GEPA population per generation. Default 2. */
    populationSize?: number;
    /** GEPA generations. Default 3. */
    maxGenerations?: number;
    /** SkillOpt epochs. Default 6. */
    maxEpochs?: number;
    mutationPrimitives?: string[];
    /** Static findings seed forwarded to each GEPA proposer's `propose()` as
     *  `ctx.findings`. Forwarded by `gepaReflectionEntry` / `gepaParetoEntry`;
     *  `skillOptEntry` runs without findings (see its doc). */
    findings?: unknown[];
    /** Per-generation findings producer: after each generation scores, this
     *  re-diagnoses and REPLACES `ctx.findings` for the
     *  next generation's `propose()`. Reuses the `runOptimization` field type so
     *  it cannot drift. GEPA entries only. */
    analyzeGeneration?: RunImprovementLoopOptions<TScenario, TArtifact>['analyzeGeneration'];
    /** Optional analysis report forwarded to `propose()` as `ctx.report`. */
    report?: unknown;
}
/** GEPA, reflection-only (single-parent, no Pareto combine). */
export declare function gepaReflectionEntry<TScenario extends Scenario, TArtifact>(config: OptimizerEntryConfig<TScenario, TArtifact>, name?: string): ProposerEntry;
/** GEPA with the Pareto frontier + combine-complementary-lessons. */
export declare function gepaParetoEntry<TScenario extends Scenario, TArtifact>(config: OptimizerEntryConfig<TScenario, TArtifact>, name?: string): ProposerEntry;
/** SkillOpt patch-mode hill-climb. Runs findings-BLIND: `runSkillOpt` owns its
 *  own epoch acceptance/budget loop and does not thread `analyzeGeneration`, so
 *  `config.findings` is intentionally NOT forwarded here. In a findings-fed
 *  comparison this entry is the blind control — do not read its result as
 *  findings-fed. (Threading findings into the SkillOpt epoch loop is a separate
 *  refactor, deferred not faked.) */
export declare function skillOptEntry<TScenario extends Scenario, TArtifact>(config: OptimizerEntryConfig<TScenario, TArtifact>, name?: string): ProposerEntry;
/** FAPO reviewed-escalation policy. This is an orchestration layer over
 * level-specific proposers, not a new mutation operator:
 * prompt -> parameter -> structural, with scope + reviewer + plateau rules in
 * `fapoProposer`. The prompt proposer defaults to GEPA+Pareto because that is
 * the package's strongest prompt-tier proposer; parameter/structural proposers
 * are opt-in so we do not fake code-generation inside agent-eval. */
export interface FapoEntryConfig<TScenario extends Scenario, TArtifact> extends OptimizerEntryConfig<TScenario, TArtifact> {
    /** Override the prompt-level proposer. Default: `gepaProposer({ combineParents: true })`. */
    promptProposer?: SurfaceProposer;
    /** Parameter/config-level proposer. If omitted, `parameterCandidates` builds one. */
    parameterProposer?: SurfaceProposer;
    /** Structural/code-level proposer, typically supplied by agent-runtime. */
    structuralProposer?: SurfaceProposer;
    /** Convenience: build a `parameterSweepProposer` from these candidates. */
    parameterCandidates?: readonly ParameterCandidate[];
    /** FAPO policy knobs: scope, reviewer, plateau thresholds. */
    fapo?: Omit<FapoProposerOptions, 'proposers' | 'promptProposer' | 'parameterProposer' | 'structuralProposer'>;
}
export declare function fapoEscalationEntry<TScenario extends Scenario, TArtifact>(config: FapoEntryConfig<TScenario, TArtifact>, name?: string): ProposerEntry;
//# sourceMappingURL=compare-proposers.d.ts.map