/**
 * Preference dataset extraction — bridge from `RunRecord[]` to RL training.
 *
 * Production RLHF / DPO / KTO / SimPO pipelines need preference triples:
 * `(prompt, chosen, rejected)`. The campaign artifact already contains the
 * ingredients — every (variantId, scenarioId, seed) cell is a candidate
 * that ran the same prompt against the same scenario, scored by the same
 * judge — but turning that into a clean preference dataset requires
 * deciding *what counts as a preference*.
 *
 * This module ships three preference-extraction strategies with explicit
 * tradeoffs, plus a unified output type compatible with HuggingFace TRL,
 * Anthropic finetuning JSONL, and OpenAI fine-tuning APIs. The strategies
 * are deliberately not auto-magical — picking the wrong one corrupts the
 * gradient.
 *
 * Strategies:
 *
 *   1. **`paired-by-scenario-and-seed`** — exact-match comparisons. For
 *      each scenario × seed pair, compare every (variantA, variantB) on
 *      that exact (scenario, seed). Matches scenarios so the comparison
 *      isolates variant effects. Highest signal-to-noise; smallest
 *      dataset (only matched pairs count).
 *
 *   2. **`paired-by-scenario`** — looser matching. For each scenario,
 *      compare every (variantA, variantB) where both have ≥ 1 run on the
 *      same scenario. Aggregates across seeds to compute mean scores per
 *      (variant, scenario), then forms preferences from the means. More
 *      data, lower per-pair signal.
 *
 *   3. **`top-vs-bottom`** — coarsest. Within each scenario, the highest-
 *      scoring run is `chosen`, the lowest is `rejected`. Smallest dataset
 *      per scenario but biggest score gap per pair. Useful for early
 *      bootstrapping when you have few variants.
 *
 * The output `PreferenceTriple` is *agent-eval-canonical* but trivially
 * mappable to TRL's `DPODataset` shape (`prompt`, `chosen`, `rejected`)
 * via the `toTRLFormat` helper.
 */
import type { RunRecord } from '../run-record';
export type PreferenceStrategy = 'paired-by-scenario-and-seed' | 'paired-by-scenario' | 'top-vs-bottom';
export interface PreferenceTriple {
    /** The scenario (input) the variants were run against. */
    scenarioId: string;
    /** RunRecord ids on each side, for traceability. */
    chosenRunId: string;
    rejectedRunId: string;
    /** Variant ids — load-bearing for the RL update. */
    chosenVariantId: string;
    rejectedVariantId: string;
    /** The score gap between chosen and rejected. Larger = stronger signal. */
    marginScore: number;
    /**
     * Optional `(chosen_score, rejected_score)` pair for soft-margin DPO
     * variants. Omitted for `top-vs-bottom` runs that don't carry meaningful
     * scalar gaps.
     */
    scores?: {
        chosen: number;
        rejected: number;
    };
    /** Tie-breaker — when multiple seeds match this scenario, the one used. */
    seed?: number;
    /**
     * Free-form metadata propagated from the run records — e.g. original
     * prompt-hash, model, etc. Lets the RL trainer reconstruct the prompt.
     */
    meta: {
        chosenPromptHash: string;
        rejectedPromptHash: string;
        chosenConfigHash: string;
        rejectedConfigHash: string;
        chosenModel: string;
        rejectedModel: string;
    };
}
export interface ExtractPreferencesOptions {
    strategy?: PreferenceStrategy;
    /**
     * Minimum score gap required to admit a pair. Pairs below this are
     * dropped — they're noise, not signal. Default 0.05 (5% of [0,1]).
     */
    minMargin?: number;
    /**
     * Optional split tag filter — restrict to runs from one split. Default
     * `'holdout'` (the canonical "real" signal).
     */
    splitTag?: RunRecord['splitTag'];
    /**
     * Optional reward extractor that overrides `outcome.holdoutScore` /
     * `outcome.searchScore`. Use to drive preferences off a verifiable
     * reward instead of the headline score.
     */
    rewardOf?: (run: RunRecord) => number | null;
}
export interface PreferenceExtractionReport {
    pairs: PreferenceTriple[];
    /** Number of (scenario, seed) cells inspected. */
    cellsInspected: number;
    /** Number of pairs filtered by `minMargin`. */
    pairsBelowMargin: number;
    /** Number of cells with only one variant (no comparison possible). */
    cellsSingleton: number;
    /** Strategy used. */
    strategy: PreferenceStrategy;
}
/**
 * Convert `RunRecord[]` to preference triples for RL training.
 *
 * Returns a structured report so callers can see how much data was
 * dropped and why (low-margin pairs, singleton cells). For production
 * pipelines, you usually want to:
 *
 *   1. Run a campaign producing 5–10 variants × 50–200 scenarios × 3 seeds
 *   2. Call this with `strategy: 'paired-by-scenario-and-seed'` and a
 *      verifiable-reward extractor as `rewardOf`
 *   3. Pass `report.pairs` to `toTRLFormat` and pipe to your DPO trainer
 */
export declare function extractPreferences(runs: RunRecord[], opts?: ExtractPreferencesOptions): PreferenceExtractionReport;
/**
 * TRL-compatible export. TRL's `DPODataset` is `{ prompt, chosen, rejected }`
 * but the prompt isn't stored on the RunRecord — only its hash. The caller
 * passes a `promptOf(promptHash)` lookup that the TRL trainer can use.
 */
export declare function toTRLFormat(triples: PreferenceTriple[], promptOf: (hash: string) => string): Array<{
    prompt: string;
    chosen: string;
    rejected: string;
}>;
/**
 * Anthropic finetuning JSONL export — `{ system, user, assistant_chosen, assistant_rejected }`
 * shape. Same caveat as TRL: prompt + outputs are content the caller has
 * to map back from the run record / raw event log.
 */
export declare function toAnthropicFormat(triples: PreferenceTriple[]): Array<{
    scenarioId: string;
    chosenRunId: string;
    rejectedRunId: string;
    margin: number;
}>;
//# sourceMappingURL=preferences.d.ts.map