/**
 * Contamination probe — held-out perturbation tests.
 *
 * The bug class: once a benchmark scenario set is published, models train
 * on it, and your scores become invalid. SWE-Bench-Verified, GPQA, and
 * MMLU-Pro all exist because their predecessors got contaminated within
 * months. The right defense is to keep a held-out *perturbed* version of
 * every scenario — same task, slightly different surface — and check
 * whether scores diverge significantly. Genuine capability transfers; rote
 * memorization doesn't.
 *
 * This module ships the probe contract:
 *
 *   1. A `ScenarioPerturbation` strategy type — function that produces a
 *      perturbed scenario from an original.
 *   2. `runContaminationProbe({ originals, perturbed, scoreFn })` — runs
 *      both halves and reports per-scenario score divergence + a global
 *      contamination verdict via paired Wilcoxon.
 *   3. Several stock perturbations: `renameVariables`, `shuffleOrder`,
 *      `paraphrasePrompt`, `injectIrrelevantClause`. Each preserves the
 *      task's structural difficulty while breaking surface memorization.
 *
 * The verdict is conservative: if the perturbed-vs-original score
 * difference is statistically significant (BH-adjusted p < 0.05) AND
 * the median drop is > 5 percentage points, we flag *contamination
 * suspected*. False positives are possible (the perturbation might
 * actually be harder); the default is to flag for review, not to
 * autoreject.
 */
export type ScenarioPerturbationKind = 'rename_variables' | 'shuffle_order' | 'paraphrase' | 'inject_irrelevant_clause' | 'custom';
export interface ScenarioPerturbation<S> {
    kind: ScenarioPerturbationKind;
    /** Apply to one scenario, return its perturbed sibling. */
    apply: (scenario: S) => Promise<S> | S;
    /** Optional id — for the report. */
    id?: string;
}
export interface ContaminationProbeInput<S> {
    /** Identity of every scenario. The probe's `runFingerprint` keys on these. */
    scenarioId: (s: S) => string;
    /** Original scenarios. */
    originals: S[];
    /**
     * Either pre-computed perturbations (one per original, same order) OR a
     * `perturbation` strategy that synthesizes them on the fly.
     */
    perturbed?: S[];
    perturbation?: ScenarioPerturbation<S>;
    /**
     * Run the policy/agent against one scenario and return a scalar score
     * in [0, 1]. The probe doesn't care what the policy is — that's the
     * caller's contract.
     */
    scoreFn: (s: S) => Promise<number>;
}
export interface ContaminationProbeOptions {
    /** Drop scores below this from the probe; treats partial failures separately. Default 0. */
    scoreFloor?: number;
    /**
     * BH-FDR threshold for declaring contamination on each per-scenario
     * delta. Default 0.05.
     */
    fdr?: number;
    /**
     * Minimum median per-scenario drop to flag global contamination. Default
     * 0.05 (5 percentage points). Smaller drops may be noise.
     */
    minMedianDrop?: number;
}
export interface ContaminationProbeReport {
    perScenario: Array<{
        scenarioId: string;
        originalScore: number;
        perturbedScore: number;
        delta: number;
        /** Per-scenario q-value (single-test BH for a single scenario). Mainly for display. */
        qValue: number;
    }>;
    /** Wilcoxon paired-test on the deltas. */
    pairedTest: {
        w: number;
        p: number;
    };
    medianDelta: number;
    meanDelta: number;
    contaminationSuspected: boolean;
    reason: string;
    /** Number of scenarios processed. */
    n: number;
}
export declare function runContaminationProbe<S>(input: ContaminationProbeInput<S>, opts?: ContaminationProbeOptions): Promise<ContaminationProbeReport>;
/**
 * Identifier-rename perturbation for code/text scenarios. Replaces every
 * occurrence of the listed identifiers with synthesized aliases. Use when
 * the scenario's structural difficulty is independent of variable names
 * (e.g. SWE-Bench-style coding tasks).
 */
export declare function renameVariables<S extends {
    prompt: string;
}>(identifiers: string[], rename?: (name: string, idx: number) => string): ScenarioPerturbation<S>;
/**
 * Order-shuffle perturbation. Reshuffles a list-shaped section of the
 * prompt (for QA scenarios that present options A/B/C/D — answer depends
 * on the option labels, not order). Caller provides the section extractor.
 */
export declare function shuffleOrder<S extends {
    prompt: string;
}>(shuffleSection: (prompt: string, rng: () => number) => string, seed: number): ScenarioPerturbation<S>;
/**
 * Inject-irrelevant-clause perturbation. Adds a benign sentence that
 * shouldn't change the answer. Tests for "did the model just memorize
 * the input string."
 */
export declare function injectIrrelevantClause<S extends {
    prompt: string;
}>(clause: string, position?: 'prefix' | 'suffix'): ScenarioPerturbation<S>;
//# sourceMappingURL=contamination.d.ts.map