/**
 * Reward hacking / Goodhart detection.
 *
 * Goodhart's Law says: when a measure becomes a target, it ceases to be
 * a good measure. In RLHF and agentic-RL settings this is the dominant
 * failure mode — the policy learns to produce outputs that score well on
 * the proxy reward (judge, rubric, test pass-rate) without producing
 * the underlying capability the proxy was meant to track.
 *
 * Krakovna et al. (2020, "Specification Gaming Examples in AI") and the
 * subsequent RLHF reward-hacking literature (Skalse et al. 2022, Kim et al.
 * 2023) converge on a few diagnostic signatures:
 *
 *   1. **Reward divergence:** the proxy reward grows while the held-out
 *      ground-truth signal stagnates or drops. Predictive validity over
 *      time captures this.
 *   2. **Distributional shift in outputs:** after RL, the policy produces
 *      outputs that no longer match the reference distribution — usually
 *      because it found a high-reward attractor that's degenerate (e.g.
 *      one-token responses, repetition, formatting tricks).
 *   3. **Disagreement between independent rewards:** if you train on
 *      reward A and a held-out independent reward B drops sharply, you're
 *      probably hacking A.
 *   4. **Calibration drift:** the verifiable / deterministic component of
 *      the reward is stable; the probabilistic / judge component drifts up
 *      while the deterministic component doesn't. The judge is being
 *      gamed.
 *
 * This module ships explicit detectors for all four signatures, plus a
 * combined verdict. The output is diagnostic — actionable signals,
 * not autoreject — because each signature has known false positives
 * (e.g., a policy that genuinely improves can show distributional shift).
 *
 * Differs from `rubricPredictiveValidity` (which is a *standing* check on
 * whether rubrics correlate with deployment outcomes) — this is a
 * *temporal* check on whether the reward-vs-truth gap is *widening over
 * time during a training run*.
 */
import type { RunRecord } from '../run-record';
import { type VerifiableRewardExtractionOptions } from './verifiable-reward';
export type RewardHackingSignal = 'reward_divergence' | 'distribution_shift' | 'reward_disagreement' | 'judge_drift';
export interface RewardHackingFinding {
    signal: RewardHackingSignal;
    /** Severity in [0, 1]. >0.5 = strong signal. */
    severity: number;
    message: string;
    /** Numeric evidence the consumer can render. */
    detail: Record<string, number>;
}
export interface RewardHackingReport {
    findings: RewardHackingFinding[];
    /**
     * Composite verdict. `'clean'` if every signal severity < 0.3;
     * `'suspect'` if at least one ≥ 0.3 but none ≥ 0.6; `'gaming'` if any ≥ 0.6.
     */
    verdict: 'clean' | 'suspect' | 'gaming';
    /** Rationale for the verdict, ready to paste into an audit log. */
    rationale: string[];
    /** Number of paired (proxy, truth) data points the report saw. */
    n: number;
}
export interface DetectRewardHackingInput {
    /**
     * Run records ordered by recency (oldest first). The detector segments
     * them into prefix/suffix windows to compute "did the gap widen."
     */
    runs: RunRecord[];
    /**
     * The metric the policy was trained to optimize. Should be present on
     * `outcome.raw` or `outcome.holdoutScore`. Default reads `outcome.holdoutScore`.
     */
    proxyOf?: (run: RunRecord) => number | null;
    /**
     * The held-out ground-truth metric. For RL on coding, this is typically
     * test pass-rate. For RLHF, it's downstream task performance or human
     * preference. For knowledge tasks, it's an independently-graded score.
     */
    truthOf?: (run: RunRecord) => number | null;
    /**
     * Independent secondary reward. Used for the `reward_disagreement`
     * signal. Default uses the verifiable reward extractor (deterministic
     * sources only).
     */
    secondaryRewardOf?: (run: RunRecord) => number | null;
    /**
     * Window size — how many of the most recent runs count as the "after"
     * cohort. Default min(50, half the runs).
     */
    windowSize?: number;
    /**
     * Severity threshold to flag a signal. Default 0.3 (suspect) and 0.6
     * (gaming).
     */
    thresholds?: {
        suspect?: number;
        gaming?: number;
    };
    /**
     * Verifiable-reward options used for the secondary-reward fallback.
     */
    verifiableRewardOptions?: VerifiableRewardExtractionOptions;
}
export declare function detectRewardHacking(input: DetectRewardHackingInput): RewardHackingReport;
//# sourceMappingURL=reward-hacking.d.ts.map