/**
 * Judge calibration — measure judge quality against human gold + bias.
 *
 * Workflow:
 *   1. Build a golden set: {itemId, humanScore}[].
 *   2. Run candidate judges; each produces {itemId, score}.
 *   3. `calibrateJudge(golden, candidate)` reports κ + Pearson + MAE.
 *   4. `calibrateJudgeContinuous(golden, candidate)` adds quadratic-weighted
 *      κ over the un-rounded [0,1] scores plus ICC(2,1), Pearson, Spearman,
 *      and bootstrap CIs — use this for fine-grained judges where rounding
 *      to int discards information (e.g. 0.78 vs 0.81 both round to 1 and
 *      look "perfectly agreed" to integer κ).
 *   5. Run bias probes (positional, verbosity, self-preference) to
 *      detect systematic score inflation.
 *   6. For N≥2 judges on the same items, `continuousAgreement(scores)`
 *      reports ICC(2,1) + κ_w + Pearson + Spearman with bootstrap CIs.
 *
 * Returns actionable diagnostics, not a single number. Consumers then
 * decide whether to trust the judge, retrain it, or add a tie-breaker.
 */
export interface GoldenItem {
    itemId: string;
    humanScore: number;
    /** Optional group used for per-group bias audits (e.g. model-of-output family). */
    group?: string;
}
export interface CandidateScore {
    itemId: string;
    score: number;
    /** Optional — enables positional-bias analysis (did order matter?). */
    positionOfAInput?: 'first' | 'second';
}
export interface CalibrationResult {
    n: number;
    pearson: number;
    /** Cohen's κ with quadratic weights over integer-rounded scores. */
    kappa: number;
    /** Mean absolute error vs human. */
    mae: number;
    /** Worst-5 miscalibrations (largest |judge - human|). */
    worstItems: Array<{
        itemId: string;
        judge: number;
        human: number;
        delta: number;
    }>;
}
export declare function calibrateJudge(golden: GoldenItem[], candidate: CandidateScore[]): CalibrationResult;
export interface PositionalBiasResult {
    /**
     * Score delta (first-position - second-position) averaged across items
     * presented in both positions. Non-zero = positional bias.
     */
    avgDelta: number;
    n: number;
}
/**
 * Feed the same items to the judge twice with A/B swapped and pass all
 * results here. Items that don't appear in both positions are ignored.
 */
export declare function positionalBias(scores: CandidateScore[]): PositionalBiasResult;
export interface VerbosityBiasResult {
    /** Pearson correlation between output length and score. Strong positive = verbosity bias. */
    pearson: number;
    n: number;
}
export declare function verbosityBias(samples: Array<{
    outputLen: number;
    score: number;
}>): VerbosityBiasResult;
export interface SelfPreferenceResult {
    /** Mean judge score when judge's family matches output's family. */
    inFamilyMean: number;
    outOfFamilyMean: number;
    deltaMean: number;
    n: number;
}
/**
 * Pass the same scenarios scored with judge-model X grading outputs from
 * model X (in-family) and model Y (out-of-family). Non-zero delta
 * indicates self-preference.
 */
export declare function selfPreference(samples: Array<{
    score: number;
    inFamily: boolean;
}>): SelfPreferenceResult;
export interface ContinuousAgreement {
    /** Cohen's κ_w with quadratic weights, computed on raw [0,1] scores. */
    weightedKappa: number;
    /** ICC(2,1): two-way random effects, absolute agreement, single rater. */
    icc: number;
    /** Pearson product-moment correlation (averaged over rater pairs if N>2). */
    pearson: number;
    /** Spearman rank correlation (averaged over rater pairs if N>2). */
    spearman: number;
    /** 95% bootstrap percentile CIs over items. */
    ci: {
        icc: [number, number];
        weightedKappa: [number, number];
    };
    /** Number of complete items (no NaN across raters). */
    n: number;
    /** Number of raters. */
    raters: number;
}
export interface ContinuousAgreementOptions {
    /** Bootstrap iterations. Default 1000. Set to 0 to skip CIs (CI = [NaN, NaN]). */
    bootstrap?: number;
    /** κ weighting scheme. Default 'quadratic'. */
    weights?: 'linear' | 'quadratic';
    /** PRNG seed for reproducible bootstrap. Default 0xC0FFEE. */
    seed?: number;
    /** Confidence level for percentile CI. Default 0.95. */
    ciLevel?: number;
}
/**
 * Inter-rater agreement on continuous (typically [0,1]) scores.
 *
 * `scores` has shape [n_items][n_raters]. Rows with any non-finite entry
 * are dropped. Returns NaN metrics if fewer than 2 raters or 2 complete
 * items remain.
 */
export declare function continuousAgreement(scores: number[][], opts?: ContinuousAgreementOptions): ContinuousAgreement;
export interface ContinuousCalibrationResult extends CalibrationResult {
    /** Cohen's κ_w computed on raw (un-rounded) scores. */
    weightedKappaContinuous: number;
    /** ICC(2,1) treating golden + candidate as two raters. */
    icc: number;
    spearman: number;
    ci: {
        icc: [number, number];
        weightedKappa: [number, number];
    };
}
/**
 * Drop-in superset of `calibrateJudge` that adds continuous-value
 * agreement metrics. The old fields (n, pearson, kappa, mae, worstItems)
 * are preserved unchanged so existing callers continue to work.
 */
export declare function calibrateJudgeContinuous(golden: GoldenItem[], candidate: CandidateScore[], opts?: ContinuousAgreementOptions): ContinuousCalibrationResult;
//# sourceMappingURL=judge-calibration.d.ts.map