import { T as TraceStore } from '../store-CKUAgsJz.js';
import { R as Run } from '../schema-m0gsnbt3.js';
import { a as OutcomeFilter, O as OutcomeStore } from '../outcome-store-D6KWmYvj.js';
export { D as DeploymentOutcome, F as FileSystemOutcomeStore, b as FileSystemOutcomeStoreOptions, I as InMemoryOutcomeStore } from '../outcome-store-D6KWmYvj.js';
export { R as RubricOutcomePair, a as RubricPredictiveValidityInput, b as RubricPredictiveValidityReport, c as RubricRanking, r as rubricPredictiveValidity } from '../rubric-predictive-validity-D_4BSXGV.js';
import '../run-record-BgTFzO2r.js';
import '../errors-Dwqw-T_m.js';

/**
 * Correlation study — "does our eval score predict real-world outcomes?"
 *
 * This is the load-bearing signal. Takes a TraceStore + OutcomeStore,
 * joins on runId, computes Pearson + Spearman + bootstrap CI for every
 * (evalMetric, outcomeMetric) pair the caller declares.
 *
 * Without this number the framework is ornamental. With it and r > 0.6
 * the framework is a moat — no other agent-eval tool publishes one.
 */

interface EvalMetricSpec {
    id: string;
    /** Extract a scalar from a run (defaults cover score/pass/durationMs/costUsd/tokens). */
    extract?: (run: Run, store: TraceStore) => Promise<number | null>;
}
interface OutcomePair {
    evalMetric: string;
    outcomeMetric: string;
}
interface CorrelationResult {
    evalMetric: string;
    outcomeMetric: string;
    n: number;
    pearson: number;
    spearman: number;
    /** 95% bootstrap CI for Pearson. */
    pearsonCi95: {
        lower: number;
        upper: number;
    };
    /** Rough verdict: 'strong' ≥ 0.7, 'moderate' ≥ 0.4, else 'weak'. */
    verdict: 'strong' | 'moderate' | 'weak';
}
interface CorrelationStudyResult {
    pairs: CorrelationResult[];
    joinedSamples: number;
    skippedRuns: number;
}
interface CorrelationStudyOptions {
    /** Only join outcomes captured within this window after run.startedAt. */
    maxCaptureLagMs?: number;
    /** Restrict to a subset of outcomes (cohort, region, source). */
    outcomeFilter?: OutcomeFilter;
    /** Which outcome per run to use when multiple exist. Default 'latest'. */
    reduction?: 'latest' | 'mean' | 'max';
    /** Bootstrap iterations for the CI. Default 500. */
    bootstrapIterations?: number;
}
declare function correlationStudy(traceStore: TraceStore, outcomeStore: OutcomeStore, evalMetrics: EvalMetricSpec[], outcomeMetricNames: string[], options?: CorrelationStudyOptions): Promise<CorrelationStudyResult>;

/**
 * Calibration curve — binned "if eval says X, what does reality show?"
 *
 * Companion to correlationStudy. Raw correlation is a single number;
 * the calibration curve shows *where* the eval is well-calibrated vs
 * overconfident / underconfident. Buckets the eval metric, computes
 * mean outcome per bucket, reports expected-calibration-error (ECE).
 */

interface CalibrationBin {
    lower: number;
    upper: number;
    n: number;
    evalMean: number;
    outcomeMean: number;
    /** |outcomeMean − evalMean|; contributes to ECE weighted by n/total. */
    gap: number;
}
interface CalibrationReport {
    evalMetric: string;
    outcomeMetric: string;
    n: number;
    bins: CalibrationBin[];
    /** Expected Calibration Error — Σ (n_i/N) × |outcomeMean_i − evalMean_i|. */
    ece: number;
    /** Max bin gap — upper bound on miscalibration. */
    maxGap: number;
}
interface CalibrationOptions {
    bins?: number;
    /** Equal-width (fixed bin edges) or equal-frequency (quantile bins). */
    binning?: 'equal-width' | 'equal-frequency';
    /** Clip eval values to [lo, hi] before binning. */
    range?: {
        lo: number;
        hi: number;
    };
}
declare function calibrationCurve(traceStore: TraceStore, outcomeStore: OutcomeStore, evalMetric: EvalMetricSpec, outcomeMetric: string, options?: CalibrationOptions): Promise<CalibrationReport | null>;

export { type CalibrationBin, type CalibrationOptions, type CalibrationReport, type CorrelationResult, type CorrelationStudyOptions, type CorrelationStudyResult, type EvalMetricSpec, OutcomeFilter, type OutcomePair, OutcomeStore, calibrationCurve, correlationStudy };
