/**
 * Reference replay — score an agent against withheld historical outcomes.
 *
 * This is the generic version of the public-audit replay pattern:
 * run a candidate system on an old task, keep the reference answers hidden
 * until after execution, then score recall/precision and gate promotion
 * across train/dev/test/holdout splits.
 */
export type ReferenceReplaySplit = 'train' | 'dev' | 'test' | 'holdout';
export type ReferenceReplayMatchStrategy = 'reference-order' | 'global-greedy';
export interface ReferenceReplayItem {
    id: string;
    title: string;
    description?: string;
    severity?: string;
    tags?: string[];
    weight?: number;
}
export interface ReferenceReplayCandidate {
    id: string;
    title: string;
    description?: string;
    severity?: string;
    tags?: string[];
    metadata?: Record<string, unknown>;
}
export interface ReferenceReplayScenario {
    id: string;
    split?: ReferenceReplaySplit;
    references: ReferenceReplayItem[];
    candidates: ReferenceReplayCandidate[];
    metadata?: Record<string, unknown>;
}
export interface ReferenceReplayCase<Input = unknown> {
    id: string;
    split?: ReferenceReplaySplit;
    input: Input;
    references: ReferenceReplayItem[];
    metadata?: Record<string, unknown>;
}
export interface ReferenceReplayExecutionScenario<Input = unknown> {
    id: string;
    split: ReferenceReplaySplit;
    input: Input;
    metadata?: Record<string, unknown>;
}
export interface ReferenceReplayRunContext {
    runId: string;
    caseIndex: number;
    abortSignal?: AbortSignal;
}
export interface ReferenceReplayAdapter<Input = unknown> {
    run(scenario: ReferenceReplayExecutionScenario<Input>, context: ReferenceReplayRunContext): Promise<ReferenceReplayCandidate[]>;
}
export type ReferenceReplayAdapterFn<Input = unknown> = (scenario: ReferenceReplayExecutionScenario<Input>, context: ReferenceReplayRunContext) => Promise<ReferenceReplayCandidate[]>;
export type ReferenceReplayAdapterLike<Input = unknown> = ReferenceReplayAdapter<Input> | ReferenceReplayAdapterFn<Input>;
export interface ReferenceReplayMatch {
    scenarioId: string;
    referenceId: string;
    candidateId: string | null;
    score: number;
    matched: boolean;
    weight: number;
    reason: string;
}
export interface ReferenceReplayScenarioScore {
    scenarioId: string;
    split: ReferenceReplaySplit;
    matched: number;
    total: number;
    falsePositives: number;
    matchedWeight: number;
    totalWeight: number;
    precision: number;
    recall: number;
    f1: number;
    matches: ReferenceReplayMatch[];
}
export interface ReferenceReplayAggregate {
    matched: number;
    total: number;
    falsePositives: number;
    matchedWeight: number;
    totalWeight: number;
    precision: number;
    recall: number;
    f1: number;
    weightedRecall: number;
}
export interface ReferenceReplayScore {
    scenarios: ReferenceReplayScenarioScore[];
    aggregate: ReferenceReplayAggregate;
    bySplit: Partial<Record<ReferenceReplaySplit, ReferenceReplayAggregate>>;
}
export interface ReferenceMatchResult {
    score: number;
    reason?: string;
}
export type ReferenceReplayMatcher = (reference: ReferenceReplayItem, candidate: ReferenceReplayCandidate, scenario: ReferenceReplayScenario) => ReferenceMatchResult;
export interface ReferenceReplayScoreOptions {
    matcher?: ReferenceReplayMatcher;
    matchThreshold?: number;
    matchStrategy?: ReferenceReplayMatchStrategy;
    includeHoldout?: boolean;
    splits?: ReferenceReplaySplit[];
}
export interface ReferenceReplayPromotionPolicy {
    /** Splits that must improve or stay flat. Default: ['dev', 'test']. */
    requiredSplits?: ReferenceReplaySplit[];
    /** Minimum aggregate F1 lift required on required splits. Default 0. */
    minF1Delta?: number;
    /** Maximum F1 drop allowed on any compared split. Default 0. */
    maxRegression?: number;
    /** If true, holdout must be present and must not regress. Default true. */
    requireHoldoutNonRegression?: boolean;
}
export interface ReferenceReplaySplitComparison {
    split: ReferenceReplaySplit;
    baselineF1: number;
    candidateF1: number;
    f1Delta: number;
    baselineRecall: number;
    candidateRecall: number;
    recallDelta: number;
}
export interface ReferenceReplayPromotionDecision {
    promote: boolean;
    reason: string;
    aggregateDelta: number;
    comparisons: ReferenceReplaySplitComparison[];
    regressions: ReferenceReplaySplitComparison[];
}
export interface ReferenceReplayCaseRun<Input = unknown> {
    caseId: string;
    split: ReferenceReplaySplit;
    input: Input;
    metadata?: Record<string, unknown>;
    references: ReferenceReplayItem[];
    candidates: ReferenceReplayCandidate[];
    score: ReferenceReplayScenarioScore;
    durationMs: number;
    error?: string;
}
export interface ReferenceReplayRun<Input = unknown> {
    id: string;
    variantId?: string;
    startedAt: number;
    completedAt: number;
    durationMs: number;
    cases: ReferenceReplayCaseRun<Input>[];
    score: ReferenceReplayScore;
    metadata?: Record<string, unknown>;
}
export interface ReferenceReplayRunOptions<Input = unknown> extends ReferenceReplayScoreOptions {
    adapter: ReferenceReplayAdapterLike<Input>;
    runId?: string;
    variantId?: string;
    metadata?: Record<string, unknown>;
    store?: ReferenceReplayRunStore<Input>;
    abortSignal?: AbortSignal;
    continueOnError?: boolean;
    now?: () => number;
}
export interface ReferenceReplayRunStore<Input = unknown> {
    save(run: ReferenceReplayRun<Input>): Promise<void>;
    list(): Promise<ReferenceReplayRun<Input>[]>;
}
export declare function runReferenceReplay<Input = unknown>(cases: ReferenceReplayCase<Input>[], options: ReferenceReplayRunOptions<Input>): Promise<ReferenceReplayRun<Input>>;
export declare function decideReferenceReplayRunPromotion(baseline: ReferenceReplayRun, candidate: ReferenceReplayRun, policy?: ReferenceReplayPromotionPolicy): ReferenceReplayPromotionDecision;
export declare function inMemoryReferenceReplayStore<Input = unknown>(initial?: ReferenceReplayRun<Input>[]): ReferenceReplayRunStore<Input>;
export declare function jsonlReferenceReplayStore<Input = unknown>(path: string): ReferenceReplayRunStore<Input>;
export declare function scoreReferenceReplay(scenarios: ReferenceReplayScenario[], options?: ReferenceReplayScoreOptions): ReferenceReplayScore;
export declare function compareReferenceReplay(baseline: ReferenceReplayScore, candidate: ReferenceReplayScore): ReferenceReplaySplitComparison[];
export declare function decideReferenceReplayPromotion(baseline: ReferenceReplayScore, candidate: ReferenceReplayScore, policy?: ReferenceReplayPromotionPolicy): ReferenceReplayPromotionDecision;
export declare function defaultReferenceReplayMatcher(reference: ReferenceReplayItem, candidate: ReferenceReplayCandidate): ReferenceMatchResult;
//# sourceMappingURL=reference-replay.d.ts.map