/**
 * pipeline/compare.ts
 *
 * Core comparison primitive for the evaluation framework.
 *
 * Takes two ScoreSummary objects (baseline and experiment) and produces a
 * structured ComparisonReport with overall, per-area, and per-dimension
 * deltas, plus improved/regressed/unchanged classification.
 *
 * This is the single function that backs all comparison scenarios:
 * doc improvement, model comparison, branch validation, etc.
 * What varies is what produced each ScoreSummary — the comparison
 * logic is always the same.
 *
 * @see docs/ideas/evaluation-roadmap.md — BP5: Make comparison a primitive
 * @see docs/ideas/metrics-design.md — Tier 4: Comparison results
 */
import { DEFAULT_NOISE_THRESHOLD, } from "./types.js";
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
/** Classify a delta as improved, regressed, or unchanged given a threshold */
export function classifyChange(delta, threshold) {
    if (delta > threshold)
        return "improved";
    if (delta < -threshold)
        return "regressed";
    return "unchanged";
}
/**
 * Compare two evaluation score summaries and produce a structured report.
 *
 * This is a pure function — no side effects, no file I/O.
 *
 * @param baseline  The "before" or "control" score summary
 * @param experiment  The "after" or "treatment" score summary
 * @param options  Optional configuration (noise threshold, etc.)
 * @returns A ComparisonReport with deltas, classifications, and breakdowns
 */
export function compare(baseline, experiment, options) {
    // Use grader consistency data for empirical threshold when available
    const graderConsistency = options?.graderConsistency;
    const empirical = graderConsistency !== undefined;
    const threshold = empirical
        ? graderConsistency.recommendedThreshold
        : (options?.noiseThreshold ?? DEFAULT_NOISE_THRESHOLD);
    // Collect all unique area names from both summaries
    const baselineAreas = new Set(baseline.scores.map((s) => s.feature));
    const experimentAreas = new Set(experiment.scores.map((s) => s.feature));
    const allAreas = new Set([...baselineAreas, ...experimentAreas]);
    // Identify mismatched areas
    const onlyInBaseline = [...baselineAreas].filter((a) => !experimentAreas.has(a));
    const onlyInExperiment = [...experimentAreas].filter((a) => !baselineAreas.has(a));
    // Build per-area deltas.
    // Areas present in only one summary get change: "not-evaluated" — comparing
    // against a missing score is meaningless (it would produce false ±100 deltas).
    const mismatchedSet = new Set([...onlyInBaseline, ...onlyInExperiment]);
    const areas = [...allAreas]
        .sort()
        .map((area) => buildAreaDelta(area, findScore(baseline.scores, area), findScore(experiment.scores, area), threshold, mismatchedSet.has(area)));
    // Classify areas — mismatched areas are excluded from all three buckets
    const improved = areas
        .filter((a) => a.change === "improved")
        .map((a) => a.area);
    const regressed = areas
        .filter((a) => a.change === "regressed")
        .map((a) => a.area);
    const unchanged = areas
        .filter((a) => a.change === "unchanged")
        .map((a) => a.area);
    const notEvaluated = areas
        .filter((a) => a.change === "not-evaluated")
        .map((a) => a.area);
    // Per-area deltas as an array (W0137 / D0041) — keyed-map shapes are
    // capped by the Sanity attribute limit because each new feature area
    // mints a permanent attribute path.
    const perArea = areas.map((a) => ({
        area: a.area,
        delta: a.delta,
    }));
    // Per-dimension average deltas (only for areas present in both summaries)
    const commonAreas = areas.filter((a) => baselineAreas.has(a.area) && experimentAreas.has(a.area));
    const commonCount = commonAreas.length || 1;
    // Collect all dimension keys from area deltas and average each
    const allDimKeys = new Set(commonAreas.flatMap((a) => Object.keys(a.dimensions)));
    const perDimension = {};
    for (const dim of allDimKeys) {
        perDimension[dim] =
            commonAreas.reduce((s, a) => s + (a.dimensions[dim]?.delta ?? 0), 0) /
                commonCount;
    }
    // Doc Lift average delta (common areas only)
    const docLift = commonAreas.reduce((s, a) => s + a.docLiftDelta, 0) / commonCount;
    // Cost delta (if both summaries have cost data)
    const baselineCost = baseline.overall.cost?.total;
    const experimentCost = experiment.overall.cost?.total;
    const costDelta = baselineCost !== undefined && experimentCost !== undefined
        ? experimentCost - baselineCost
        : undefined;
    // Overall score delta
    const overallDelta = experiment.overall.avgScore - baseline.overall.avgScore;
    // Per-model deltas (when both summaries have per-model data)
    // Guard: older reports may have perModel as a Record instead of an array
    let perModelDeltas;
    if (Array.isArray(baseline.perModel) && Array.isArray(experiment.perModel)) {
        const allModelIds = new Set([
            ...baseline.perModel.map((m) => m.modelId),
            ...experiment.perModel.map((m) => m.modelId),
        ]);
        perModelDeltas = [];
        for (const modelId of allModelIds) {
            const bScore = baseline.perModel.find((m) => m.modelId === modelId)?.overall
                .avgScore ?? 0;
            const eScore = experiment.perModel.find((m) => m.modelId === modelId)?.overall
                .avgScore ?? 0;
            perModelDeltas.push({ delta: eScore - bScore, modelId });
        }
    }
    return {
        areas,
        baseline,
        deltas: {
            docLift,
            overall: overallDelta,
            perArea,
            perDimension,
            ...(costDelta !== undefined && { cost: costDelta }),
            ...(perModelDeltas && { perModel: perModelDeltas }),
        },
        experiment,
        generatedAt: new Date().toISOString(),
        improved,
        mismatched: {
            onlyInBaseline,
            onlyInExperiment,
        },
        noiseThreshold: threshold,
        noiseThresholdEmpirical: empirical,
        notEvaluated,
        regressed,
        unchanged,
    };
}
/** Build an AreaDelta from baseline and experiment scores for a single area */
function buildAreaDelta(area, baselineScore, experimentScore, threshold, isMismatched = false) {
    const b = baselineScore;
    const e = experimentScore;
    const bTotal = b?.totalScore ?? 0;
    const eTotal = e?.totalScore ?? 0;
    const delta = eTotal - bTotal;
    // Support both new field names and legacy data (old baselines/Sanity docs)
    const bRaw = b;
    const eRaw = e;
    const bLift = b?.docLift ?? bRaw?.["liftFromDocs"] ?? 0;
    const eLift = e?.docLift ?? eRaw?.["liftFromDocs"] ?? 0;
    const bCeiling = b?.ceilingScore ?? bRaw?.["withDocsScore"] ?? 0;
    const eCeiling = e?.ceilingScore ?? eRaw?.["withDocsScore"] ?? 0;
    const bFloor = b?.floorScore ?? bRaw?.["withoutDocsScore"] ?? 0;
    const eFloor = e?.floorScore ?? eRaw?.["withoutDocsScore"] ?? 0;
    const bCost = b?.totalCost ?? 0;
    const eCost = e?.totalCost ?? 0;
    const hasCost = bCost > 0 || eCost > 0;
    // Actual (agentic) score deltas — only when both runs have actual data
    const bActual = b?.actualScore;
    const eActual = e?.actualScore;
    const hasActual = bActual !== undefined && eActual !== undefined;
    // Retrieval gap deltas — only when both runs have retrieval gap data
    const bRetGap = b?.retrievalGap;
    const eRetGap = e?.retrievalGap;
    const hasRetGap = bRetGap !== undefined && eRetGap !== undefined;
    // Infrastructure efficiency deltas
    const bInfra = b?.infrastructureEfficiency;
    const eInfra = e?.infrastructureEfficiency;
    const hasInfra = bInfra != null && eInfra != null;
    return {
        ...(hasActual && { actualDelta: eActual - bActual }),
        area,
        baseline: bTotal,
        ceilingDelta: eCeiling - bCeiling,
        change: isMismatched ? "not-evaluated" : classifyChange(delta, threshold),
        delta,
        dimensions: buildDimensionDeltas(b, e),
        docLiftDelta: eLift - bLift,
        experiment: eTotal,
        floorDelta: eFloor - bFloor,
        ...(hasInfra && {
            infrastructureEfficiencyDelta: eInfra - bInfra,
        }),
        ...(hasRetGap && { retrievalGapDelta: eRetGap - bRetGap }),
        ...(hasCost && { costDelta: eCost - bCost }),
    };
}
/**
 * Build per-dimension deltas from the generic dimensions map when available,
 * falling back to the three legacy literacy fields for backward compatibility.
 *
 * This ensures non-literacy modes (agent-harness, mcp-server, etc.) get their
 * actual dimensions (e.g., agentOutput, toolUsage) in comparison reports
 * instead of hardcoded zeros for codeCorrectness/docCoverage/taskCompletion.
 */
function buildDimensionDeltas(b, e) {
    const bDims = b?.dimensions;
    const eDims = e?.dimensions;
    // When the generic dimensions map is populated, use it — this covers
    // agent-harness (agentOutput, toolUsage), literacy (taskCompletion,
    // codeCorrectness, docCoverage), and any future mode dimensions.
    if (bDims || eDims) {
        const allKeys = new Set([
            ...Object.keys(bDims ?? {}),
            ...Object.keys(eDims ?? {}),
        ]);
        const result = {};
        for (const key of allKeys) {
            const bVal = bDims?.[key] ?? 0;
            const eVal = eDims?.[key] ?? 0;
            result[key] = { baseline: bVal, delta: eVal - bVal, experiment: eVal };
        }
        return result;
    }
    // Legacy fallback: older reports may lack the dimensions map entirely.
    // Read from the three named FeatureScore fields instead.
    const bTask = b?.taskCompletion ?? 0;
    const eTask = e?.taskCompletion ?? 0;
    const bCode = b?.codeCorrectness ?? 0;
    const eCode = e?.codeCorrectness ?? 0;
    const bDoc = b?.docCoverage ?? 0;
    const eDoc = e?.docCoverage ?? 0;
    return {
        codeCorrectness: {
            baseline: bCode,
            delta: eCode - bCode,
            experiment: eCode,
        },
        docCoverage: { baseline: bDoc, delta: eDoc - bDoc, experiment: eDoc },
        taskCompletion: {
            baseline: bTask,
            delta: eTask - bTask,
            experiment: eTask,
        },
    };
}
// ---------------------------------------------------------------------------
// Main compare function
// ---------------------------------------------------------------------------
/** Safely get a feature score from a scores array */
function findScore(scores, area) {
    return scores.find((s) => s.feature === area);
}
