/**
 * pipeline/grader-compare-runner.ts
 *
 * Orchestration module for inter-grader comparison (Phase 3).
 *
 * Reads eval results, extracts grading judgments, re-grades each with
 * candidate models, and calls `compareGraders()` from the pure
 * computation module.
 *
 * Migrated from lib/grader-compare.ts — no process.argv, no process.exit(),
 * no module-level constants. Accepts rootDir as parameter.
 *
 * @see docs/archive/exec-plans/grader-reliability.md — Phase 3
 */
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
import { join } from "path";
import { ConsoleLogger } from "../adapters/loggers/index.js";
import { loadConfigFile } from "./compiler/config-loader.js";
import { compareGraders, } from "./grader-comparison.js";
import { classifyCorrelation } from "./grader-validation.js";
import { gradeOnce } from "./grader-api.js";
// ---------------------------------------------------------------------------
// Internal helpers
// ---------------------------------------------------------------------------
function classifyDimension(component) {
    // Prefer structured metadata — pass through any dimension name directly,
    // enabling non-literacy profiles (MCP, agent, knowledge-probe)
    const metadata = component.assertion?.metadata;
    if (metadata?.dimension) {
        return metadata.dimension;
    }
    // Fallback: heuristic name matching (returns kebab-case)
    const value = (component.assertion?.value ?? "").toLowerCase();
    if (value.includes("task completion"))
        return "task-completion";
    if (value.includes("code correctness"))
        return "code-correctness";
    if (value.includes("documentation coverage") || value.includes("hallucinate"))
        return "doc-coverage";
    return null;
}
function detectFeatureArea(description) {
    const desc = description.toLowerCase();
    if (desc.includes("portable text"))
        return "portable-text";
    if (desc.includes("content lake"))
        return "content-lake";
    if (desc.includes("image handling") || desc.includes("image asset"))
        return "image-handling";
    if (desc.includes("studio"))
        return "studio-setup";
    if (desc.includes("visual") ||
        desc.includes("presentation") ||
        desc.includes("live preview"))
        return "visual-editing";
    if (desc.includes("groq"))
        return "groq";
    if (desc.includes("function") || desc.includes("webhook"))
        return "functions";
    if (desc.includes("next") || desc.includes("app router"))
        return "nextjs-live";
    if (desc.includes("remix") ||
        desc.includes("nuxt") ||
        desc.includes("svelte"))
        return "frameworks";
    return "other";
}
/**
 * Extract llm-rubric judgments from eval results.
 * Only includes gold (with-docs) tests.
 */
function extractJudgments(file) {
    const judgments = [];
    for (const result of file.results.results) {
        if (!result.gradingResult)
            continue;
        const description = result.testCase?.description ?? "unknown";
        const hasDocs = result.vars?.docs && result.vars.docs.trim().length > 0;
        if (!hasDocs)
            continue;
        const area = detectFeatureArea(description);
        for (const comp of result.gradingResult.componentResults) {
            if (comp.assertion?.type !== "llm-rubric")
                continue;
            const dimension = classifyDimension(comp);
            if (!dimension)
                continue;
            const rubricText = typeof comp.assertion.value === "string" ? comp.assertion.value : "";
            if (!rubricText)
                continue;
            judgments.push({
                area,
                description,
                dimension,
                responseText: result.response?.output ?? "",
                rubricText,
            });
        }
    }
    return judgments;
}
/**
 * Load config: resolve baseline grader and candidate graders.
 * Candidate overrides take precedence over config/models.
 */
function loadConfig(rootDir, candidateOverrides) {
    const data = loadConfigFile("models", rootDir).data;
    const baseline = {
        id: data?.grader?.id ?? "openai:gpt-5",
        label: data?.grader?.label ?? "GPT-5 (grader)",
    };
    let candidates;
    if (candidateOverrides && candidateOverrides.length > 0) {
        candidates = candidateOverrides;
    }
    else {
        const configCandidates = data?.["grader-candidates"] ?? [];
        candidates = configCandidates.map((c) => ({
            id: c.id,
            label: c.label ?? c.id.split(":").pop() ?? c.id,
        }));
    }
    return { baseline, candidates };
}
// ---------------------------------------------------------------------------
// Report formatting (pure)
// ---------------------------------------------------------------------------
/**
 * Format a GraderComparison result as a human-readable table report.
 * Returns a string — does NOT print to console.
 */
export function formatComparisonReport(result) {
    const lines = [];
    lines.push("=".repeat(80));
    lines.push("                    GRADER COMPARISON REPORT");
    lines.push("=".repeat(80));
    lines.push("");
    lines.push(`  Baseline:     ${result.baselineGrader}`);
    lines.push(`  Candidates:   ${result.candidateGraders.join(", ")}`);
    lines.push(`  Generated:    ${result.generatedAt}`);
    lines.push("");
    for (const pair of result.pairwise) {
        lines.push("-".repeat(80));
        lines.push(`${pair.graderA} vs ${pair.graderB}`);
        lines.push("-".repeat(80));
        lines.push("");
        lines.push(`  Correlation:  r=${pair.correlation} (${classifyCorrelation(pair.correlation)})`);
        lines.push(`  Bias:         ${pair.bias > 0 ? "+" : ""}${pair.bias} (${pair.bias > 0 ? "candidate scores higher" : pair.bias < 0 ? "candidate scores lower" : "no bias"})`);
        lines.push(`  Mean |Δ|:     ${pair.meanAbsDiff} points`);
        lines.push("");
        // Per-dimension table
        const h = "| Dimension        | Correlation | Bias   | Mean |Δ| | Count |";
        const sep = "|------------------|-------------|--------|---------|-------|";
        lines.push(h);
        lines.push(sep);
        // Derive display rows dynamically from whatever dimensions are present
        const dims = Object.entries(pair.perDimension).map(([key, data]) => ({
            data,
            // kebab-case → Title Case (e.g. 'task-completion' → 'Task Completion')
            name: key
                .split("-")
                .map((w) => w.charAt(0).toUpperCase() + w.slice(1))
                .join(" "),
        }));
        for (const { data, name } of dims) {
            const biasStr = data.bias > 0 ? `+${data.bias}` : `${data.bias}`;
            lines.push(`| ${name.padEnd(16)} | r=${String(data.correlation).padStart(9)} | ${biasStr.padStart(6)} | ${String(data.meanAbsDiff).padStart(7)} | ${String(data.count).padStart(5)} |`);
        }
        lines.push("");
    }
    // Recommendations
    lines.push("-".repeat(80));
    lines.push("RECOMMENDATIONS");
    lines.push("-".repeat(80));
    lines.push("");
    for (const rec of result.recommendations) {
        const icon = rec.recommendation === "comparable"
            ? "✅"
            : rec.recommendation === "divergent"
                ? "⚠️"
                : "❌";
        lines.push(`  ${icon} ${rec.modelId}: ${rec.recommendation}`);
        lines.push(`     ${rec.reason}`);
    }
    lines.push("");
    return lines.join("\n");
}
// ---------------------------------------------------------------------------
// Main runner
// ---------------------------------------------------------------------------
/**
 * Run inter-grader comparison.
 *
 * Reads eval results, grades each judgment with the baseline and candidate
 * graders, and produces a comparison report.
 *
 * @throws Error if results file not found, no candidates configured, or no judgments found
 */
export async function runGraderCompare(options) {
    const { rootDir, format = "table" } = options;
    const log = options.logger ?? new ConsoleLogger();
    const resultsPath = options.resultsPath
        ? join(rootDir, options.resultsPath)
        : join(rootDir, "results", "latest", "eval-results.json");
    log.info("=== Grader Comparison ===\n");
    // Load config
    const { baseline, candidates } = loadConfig(rootDir, options.candidates);
    if (candidates.length === 0) {
        throw new Error("No candidate graders configured. " +
            "Add grader-candidates to config/models or pass --candidate.");
    }
    // Load eval results
    if (!existsSync(resultsPath)) {
        throw new Error(`Results file not found: ${resultsPath}. Run 'pnpm eval' first.`);
    }
    const file = JSON.parse(readFileSync(resultsPath, "utf-8"));
    // Extract judgments
    const judgments = extractJudgments(file);
    log.info(`  Baseline:     ${baseline.label} (${baseline.id})`);
    log.info(`  Candidates:   ${candidates.map((c) => c.label).join(", ")}`);
    log.info(`  Judgments:    ${judgments.length}`);
    if (judgments.length === 0) {
        throw new Error("No gradable judgments found in results.");
    }
    const totalCalls = judgments.length * (1 + candidates.length);
    const estimatedCost = totalCalls * 0.005;
    log.info(`  API calls:    ${totalCalls} (${judgments.length} × ${1 + candidates.length} models)`);
    log.info(`  Est. cost:    ~$${estimatedCost.toFixed(2)}`);
    log.info("");
    // Grade with baseline
    log.info(`  Grading with baseline: ${baseline.label}...`);
    const baselineScores = await gradeJudgments(judgments, baseline.id, log);
    // Grade with each candidate
    const candidateScoreSets = [];
    for (const candidate of candidates) {
        log.info(`  Grading with candidate: ${candidate.label}...`);
        const scores = await gradeJudgments(judgments, candidate.id, log);
        candidateScoreSets.push({
            label: candidate.label,
            modelId: candidate.id,
            scores,
        });
    }
    log.info("");
    // Compare
    const baselineScoreSet = {
        label: baseline.label,
        modelId: baseline.id,
        scores: baselineScores,
    };
    const result = compareGraders(baselineScoreSet, candidateScoreSets);
    // Output
    if (format === "table") {
        log.info(formatComparisonReport(result));
    }
    else {
        log.info(JSON.stringify(result, null, 2));
    }
    // Write output
    const outPath = options.outputPath ??
        join(rootDir, "results", "latest", "grader-comparison.json");
    const outDir = join(outPath, "..");
    mkdirSync(outDir, { recursive: true });
    writeFileSync(outPath, JSON.stringify(result, null, 2));
    log.info(`\n  📄 Results written to ${outPath}`);
    return result;
}
// ---------------------------------------------------------------------------
// Grading helper
// ---------------------------------------------------------------------------
/**
 * Grade a set of judgments with a specific grader model.
 * Returns GraderScore[] with one score per judgment.
 */
async function gradeJudgments(judgments, graderModel, log) {
    const scores = [];
    let completed = 0;
    let failed = 0;
    for (const judgment of judgments) {
        const score = await gradeOnce(graderModel, judgment.responseText, judgment.rubricText);
        completed++;
        if (completed % 10 === 0 || completed === judgments.length) {
            const pct = Math.round((completed / judgments.length) * 100);
            log.info(`    Progress: ${completed}/${judgments.length} (${pct}%)`);
        }
        if (score === null) {
            failed++;
            continue;
        }
        scores.push({
            area: judgment.area,
            dimension: judgment.dimension,
            score,
            taskId: judgment.description,
        });
    }
    log.info(""); // newline after progress
    if (failed > 0) {
        log.warn(`    ⚠ ${failed} grading calls failed (excluded)`);
    }
    return scores;
}
