/**
 * pipeline/grader-validate-runner.ts
 *
 * Orchestration module for grader validation against human reference grades
 * (Phase 2).
 *
 * Loads human-graded reference samples from canonical/grader-references/,
 * runs the grader model on each sample, and compares against human scores
 * using `validateGrader()` from the pure computation module.
 *
 * Migrated from lib/grader-validate.ts — no process.argv, no process.exit(),
 * no module-level constants. Accepts rootDir as parameter.
 *
 * @see docs/archive/exec-plans/grader-reliability.md — Phase 2
 */
import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync, } from "fs";
import { join } from "path";
import { load } from "js-yaml";
import { ConsoleLogger } from "../adapters/loggers/index.js";
import { gradeOnce, loadGraderModel } from "./grader-api.js";
import { classifyCorrelation, validateGrader, } from "./grader-validation.js";
// ---------------------------------------------------------------------------
// Internal helpers
// ---------------------------------------------------------------------------
function mapDimension(dim) {
    switch (dim) {
        case "code-correctness":
            return "codeCorrectness";
        case "doc-coverage":
            return "docCoverage";
        case "task-completion":
            return "taskCompletion";
        default:
            return null;
    }
}
/**
 * Load human reference grades from canonical/grader-references/.
 * @throws Error if directory or files not found
 */
function loadReferenceGrades(rootDir) {
    const refsDir = join(rootDir, "canonical", "grader-references");
    if (!existsSync(refsDir)) {
        throw new Error(`Reference grades directory not found: ${refsDir}. ` +
            "Create canonical/grader-references/ with YAML reference files. " +
            "See docs/archive/exec-plans/grader-reliability.md — Phase 2.");
    }
    const files = readdirSync(refsDir)
        .filter((f) => f.endsWith(".yaml") || f.endsWith(".yml"))
        .sort();
    if (files.length === 0) {
        throw new Error(`No YAML files found in ${refsDir}`);
    }
    const allGrades = [];
    for (const file of files) {
        const filePath = join(refsDir, file);
        const raw = readFileSync(filePath, "utf-8");
        const parsed = load(raw);
        if (Array.isArray(parsed)) {
            allGrades.push(...parsed);
        }
        else if (typeof parsed === "object" && parsed !== null) {
            allGrades.push(parsed);
        }
    }
    return allGrades;
}
// ---------------------------------------------------------------------------
// Report formatting (pure)
// ---------------------------------------------------------------------------
/**
 * Format a GraderValidation result as a human-readable table report.
 * Returns a string — does NOT print to console.
 */
export function formatValidationReport(result) {
    const lines = [];
    lines.push("=".repeat(80));
    lines.push("                    GRADER VALIDATION REPORT");
    lines.push("=".repeat(80));
    lines.push("");
    lines.push(`  Grader:          ${result.graderModel}`);
    lines.push(`  Observations:    ${result.totalObservations}`);
    lines.push("");
    // Overall metrics
    lines.push("-".repeat(80));
    lines.push("OVERALL METRICS");
    lines.push("-".repeat(80));
    lines.push("");
    lines.push(`  MAE:             ${result.overallMae} points`);
    lines.push(`  Correlation:     r=${result.overallCorrelation} (${classifyCorrelation(result.overallCorrelation)})`);
    lines.push(`  Bias:            ${result.overallBias > 0 ? "+" : ""}${result.overallBias} (${result.overallBias > 0 ? "grader scores higher" : result.overallBias < 0 ? "grader scores lower" : "no bias"})`);
    lines.push("");
    // Per-dimension table
    lines.push("-".repeat(80));
    lines.push("PER-DIMENSION VALIDITY");
    lines.push("-".repeat(80));
    lines.push("");
    const h = "| Dimension        |  MAE  | Correlation | Quality   | Bias   | Count |";
    const sep = "|------------------|-------|-------------|-----------|--------|-------|";
    lines.push(h);
    lines.push(sep);
    const dims = Object.entries(result.perDimension).map(([key, data]) => ({
        data,
        name: key
            .split(/[-_]/)
            .map((w) => w.charAt(0).toUpperCase() + w.slice(1))
            .join(" "),
    }));
    for (const { data, name } of dims) {
        const quality = classifyCorrelation(data.correlation);
        const biasStr = data.bias > 0 ? `+${data.bias}` : `${data.bias}`;
        lines.push(`| ${name.padEnd(16)} | ${String(data.mae).padStart(5)} | r=${String(data.correlation).padStart(9)} | ${quality.padEnd(9)} | ${biasStr.padStart(6)} | ${String(data.count).padStart(5)} |`);
    }
    lines.push("");
    // Pass/fail verdict
    lines.push("-".repeat(80));
    lines.push("VERDICT");
    lines.push("-".repeat(80));
    lines.push("");
    if (result.passesThreshold) {
        lines.push(`  ✅ PASSED: MAE ${result.overallMae} < threshold ${result.maeThreshold}`);
    }
    else {
        lines.push(`  ❌ FAILED: MAE ${result.overallMae} >= threshold ${result.maeThreshold}`);
    }
    lines.push("");
    // Largest disagreements
    const topN = Math.min(5, result.largestDisagreements.length);
    if (topN > 0) {
        lines.push("-".repeat(80));
        lines.push(`TOP ${topN} LARGEST DISAGREEMENTS`);
        lines.push("-".repeat(80));
        lines.push("");
        for (let i = 0; i < topN; i++) {
            const d = result.largestDisagreements[i];
            const sign = d.signedError > 0 ? "+" : "";
            lines.push(`  ${i + 1}. ${d.taskId} — ${d.dimension}`);
            lines.push(`     Human=${d.humanScore}, Grader=${d.graderScore} (${sign}${d.signedError})`);
            if (d.notes) {
                lines.push(`     Note: ${d.notes}`);
            }
        }
        lines.push("");
    }
    return lines.join("\n");
}
// ---------------------------------------------------------------------------
// Main runner
// ---------------------------------------------------------------------------
/**
 * Run grader validation against human reference grades.
 *
 * Loads human-graded samples, grades each with the grader model,
 * and computes validation metrics (MAE, correlation, bias).
 *
 * @throws Error if no reference grades found, or no grades to analyze
 */
export async function runGraderValidate(options) {
    const { rootDir } = options;
    const log = options.logger ?? new ConsoleLogger();
    const maeThreshold = options.maeThreshold ?? 10;
    log.section("Grader Validation");
    // Resolve grader model
    const graderModel = options.graderModel ?? loadGraderModel(rootDir).id;
    log.info(`Grader:     ${graderModel}`);
    log.info(`Threshold:  MAE < ${maeThreshold}`);
    // Load reference grades
    const rawGrades = loadReferenceGrades(rootDir);
    log.info(`Samples:    ${rawGrades.length} reference-graded responses`);
    // Count total rubric judgments
    let totalJudgments = 0;
    for (const rg of rawGrades) {
        totalJudgments += rg.rubrics.length;
    }
    log.info(`Judgments:  ${totalJudgments} (response × rubric pairs)`);
    const estimatedCost = totalJudgments * 0.005;
    log.info(`Est. cost:  ~$${estimatedCost.toFixed(2)}`);
    log.info("");
    // Grade each reference sample
    log.info("Running grader on reference samples...");
    const grades = [];
    let completed = 0;
    let failed = 0;
    for (const ref of rawGrades) {
        for (const rubric of ref.rubrics) {
            const dimension = mapDimension(rubric.dimension);
            if (!dimension) {
                log.error(`⚠ Unknown dimension '${rubric.dimension}' — skipping`);
                continue;
            }
            const graderScore = await gradeOnce(graderModel, ref.response, rubric.rubricText);
            completed++;
            if (completed % 5 === 0 || completed === totalJudgments) {
                process.stdout.write(`\r  Progress: ${completed}/${totalJudgments}`);
            }
            if (graderScore === null) {
                failed++;
                continue;
            }
            grades.push({
                area: ref.area,
                dimension,
                graderScore,
                humanScore: rubric.humanScore,
                taskId: ref.taskId,
                ...(rubric.notes && { notes: rubric.notes }),
            });
        }
    }
    log.info(""); // newline after progress
    if (failed > 0) {
        log.warn(`${failed} grading calls failed (excluded from analysis)`);
    }
    log.info("");
    if (grades.length === 0) {
        throw new Error("No grades to analyze. All grading calls failed.");
    }
    // Validate
    const result = validateGrader(grades, graderModel, { maeThreshold });
    // Print report
    log.info(formatValidationReport(result));
    // Write output
    const outDir = join(rootDir, "results", "latest");
    mkdirSync(outDir, { recursive: true });
    const outPath = join(outDir, "grader-validation.json");
    writeFileSync(outPath, JSON.stringify(result, null, 2));
    log.info(`\n📄 Results written to ${outPath}`);
    // Throw if threshold not met (instead of process.exit)
    if (!result.passesThreshold) {
        throw new Error(`VALIDATION FAILED: MAE ${result.overallMae} exceeds threshold ${maeThreshold}`);
    }
    return result;
}
