/**
 * grader subcommand group — tools for measuring grader reliability.
 *
 * Exposes four subcommands:
 *   ailf grader consistency   — measure grading variance (Phase 1)
 *   ailf grader compare       — inter-grader comparison (Phase 3)
 *   ailf grader sensitivity   — discrimination power testing (Phase 4)
 *   ailf grader validate      — accuracy against human references (Phase 2)
 */
import { dirname, join, resolve } from "path";
import { fileURLToPath } from "url";
import { Command } from "commander";
import { runGraderCompare } from "../../pipeline/grader-compare-runner.js";
import { runGraderConsistency } from "../../pipeline/grader-consistency-runner.js";
import { runGraderSensitivity } from "../../pipeline/grader-sensitivity-runner.js";
import { runGraderValidate } from "../../pipeline/grader-validate-runner.js";
import { collect } from "../shared/options.js";
const __dirname = dirname(fileURLToPath(import.meta.url));
const ROOT = resolve(__dirname, "..", "..", "..");
export function createGraderCommand() {
    const cmd = new Command("grader").description("Grader reliability tools");
    // ── consistency ──────────────────────────────────────────────────────
    cmd
        .command("consistency")
        .description("Measure grader consistency by re-grading existing responses N times")
        .option("-r, --replications <n>", "Number of additional grading replications", parseInt, 5)
        .option("--results <path>", "Path to eval-results.json")
        .action(async (opts) => {
        try {
            await runGraderConsistency({
                replications: opts.replications,
                resultsPath: opts.results ??
                    join(ROOT, "results", "latest", "eval-results.json"),
                rootDir: ROOT,
            });
        }
        catch (err) {
            process.exitCode = 1;
            if (err instanceof Error)
                console.error(err.message);
        }
    });
    // ── compare ──────────────────────────────────────────────────────────
    cmd
        .command("compare")
        .description("Compare multiple grader models on the same responses")
        .option("-c, --candidate <model>", "Candidate grader model ID (repeatable)", collect, [])
        .option("--results <path>", "Path to eval results")
        .option("-f, --format <fmt>", "Output format: table or json", "table")
        .option("-o, --output <path>", "Write JSON report to file")
        .action(async (opts) => {
        try {
            const candidates = opts.candidate.map((id) => ({
                id,
                label: id.split(":").pop() ?? id,
            }));
            await runGraderCompare({
                candidates,
                format: opts.format,
                outputPath: opts.output,
                resultsPath: opts.results,
                rootDir: ROOT,
            });
        }
        catch (err) {
            process.exitCode = 1;
            if (err instanceof Error)
                console.error(err.message);
        }
    });
    // ── sensitivity ──────────────────────────────────────────────────────
    cmd
        .command("sensitivity")
        .description("Test grader discrimination power using programmatic code degradation")
        .option("-a, --area <name>", "Test only reference solutions in this area")
        .option("-f, --format <fmt>", "Output format: table or json", "table")
        .option("-o, --output <path>", "Write JSON report to file")
        .action(async (opts) => {
        try {
            await runGraderSensitivity({
                areaFilter: opts.area,
                format: opts.format,
                outputPath: opts.output,
                rootDir: ROOT,
            });
        }
        catch (err) {
            process.exitCode = 1;
            if (err instanceof Error)
                console.error(err.message);
        }
    });
    // ── validate ─────────────────────────────────────────────────────────
    cmd
        .command("validate")
        .description("Validate grader accuracy against human reference grades")
        .option("-g, --grader <model>", "Grader model to validate")
        .option("--mae-threshold <n>", "MAE threshold for pass/fail", parseFloat, 10)
        .action(async (opts) => {
        try {
            const result = await runGraderValidate({
                graderModel: opts.grader,
                maeThreshold: opts.maeThreshold,
                rootDir: ROOT,
            });
            if (!result.passesThreshold) {
                console.error(`\n  ❌ VALIDATION FAILED: MAE ${result.overallMae} exceeds threshold ${opts.maeThreshold}`);
                process.exit(1);
            }
        }
        catch (err) {
            process.exitCode = 1;
            if (err instanceof Error)
                console.error(err.message);
        }
    });
    return cmd;
}
