/**
 * pipeline/grader-sensitivity-runner.ts
 *
 * Orchestration module for grader sensitivity (discrimination power) testing
 * (Phase 4).
 *
 * Discovers canonical reference solutions, applies programmatic degradations,
 * grades each original/degraded pair, and calls `analyzeSensitivity()` from
 * the pure computation module.
 *
 * Migrated from lib/grader-sensitivity.ts — no process.argv, no process.exit(),
 * no module-level constants. Accepts rootDir as parameter.
 *
 * @see docs/archive/exec-plans/grader-reliability.md — Phase 4
 */
import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync, } from "fs";
import { basename, join } from "path";
import { ConsoleLogger } from "../adapters/loggers/index.js";
import { DEGRADATION_STRATEGIES } from "./degradations.js";
import { gradeOnce, loadGraderModel } from "./grader-api.js";
import { analyzeSensitivity, } from "./grader-sensitivity.js";
// ---------------------------------------------------------------------------
// Internal helpers
// ---------------------------------------------------------------------------
/**
 * Discover canonical reference solutions.
 * Scans canonical/reference-solutions/<area>/ for source files.
 */
function discoverReferenceSolutions(rootDir, areaFilter) {
    const refsDir = join(rootDir, "canonical", "reference-solutions");
    if (!existsSync(refsDir)) {
        // Reference solutions now live in the Content Lake (ailf.referenceSolution).
        // Return empty array when local files don't exist.
        return [];
    }
    const areas = readdirSync(refsDir, { withFileTypes: true })
        .filter((d) => d.isDirectory())
        .map((d) => d.name)
        .filter((name) => !areaFilter || name === areaFilter)
        .sort();
    if (areas.length === 0) {
        throw new Error(areaFilter
            ? `No reference solutions found for area "${areaFilter}".`
            : "No reference solution directories found.");
    }
    const solutions = [];
    for (const area of areas) {
        const areaDir = join(refsDir, area);
        const files = readdirSync(areaDir)
            .filter((f) => f.endsWith(".ts") ||
            f.endsWith(".tsx") ||
            f.endsWith(".js") ||
            f.endsWith(".jsx"))
            .sort();
        for (const file of files) {
            const filePath = join(areaDir, file);
            const content = readFileSync(filePath, "utf-8");
            solutions.push({
                area,
                content,
                sourcePath: `canonical/reference-solutions/${area}/${file}`,
            });
        }
    }
    return solutions;
}
/**
 * Generate all degraded pairs from reference solutions.
 * Each solution × each degradation strategy = one pair.
 */
function generateDegradedPairs(solutions) {
    const pairs = [];
    for (const solution of solutions) {
        for (const strategy of DEGRADATION_STRATEGIES) {
            const degraded = strategy.apply(solution.content);
            // Only include if degradation actually changed the code
            if (degraded !== solution.content) {
                pairs.push({
                    degradation: strategy,
                    degraded,
                    original: solution.content,
                    sourcePath: solution.sourcePath,
                });
            }
        }
    }
    return pairs;
}
// ---------------------------------------------------------------------------
// Report formatting (pure)
// ---------------------------------------------------------------------------
/**
 * Format a GraderSensitivityResult as a human-readable table report.
 * Returns a string — does NOT print to console.
 */
export function formatSensitivityReport(result) {
    const lines = [];
    lines.push("=".repeat(80));
    lines.push("                    GRADER SENSITIVITY REPORT");
    lines.push("=".repeat(80));
    lines.push("");
    lines.push(`  Grader:       ${result.graderModel}`);
    lines.push(`  Total pairs:  ${result.totalPairs}`);
    lines.push("");
    // Overall metrics
    lines.push("-".repeat(80));
    lines.push("OVERALL");
    lines.push("-".repeat(80));
    lines.push("");
    lines.push(`  Concordance:  ${result.concordanceRate}%`);
    lines.push(`  Avg separation: ${result.avgSeparation} points`);
    lines.push("");
    // Per-dimension table
    lines.push("-".repeat(80));
    lines.push("PER-DIMENSION SENSITIVITY");
    lines.push("-".repeat(80));
    lines.push("");
    const h = "| Dimension        | Concordance | Avg Sep | Tied  | Pairs |";
    const sep = "|------------------|-------------|---------|-------|-------|";
    lines.push(h);
    lines.push(sep);
    const dims = Object.entries(result.perDimension).map(([key, data]) => ({
        data,
        name: key
            .split(/[-_]/)
            .map((w) => w.charAt(0).toUpperCase() + w.slice(1))
            .join(" "),
    }));
    for (const { data, name } of dims) {
        lines.push(`| ${name.padEnd(16)} | ${String(data.concordanceRate + "%").padStart(11)} | ${String(data.avgSeparation).padStart(7)} | ${String(data.tiedRate + "%").padStart(5)} | ${String(data.pairCount).padStart(5)} |`);
    }
    lines.push("");
    // Cross-dimension analysis
    lines.push("-".repeat(80));
    lines.push("CROSS-DIMENSION SENSITIVITY");
    lines.push("-".repeat(80));
    lines.push("");
    lines.push(`  On-target:   ${result.crossDimension.onTarget.concordanceRate}% concordance, ${result.crossDimension.onTarget.avgSeparation} avg sep (${result.crossDimension.onTarget.pairCount} pairs)`);
    lines.push(`  Off-target:  ${result.crossDimension.offTarget.concordanceRate}% concordance, ${result.crossDimension.offTarget.avgSeparation} avg sep (${result.crossDimension.offTarget.pairCount} pairs)`);
    lines.push("");
    // Per-degradation breakdown
    if (result.byDegradation.length > 0) {
        lines.push("-".repeat(80));
        lines.push("PER-DEGRADATION TYPE");
        lines.push("-".repeat(80));
        lines.push("");
        const dh = "| Degradation                                  | Concordance | Avg Sep | Pairs |";
        const ds = "|----------------------------------------------|-------------|---------|-------|";
        lines.push(dh);
        lines.push(ds);
        for (const deg of result.byDegradation) {
            const desc = deg.description.slice(0, 44).padEnd(44);
            lines.push(`| ${desc} | ${String(deg.concordanceRate + "%").padStart(11)} | ${String(deg.avgSeparation).padStart(7)} | ${String(deg.pairCount).padStart(5)} |`);
        }
        lines.push("");
    }
    // Failed pairs (worst failures)
    const topN = Math.min(5, result.failedPairs.length);
    if (topN > 0) {
        lines.push("-".repeat(80));
        lines.push(`TOP ${topN} REVERSED PAIRS (grader ranked degraded higher)`);
        lines.push("-".repeat(80));
        lines.push("");
        for (let i = 0; i < topN; i++) {
            const p = result.failedPairs[i];
            const delta = p.degradedScore - p.originalScore;
            lines.push(`  ${i + 1}. ${basename(p.sourcePath)} — ${p.dimension}`);
            lines.push(`     Original=${p.originalScore}, Degraded=${p.degradedScore} (Δ=+${delta})`);
            lines.push(`     Degradation: ${p.degradationDescription}`);
        }
        lines.push("");
    }
    return lines.join("\n");
}
// ---------------------------------------------------------------------------
// Main runner
// ---------------------------------------------------------------------------
/**
 * Run grader sensitivity analysis.
 *
 * Discovers reference solutions, applies degradations, grades each pair
 * with the configured grader, and analyzes discrimination power.
 *
 * @throws Error if no reference solutions found or no pairs generated
 */
export async function runGraderSensitivity(options) {
    const { rootDir, areaFilter, format = "table" } = options;
    const log = options.logger ?? new ConsoleLogger();
    log.info("=== Grader Sensitivity Analysis ===\n");
    // Resolve grader model
    const grader = loadGraderModel(rootDir);
    log.info(`  Grader:       ${grader.label} (${grader.id})`);
    // Discover reference solutions
    const solutions = discoverReferenceSolutions(rootDir, areaFilter);
    log.info(`  Solutions:    ${solutions.length} reference files`);
    if (areaFilter) {
        log.info(`  Area filter:  ${areaFilter}`);
    }
    // Generate degraded pairs
    const degradedPairs = generateDegradedPairs(solutions);
    log.info(`  Pairs:        ${degradedPairs.length} (solutions × degradations)`);
    if (degradedPairs.length === 0) {
        throw new Error("No degraded pairs generated. Check reference solutions.");
    }
    // Each pair needs 2 grades (original + degraded) × 3 dimensions = 6 calls
    const dimensions = [
        "taskCompletion",
        "codeCorrectness",
        "docCoverage",
    ];
    const totalCalls = degradedPairs.length * dimensions.length * 2;
    const estimatedCost = totalCalls * 0.005;
    log.info(`  API calls:    ${totalCalls}`);
    log.info(`  Est. cost:    ~$${estimatedCost.toFixed(2)}`);
    log.info("");
    // Grade each pair
    log.info("  Grading original and degraded pairs...");
    const rubricTemplate = (dim) => {
        const labels = {
            codeCorrectness: "Evaluate code correctness: Does the code follow best practices, use correct APIs, and avoid anti-patterns? Score 0–100.",
            docCoverage: "Evaluate documentation coverage: Does the response demonstrate knowledge from official documentation? Are imports, API names, and configuration correct per the docs? Score 0–100.",
            taskCompletion: "Evaluate task completion: Does the response fully implement the requested feature? Are all requirements addressed? Score 0–100.",
        };
        return labels[dim] ?? `Evaluate ${dim}. Score 0–100.`;
    };
    const sensitivityPairs = [];
    let completed = 0;
    let failed = 0;
    for (const pair of degradedPairs) {
        const area = pair.sourcePath
            .split("/")
            .find((_s, i, arr) => arr[i - 1] === "reference-solutions") ?? "unknown";
        for (const dimension of dimensions) {
            const rubric = rubricTemplate(dimension);
            // Grade original
            const originalScore = await gradeOnce(grader.id, pair.original, rubric);
            // Grade degraded
            const degradedScore = await gradeOnce(grader.id, pair.degraded, rubric);
            completed++;
            if (completed % 10 === 0 ||
                completed === degradedPairs.length * dimensions.length) {
                const total = degradedPairs.length * dimensions.length;
                const pct = Math.round((completed / total) * 100);
                log.info(`    Progress: ${completed}/${total} (${pct}%)`);
            }
            if (originalScore === null || degradedScore === null) {
                failed++;
                continue;
            }
            sensitivityPairs.push({
                area,
                degradationDescription: pair.degradation.description,
                degradedScore,
                dimension,
                originalScore,
                sourcePath: pair.sourcePath,
                targetDimension: pair.degradation.targetDimension,
            });
        }
    }
    log.info(""); // newline after progress
    if (failed > 0) {
        log.warn(`    ⚠ ${failed} grading pairs failed (excluded)`);
    }
    log.info("");
    if (sensitivityPairs.length === 0) {
        throw new Error("No sensitivity pairs to analyze. All grading calls failed.");
    }
    // Analyze
    const result = analyzeSensitivity(sensitivityPairs, grader.id);
    // Output
    if (format === "table") {
        log.info(formatSensitivityReport(result));
    }
    else {
        log.info(JSON.stringify(result, null, 2));
    }
    // Write output
    const outPath = options.outputPath ??
        join(rootDir, "results", "latest", "grader-sensitivity.json");
    const outDir = join(outPath, "..");
    mkdirSync(outDir, { recursive: true });
    writeFileSync(outPath, JSON.stringify(result, null, 2));
    log.info(`\n  📄 Results written to ${outPath}`);
    return result;
}
