/**
 * grader-consistency-runner.ts
 *
 * Orchestration logic for grader consistency analysis.
 *
 * Reads existing eval results, re-runs ONLY the grading assertions N additional
 * times with the configured grader model, and analyzes score variance.
 *
 * This does NOT re-run the models under test — it only re-grades the same
 * responses. Cost is low: ~$0.005 per grading call × N replications.
 *
 * Migrated from lib/grader-consistency.ts — no process.argv, no process.exit(),
 * no module-level constants.
 *
 * @see docs/archive/exec-plans/grader-reliability.md — Phase 1
 */
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
import { join } from "path";
import { ConsoleLogger } from "../adapters/loggers/index.js";
import { gradeOnce } from "./grader-api.js";
import { analyzeConsistency, } from "./grader-consistency.js";
// ---------------------------------------------------------------------------
// Rubric dimension classification (similar to calculate-scores)
// ---------------------------------------------------------------------------
function classifyDimension(component) {
    // Prefer structured metadata — pass through any dimension name directly,
    // enabling non-literacy profiles (MCP, agent, knowledge-probe)
    const metadata = component.assertion?.metadata;
    if (metadata?.dimension) {
        return metadata.dimension;
    }
    // Fallback: heuristic name matching (returns kebab-case)
    const value = (component.assertion?.value ?? "").toLowerCase();
    if (value.includes("task completion"))
        return "task-completion";
    if (value.includes("code correctness"))
        return "code-correctness";
    if (value.includes("documentation coverage") || value.includes("hallucinate"))
        return "doc-coverage";
    return null;
}
// ---------------------------------------------------------------------------
// Grading judgment extraction
// ---------------------------------------------------------------------------
function detectFeatureArea(description) {
    const desc = description.toLowerCase();
    if (desc.includes("portable text"))
        return "portable-text";
    if (desc.includes("content lake"))
        return "content-lake";
    if (desc.includes("image handling") || desc.includes("image asset"))
        return "image-handling";
    if (desc.includes("studio"))
        return "studio-setup";
    if (desc.includes("visual") ||
        desc.includes("presentation") ||
        desc.includes("live preview"))
        return "visual-editing";
    if (desc.includes("groq"))
        return "groq";
    if (desc.includes("function") || desc.includes("webhook"))
        return "functions";
    if (desc.includes("next") || desc.includes("app router"))
        return "nextjs-live";
    if (desc.includes("remix") ||
        desc.includes("nuxt") ||
        desc.includes("svelte"))
        return "frameworks";
    return "other";
}
/**
 * Extract all llm-rubric grading judgments from eval results.
 * Only includes gold (with-docs) tests to keep the analysis focused.
 */
export function extractGradingJudgments(file) {
    const judgments = [];
    for (const result of file.results.results) {
        if (!result.gradingResult)
            continue;
        const description = result.testCase?.description ?? "unknown";
        const hasDocs = result.vars?.docs && result.vars.docs.trim().length > 0;
        // Only grade "gold" (with-docs) tests — baseline tests use the output-only
        // scoring profile and doc-coverage is undefined without docs
        if (!hasDocs)
            continue;
        const area = detectFeatureArea(description);
        const providerId = result.provider?.id;
        for (const comp of result.gradingResult.componentResults) {
            if (comp.assertion?.type !== "llm-rubric")
                continue;
            const dimension = classifyDimension(comp);
            if (!dimension)
                continue;
            const rubricText = typeof comp.assertion.value === "string" ? comp.assertion.value : "";
            if (!rubricText)
                continue;
            judgments.push({
                area,
                description,
                dimension,
                originalScore: typeof comp.score === "number" ? comp.score : 0,
                providerId,
                responseText: result.response?.output ?? "",
                rubricText,
            });
        }
    }
    return judgments;
}
// ---------------------------------------------------------------------------
// Report formatting (pure)
// ---------------------------------------------------------------------------
export function formatConsistencyReport(result, graderModel) {
    const lines = [];
    lines.push("=".repeat(80));
    lines.push("                    GRADER CONSISTENCY REPORT");
    lines.push("=".repeat(80));
    lines.push("");
    lines.push(`  Grader model:  ${graderModel}`);
    lines.push(`  Replications:  ${result.replications} (incl. original)`);
    lines.push(`  Judgments:     ${result.totalJudgments}`);
    lines.push("");
    // Overall stats
    lines.push("-".repeat(80));
    lines.push("OVERALL");
    lines.push("-".repeat(80));
    lines.push("");
    lines.push(`  Avg σ:          ${result.avgStdDev}`);
    lines.push(`  Max σ:          ${result.maxStdDev}`);
    lines.push(`  Avg range:      ${result.avgRange} points`);
    lines.push("");
    // Per-dimension table
    lines.push("-".repeat(80));
    lines.push("PER-DIMENSION CONSISTENCY");
    lines.push("-".repeat(80));
    lines.push("");
    const h = "| Dimension        | Avg σ | Max σ | Avg Range | Judgments |";
    const sep = "|------------------|-------|-------|-----------|-----------| ";
    lines.push(h);
    lines.push(sep);
    // Derive display rows dynamically from whatever dimensions are present
    const dims = Object.entries(result.perDimension).map(([key, data]) => ({
        data,
        // kebab-case → Title Case (e.g. 'task-completion' → 'Task Completion')
        name: key
            .split("-")
            .map((w) => w.charAt(0).toUpperCase() + w.slice(1))
            .join(" "),
    }));
    for (const { data, name } of dims) {
        lines.push(`| ${name.padEnd(16)} | ${String(data.avgStdDev).padStart(5)} | ${String(data.maxStdDev).padStart(5)} | ${String(data.avgRange).padStart(9)} | ${String(data.judgmentCount).padStart(9)} |`);
    }
    lines.push("");
    // Noise threshold recommendation
    lines.push("-".repeat(80));
    lines.push("NOISE THRESHOLD RECOMMENDATION");
    lines.push("-".repeat(80));
    lines.push("");
    lines.push(`  Current default:     ±2 (DEFAULT_NOISE_THRESHOLD)`);
    lines.push(`  Recommended:         ±${result.recommendedThreshold} (based on 2× max dimension σ)`);
    if (result.recommendedThreshold > 2) {
        lines.push(`  ⚠ Current threshold may be too low — comparison deltas within ±${result.recommendedThreshold}`);
        lines.push(`    should be treated as noise, not real changes.`);
    }
    else {
        lines.push(`  ✅ Current threshold is adequate for this grader's consistency.`);
    }
    lines.push("");
    // Top 5 noisiest judgments
    const topN = Math.min(5, result.judgments.length);
    if (topN > 0) {
        lines.push("-".repeat(80));
        lines.push(`TOP ${topN} NOISIEST JUDGMENTS`);
        lines.push("-".repeat(80));
        lines.push("");
        for (let i = 0; i < topN; i++) {
            const j = result.judgments[i];
            const provider = j.providerId ? ` [${j.providerId}]` : "";
            lines.push(`  ${i + 1}. ${j.taskId}${provider}`);
            lines.push(`     ${j.dimension}: σ=${j.stdDev}, range=${j.range} (${j.min}–${j.max}), mean=${j.mean}`);
        }
        lines.push("");
    }
    return lines.join("\n");
}
// ---------------------------------------------------------------------------
// Main runner
// ---------------------------------------------------------------------------
/**
 * Run the grader consistency analysis.
 *
 * Reads eval results, re-grades each judgment N times, and writes the
 * consistency report to results/latest/grader-consistency.json.
 *
 * @throws Error if results file not found, replications < 2, or no judgments found
 */
export async function runGraderConsistency(options) {
    const { replications, resultsPath, rootDir } = options;
    const log = options.logger ?? new ConsoleLogger();
    log.section("Grader Consistency Analysis");
    // Validate inputs
    if (!existsSync(resultsPath)) {
        throw new Error(`Results file not found: ${resultsPath}. Run 'pnpm eval' first to generate results.`);
    }
    if (replications < 2) {
        throw new Error("Need at least 2 replications for meaningful analysis.");
    }
    // Load eval results
    log.info(`Results:      ${resultsPath}`);
    log.info(`Replications: ${replications}`);
    const file = JSON.parse(readFileSync(resultsPath, "utf-8"));
    // Extract grader model
    const graderModel = file.config?.defaultTest?.options?.rubricProvider ??
        file.config?.defaultTest?.options?.provider;
    if (!graderModel) {
        throw new Error("Could not determine grader model from eval results config.");
    }
    log.info(`Grader:       ${graderModel}`);
    // Extract judgments
    const judgments = extractGradingJudgments(file);
    log.info(`Judgments:    ${judgments.length} (gold tests × rubric dimensions)`);
    if (judgments.length === 0) {
        throw new Error("No gradable judgments found in results.");
    }
    const totalCalls = judgments.length * replications;
    const estimatedCost = totalCalls * 0.005;
    log.info(`API calls:    ${totalCalls} (${judgments.length} × ${replications})`);
    log.info(`Est. cost:    ~$${estimatedCost.toFixed(2)}`);
    log.info("");
    // Re-grade each judgment N times
    log.info(`Running ${replications} replications per judgment...`);
    const gradings = [];
    let completed = 0;
    let failed = 0;
    for (const judgment of judgments) {
        const scores = [judgment.originalScore]; // Include the original as replication #0
        for (let i = 0; i < replications; i++) {
            const score = await gradeOnce(graderModel, judgment.responseText, judgment.rubricText);
            if (score !== null) {
                scores.push(score);
            }
            else {
                failed++;
            }
        }
        completed++;
        if (completed % 10 === 0 || completed === judgments.length) {
            const pct = Math.round((completed / judgments.length) * 100);
            process.stdout.write(`\r  Progress: ${completed}/${judgments.length} (${pct}%)`);
        }
        gradings.push({
            area: judgment.area,
            dimension: judgment.dimension,
            providerId: judgment.providerId,
            scores,
            taskId: judgment.description,
        });
    }
    log.info(""); // newline after progress
    if (failed > 0) {
        log.warn(`${failed} grading calls failed (excluded from analysis)`);
    }
    log.info("");
    // Analyze consistency
    const result = analyzeConsistency(gradings);
    // Print report
    log.info(formatConsistencyReport(result, graderModel));
    // Write output
    const outDir = join(rootDir, "results", "latest");
    mkdirSync(outDir, { recursive: true });
    const outPath = join(outDir, "grader-consistency.json");
    writeFileSync(outPath, JSON.stringify(result, null, 2));
    log.info(`\n📄 Results written to ${outPath}`);
    return result;
}
