/**
 * pipeline/failure-modes.ts
 *
 * Ceiling-cross-check failure-mode validator + report assembly + keyword
 * fallback classifier.
 *
 * The grader emits `failureMode` directly under the per-dimension taxonomy
 * (Plan 03-02 — `packages/eval/src/grader/`) when its structured response
 * is available to the pipeline. In practice (W0273 discovery), Promptfoo's
 * `llm-rubric` post-processor extracts `score` + `reason` from the grader's
 * JSON envelope and discards the rest of the structured surface — including
 * `failureMode`. The wire-shape footer instructs the LLM correctly but the
 * structured fields never reach `extractGraderJudgments`, so every emission
 * arrives as the synthesized `failureMode: "unclassified"` placeholder.
 *
 * To restore the pre-2026-05-11 classification rate (15-23% → 0% → 15-23%),
 * a keyword-pattern classifier is run as a FALLBACK when the grader's
 * emitted mode is `"unclassified"` and the score is below the classification
 * threshold. Plan 03-03 deleted this classifier in favor of grader-emission
 * source-of-truth; W0273 reinstates it because the grader-emission path is
 * blocked by Promptfoo's lossy `llm-rubric` parsing. The long-term fix
 * (capturing the grader's full structured response) is tracked separately.
 *
 * `classifyByCeiling` continues to serve as the confidence cross-check.
 *
 * @see docs/decisions/D0005-grader-model-separation.md — single grader emits
 *      failureMode under the per-dimension taxonomy
 * @see docs/decisions/D0049-shared-confidence-contract.md — Confidence triple
 *      shape and `ceiling-cross-check` derivation tag
 * @see docs/audits/2026-05-22-empty-gap-analysis-regression.md — W0273 root
 *      cause (Promptfoo strips structured fields)
 */
import { LEGACY_FAILURE_MODES, detectFeatureArea } from "../_vendor/ailf-core/index.js";
// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------
/** Only classify judgments with scores below this threshold */
const CLASSIFICATION_THRESHOLD = 60;
// ---------------------------------------------------------------------------
// Keyword patterns (W0273 fallback)
//
// Verbatim from the pre-Plan-03-03 implementation. Used only when the
// grader's emitted `failureMode` is `"unclassified"` — the grader's
// emission still wins whenever it actually reaches the pipeline.
// ---------------------------------------------------------------------------
/** API error pattern — checked FIRST to prevent timeout errors containing
 *  "deprecated" from being misclassified as outdated-docs. */
const API_ERROR_PATTERN = /\[api-error\]|timeout|timed out|rate limit|429|503|ECONNRESET|ETIMEDOUT|socket hang up|fetch failed/i;
const OUTDATED_PATTERN = /deprecated|old api|v[0-9]+ syntax|no longer supported|legacy|previous version|outdated|superseded|replaced by/i;
const MISSING_PATTERN = /no documentation|not covered|had to guess|not found|missing.*doc|no.*information|undocumented|couldn't find|without.*documentation/i;
const INCORRECT_PATTERN = /contradicts|incorrect.*doc|doc.*incorrect|wrong.*doc|doc.*wrong|documentation says.*but|factual error|inaccurate|misleading.*doc/i;
const POOR_STRUCTURE_PATTERN = /unclear|ambiguous|couldn't determine|conflicting|confusing|hard to follow|poorly organized|scattered|fragmented/i;
// ---------------------------------------------------------------------------
// Public API
// ---------------------------------------------------------------------------
/**
 * Build a complete failure mode report from grader judgments and scores.
 *
 * The grader-emitted `judgment.failureMode` is the source of truth (Plan
 * 03-03 — keyword classifier deleted). `validateFailureMode` cross-checks
 * the emission against ceiling decomposition and stamps a D0049 confidence.
 *
 * The `FailureMode` triple shape (`mode`, `confidence`, `source`) is
 * preserved for backward compatibility with downstream consumers
 * (gap-analysis, manifest emission) — the bucketed `confidence` enum maps
 * 1:1 from `Confidence.level`, and `source` is always `"ceiling"` now that
 * the keyword path is gone.
 *
 * @param judgments - All grader judgments from the evaluation
 * @param scores - Per-area feature scores (for ceiling decomposition)
 * @returns Failure mode report with per-area breakdowns
 */
export function buildFailureModeReport(judgments, scores) {
    // Build a lookup from area → scores
    const scoreByArea = new Map();
    for (const score of scores) {
        scoreByArea.set(score.feature, score);
    }
    const classifiedJudgments = [];
    const summary = initModeCounts();
    const byArea = {};
    for (const judgment of judgments) {
        // Extract area from taskId description (e.g., "GROQ - Blog queries..." → "groq")
        const area = resolveArea(judgment.taskId, scoreByArea);
        const areaScore = area ? scoreByArea.get(area) : undefined;
        const ceilingScore = areaScore?.ceilingScore ?? 100;
        const floorScore = areaScore?.floorScore ?? 0;
        // Source the failure mode from the grader's emission. CR-02:
        // FailureModeType is open-set (`string`) since Plan 03-02
        // introduced per-dimension extensions (`false-floor`,
        // `spec-mismatch`, `tool-misuse`, `factual-error`, …). The report
        // surfaces the emission directly so downstream consumers see the
        // grader's actual taxonomy choice rather than a collapsed
        // `"unclassified"` bucket.
        const emittedMode = readEmittedMode(judgment);
        // W0273 fallback — when the grader's emitted mode is "unclassified"
        // (the synthesized-unparsed-judgment placeholder; in practice this
        // is every judgment today because Promptfoo's llm-rubric strips the
        // grader's structured response), try keyword classification against
        // the reason prose. Gated on score < CLASSIFICATION_THRESHOLD so
        // passing judgments don't get spurious classifications.
        const keywordFallback = emittedMode === "unclassified" &&
            judgment.score < CLASSIFICATION_THRESHOLD
            ? classifyByKeyword(judgment.reason)
            : null;
        // Cross-check the grader's emission against ceiling decomposition.
        const stamp = validateFailureMode(judgment, ceilingScore, floorScore);
        const classification = keywordFallback
            ? keywordFallback
            : {
                confidence: stamp.level,
                mode: emittedMode,
                source: "ceiling",
            };
        classifiedJudgments.push({ classification, judgment });
        summary[classification.mode] = (summary[classification.mode] ?? 0) + 1;
        // Per-area tracking
        if (area) {
            if (!byArea[area]) {
                byArea[area] = {
                    area,
                    modes: initModeCounts(),
                    topMode: "unclassified",
                    totalJudgments: 0,
                };
            }
            byArea[area].modes[classification.mode] =
                (byArea[area].modes[classification.mode] ?? 0) + 1;
            byArea[area].totalJudgments++;
        }
    }
    // Compute top mode per area
    for (const areaData of Object.values(byArea)) {
        areaData.topMode = findTopMode(areaData.modes);
    }
    const totalJudgments = judgments.length;
    const classified = totalJudgments - (summary["unclassified"] ?? 0);
    const classificationRate = totalJudgments > 0 ? (classified / totalJudgments) * 100 : 0;
    return {
        byArea,
        classificationRate,
        classifiedJudgments,
        summary,
        totalJudgments,
    };
}
/**
 * Cross-check a grader-emitted `failureMode` against ceiling decomposition
 * and emit a D0049 `Confidence` triple stamped with
 * `derivation: "ceiling-cross-check"`.
 *
 * Replaces the deleted keyword-pattern + ceiling combine classifier — the
 * grader's emission is now the source of truth for the mode itself; this
 * function only stamps confidence based on whether the structural ceiling
 * signal agrees.
 *
 *  - `level: "high"` (`signalsPresent: 2`) — grader emission and ceiling
 *    decomposition agree on the same mode.
 *  - `level: "medium"` (`signalsPresent: 2`) — both signals present but
 *    disagree. The live pipeline increments
 *    `GraderReliability.failureModeCalibration` ONLY on this branch — a
 *    true calibration miss requires both signals to be present.
 *  - `level: "low"` (`signalsPresent: 1`) — only the grader's emission;
 *    ceiling decomposition produced no structural signal. Not a
 *    calibration miss (we have nothing to cross-check against).
 *  - `level: "low"` (`signalsPresent: 0`) — passing scores
 *    (`>= CLASSIFICATION_THRESHOLD`) don't classify; emit absent.
 *
 * @param judgment - The grader judgment carrying `failureMode` + `score`
 * @param ceilingScore - The area's ceiling score (with-docs best case)
 * @param floorScore - The area's floor score (no-docs baseline)
 * @returns D0049 Confidence triple stamped `derivation: "ceiling-cross-check"`
 *
 * @see docs/decisions/D0005-grader-model-separation.md
 * @see docs/decisions/D0049-shared-confidence-contract.md
 */
export function validateFailureMode(judgment, ceilingScore, floorScore) {
    // Passing scores don't classify — emit low-confidence absent.
    if (judgment.score >= CLASSIFICATION_THRESHOLD) {
        return {
            level: "low",
            signalsPresent: 0,
            derivation: "ceiling-cross-check",
        };
    }
    const ceilingMode = classifyByCeiling(judgment.score, ceilingScore, floorScore);
    if (!ceilingMode) {
        // No structural ceiling signal — the grader's emission stands but
        // there's nothing to cross-check against. Surface as low-confidence
        // (signalsPresent: 1) so the caller can distinguish "we have one
        // signal, not two" from "the two signals disagree" — and leave
        // failureModeCalibration alone (folding this case in over-counts
        // the reliability metric, see CR-04 in the Phase 3 review).
        return {
            level: "low",
            signalsPresent: 1,
            derivation: "ceiling-cross-check",
        };
    }
    if (ceilingMode.mode === judgment.failureMode) {
        // Both signals agree → high confidence stamp.
        return {
            level: "high",
            signalsPresent: 2,
            derivation: "ceiling-cross-check",
        };
    }
    // Both signals present and disagree — the actual calibration-miss
    // branch. The caller increments GraderReliability.failureModeCalibration
    // only when signalsPresent === 2 here.
    return {
        level: "medium",
        signalsPresent: 2,
        derivation: "ceiling-cross-check",
    };
}
// ---------------------------------------------------------------------------
// Formatting
// ---------------------------------------------------------------------------
/**
 * Format a failure mode report for console output.
 */
export function formatFailureModesConsole(report) {
    const lines = [];
    lines.push("🔍 FAILURE MODE ANALYSIS");
    lines.push("");
    lines.push(`  ${report.totalJudgments} judgments analyzed, ${report.classificationRate.toFixed(0)}% classified`);
    lines.push("");
    // Summary table — legacy modes first (in canonical order), then any
    // per-dimension extensions present in the run sorted by count desc.
    // CR-02: extensions are no longer narrowed to "unclassified"; the
    // formatter now surfaces them rather than dropping the signal.
    lines.push("  Mode                Count");
    lines.push("  ──────────────────  ─────");
    for (const mode of orderedSummaryKeys(report.summary)) {
        const count = report.summary[mode] ?? 0;
        if (count > 0) {
            const icon = modeIcon(mode);
            lines.push(`  ${icon} ${mode.padEnd(18)} ${count}`);
        }
    }
    lines.push("");
    // Per-area breakdown
    if (Object.keys(report.byArea).length > 0) {
        lines.push("  Per-area top failure modes:");
        for (const [area, data] of Object.entries(report.byArea).sort(([a], [b]) => a.localeCompare(b))) {
            const icon = modeIcon(data.topMode);
            lines.push(`    ${area}: ${icon} ${data.topMode} (${data.totalJudgments} judgments)`);
        }
        lines.push("");
    }
    return lines.join("\n");
}
/**
 * Format a failure mode report as markdown for PR comments.
 */
export function formatFailureModesMarkdown(report) {
    const lines = [];
    lines.push("### 🔍 Failure Mode Analysis");
    lines.push("");
    if (report.totalJudgments === 0) {
        lines.push("No judgments analyzed.");
        return lines.join("\n");
    }
    lines.push(`**${report.totalJudgments} judgments** analyzed, **${report.classificationRate.toFixed(0)}%** classified`);
    lines.push("");
    // Summary table — legacy modes first, per-dimension extensions after
    // (CR-02 — emission is now visible in aggregation rather than being
    // collapsed to 'unclassified').
    lines.push("| Mode | Count | % |");
    lines.push("|------|-------|---|");
    for (const mode of orderedSummaryKeys(report.summary)) {
        const count = report.summary[mode] ?? 0;
        if (count > 0) {
            const pct = report.totalJudgments > 0
                ? ((count / report.totalJudgments) * 100).toFixed(0)
                : "0";
            const icon = modeIcon(mode);
            lines.push(`| ${icon} ${mode} | ${count} | ${pct}% |`);
        }
    }
    lines.push("");
    // Per-area table
    if (Object.keys(report.byArea).length > 0) {
        lines.push("<details>");
        lines.push("<summary>Per-area breakdown</summary>");
        lines.push("");
        lines.push("| Area | Top Mode | Judgments |");
        lines.push("|------|----------|----------|");
        for (const [area, data] of Object.entries(report.byArea).sort(([a], [b]) => a.localeCompare(b))) {
            const icon = modeIcon(data.topMode);
            lines.push(`| ${area} | ${icon} ${data.topMode} | ${data.totalJudgments} |`);
        }
        lines.push("");
        lines.push("</details>");
        lines.push("");
    }
    return lines.join("\n");
}
// ---------------------------------------------------------------------------
// Internal helpers
// ---------------------------------------------------------------------------
/**
 * Read the grader's emitted failureMode as the open-set
 * `FailureModeType` (string). Per-dimension extensions from Plan 03-02
 * (`false-floor`, `spec-mismatch`, `tool-misuse`, `factual-error`, …)
 * survive the report aggregation as their own buckets — narrowing them
 * to `"unclassified"` (the pre-CR-02 behavior) silently dropped at
 * least 11 documented legal modes from `report.summary` and
 * `report.byArea[*].topMode`. An absent or empty `failureMode` still
 * buckets as `"unclassified"` so consumers see a stable label rather
 * than an empty key.
 */
function readEmittedMode(judgment) {
    const emitted = judgment.failureMode;
    if (typeof emitted !== "string" || emitted.length === 0) {
        return "unclassified";
    }
    return emitted;
}
/**
 * Classify the failure mode of a low-scoring grader judgment by matching
 * keyword patterns against the reason prose. Returns `null` when no
 * pattern matches. Patterns checked in priority order (API errors first
 * so timeout messages containing "deprecated" don't get misclassified
 * as outdated-docs).
 *
 * W0273 — reinstated as a fallback when the grader's emitted failureMode
 * is "unclassified". Plan 03-03 deleted this code in favor of grader-
 * emission source-of-truth; the deletion is reversed here because
 * Promptfoo's llm-rubric post-processor strips the grader's structured
 * response (only score + reason survive into `comp.*`), so the
 * grader-emission path produces 0% classification on every run.
 */
function classifyByKeyword(reason) {
    const lower = reason.toLowerCase();
    if (API_ERROR_PATTERN.test(lower)) {
        return { confidence: "high", mode: "api-error", source: "keyword" };
    }
    if (OUTDATED_PATTERN.test(lower)) {
        return { confidence: "high", mode: "outdated-docs", source: "keyword" };
    }
    if (MISSING_PATTERN.test(lower)) {
        return { confidence: "high", mode: "missing-docs", source: "keyword" };
    }
    if (INCORRECT_PATTERN.test(lower)) {
        return { confidence: "medium", mode: "incorrect-docs", source: "keyword" };
    }
    if (POOR_STRUCTURE_PATTERN.test(lower)) {
        return { confidence: "medium", mode: "poor-structure", source: "keyword" };
    }
    return null;
}
/**
 * Classify by ceiling-decomposition structural signals — preserved
 * verbatim from the pre-Plan-03-03 implementation. The function itself
 * does not change; only its CALLER (`validateFailureMode`) changes how
 * the output is consumed (confidence stamp instead of parallel
 * classification signal).
 */
function classifyByCeiling(score, ceilingScore, floorScore) {
    const docLift = ceilingScore - floorScore;
    // Negative Doc Lift: docs are actively harmful
    if (docLift < 0) {
        return { confidence: "medium", mode: "outdated-docs", source: "ceiling" };
    }
    // High ceiling, low actual: model can't use good docs → model limitation
    if (ceilingScore > 70 && score < 40) {
        return {
            confidence: "medium",
            mode: "model-limitation",
            source: "ceiling",
        };
    }
    // High floor, low ceiling: model knows better than docs → outdated
    if (floorScore > 50 && ceilingScore < floorScore + 10) {
        return { confidence: "medium", mode: "outdated-docs", source: "ceiling" };
    }
    // Low ceiling, low floor: both docs and model knowledge insufficient
    if (ceilingScore < 40 && floorScore < 30) {
        return { confidence: "low", mode: "missing-docs", source: "ceiling" };
    }
    return null;
}
/**
 * Resolve area name from a task ID or description.
 *
 * Task IDs in grader judgments use human-readable descriptions
 * (e.g., "GROQ - Blog queries with filtering and pagination (gold)")
 * while score areas use kebab-case slugs (e.g., "groq").
 *
 * Strategy:
 * 1. Use detectFeatureArea() which handles human-readable descriptions
 * 2. Fall back to prefix matching for kebab-case task IDs
 * 3. Return undefined if no match is found
 */
function resolveArea(taskId, scoreByArea) {
    // Strategy 1: Use the shared feature area detector (handles descriptions)
    const detected = detectFeatureArea(taskId);
    if (detected !== "other" && scoreByArea.has(detected))
        return detected;
    // Strategy 2: Direct prefix match (kebab-case task IDs like "groq-blog-queries")
    for (const area of scoreByArea.keys()) {
        if (taskId.startsWith(area))
            return area;
    }
    // Strategy 3: Progressive prefix matching on hyphens
    const parts = taskId.split("-");
    if (parts.length > 1) {
        for (let i = parts.length - 1; i >= 1; i--) {
            const candidate = parts.slice(0, i).join("-");
            if (scoreByArea.has(candidate))
                return candidate;
        }
    }
    return undefined;
}
/**
 * Find the most common failure mode in the per-area `modes` record.
 *
 * Iterates every key in the record (CR-02 — record is open-set since
 * Plan 03-02 introduced per-dimension extensions) and picks the
 * highest-count classified mode. Falls back to "unclassified" when
 * the area has no classified emissions at all.
 */
function findTopMode(modes) {
    let topMode = "unclassified";
    let topCount = 0;
    for (const [mode, count] of Object.entries(modes)) {
        if (mode === "unclassified")
            continue; // Prefer classified modes
        if (count > topCount) {
            topCount = count;
            topMode = mode;
        }
    }
    // If nothing classified, return unclassified
    if (topCount === 0)
        return "unclassified";
    return topMode;
}
/**
 * Initialize the per-area / per-summary mode-count record.
 *
 * Pre-allocates buckets for the legacy literacy modes (CR-02 — keeps
 * stable presence for downstream consumers like Studio columns) and
 * leaves per-dimension extensions to be added on first emission.
 */
function initModeCounts() {
    const counts = {};
    for (const mode of LEGACY_FAILURE_MODES) {
        counts[mode] = 0;
    }
    return counts;
}
/** Stable display order for the summary tables — legacy first, then extensions. */
function orderedSummaryKeys(summary) {
    const legacy = LEGACY_FAILURE_MODES.filter((m) => m in summary);
    const extensions = Object.keys(summary)
        .filter((m) => !isLegacyMode(m))
        .sort((a, b) => (summary[b] ?? 0) - (summary[a] ?? 0));
    return [...legacy, ...extensions];
}
const LEGACY_MODE_SET = new Set(LEGACY_FAILURE_MODES);
function isLegacyMode(mode) {
    return LEGACY_MODE_SET.has(mode);
}
/** Get icon for a failure mode — legacy modes have dedicated icons; extensions fall back to a neutral marker. */
function modeIcon(mode) {
    switch (mode) {
        case "api-error":
            return "⚡";
        case "incorrect-docs":
            return "❌";
        case "missing-docs":
            return "📭";
        case "model-limitation":
            return "🤖";
        case "outdated-docs":
            return "📅";
        case "poor-structure":
            return "🏗️";
        case "unclassified":
            return "❓";
        default:
            // Per-dimension extensions (Plan 03-02) — neutral icon, the mode
            // name in the table still identifies the family.
            return "•";
    }
}
