/**
 * pipeline/retrieval-metrics.ts
 *
 * Computes retrieval precision and recall by comparing agent-retrieved
 * doc slugs against canonical_docs defined in task definitions.
 *
 * This is a pure computation module — no file I/O beyond reading task files.
 */
import { existsSync } from "fs";
import { discoverTsTaskFiles, loadTsTaskFileSync, } from "../adapters/task-sources/task-file-loader.js";
import { resolveVendoredSubdir } from "./compiler/config-loader.js";
// --- Types for task definitions (just the canonical_docs part) ---
/**
 * Compute retrieval metrics from agentic behavior data.
 *
 * @param rootDir  Package root (to find tasks/*.yaml)
 * @param behaviors  Agentic behavior data per task
 * @returns RetrievalMetrics or null if no canonical docs or behavior data
 */
export function computeRetrievalMetrics(rootDir, behaviors) {
    const canonicalMap = loadCanonicalDocs(rootDir);
    if (canonicalMap.size === 0)
        return null;
    if (behaviors.length === 0)
        return null;
    // Compute per-task metrics
    const taskMetrics = [];
    for (const behavior of behaviors) {
        const taskId = behavior.taskId;
        if (!taskId)
            continue;
        const canonical = canonicalMap.get(taskId);
        if (!canonical)
            continue;
        taskMetrics.push(computeTaskMetrics(taskId, canonical.area, behavior.docSlugsVisited, canonical.slugs));
    }
    if (taskMetrics.length === 0)
        return null;
    // Group by area
    const byArea = new Map();
    for (const tm of taskMetrics) {
        if (!byArea.has(tm.area))
            byArea.set(tm.area, []);
        byArea.get(tm.area).push(tm);
    }
    const areas = [...byArea.entries()]
        .sort(([a], [b]) => a.localeCompare(b))
        .map(([area, tasks]) => {
        const avgPrecision = tasks.reduce((s, t) => s + t.precision, 0) / tasks.length;
        const avgRecall = tasks.reduce((s, t) => s + t.recall, 0) / tasks.length;
        const avgF1 = tasks.reduce((s, t) => s + t.f1, 0) / tasks.length;
        // Invisible docs: canonical docs never retrieved across tasks in this area
        const allRetrieved = new Set(tasks.flatMap((t) => t.retrieved));
        const allExpected = new Set(tasks.flatMap((t) => t.expected));
        const invisibleDocs = [...allExpected]
            .filter((s) => !allRetrieved.has(s))
            .sort();
        return {
            area,
            avgF1,
            avgPrecision,
            avgRecall,
            invisibleDocs,
            taskCount: tasks.length,
            tasks,
        };
    });
    // Overall averages
    const overall = {
        avgF1: areas.reduce((s, a) => s + a.avgF1, 0) / (areas.length || 1),
        avgPrecision: areas.reduce((s, a) => s + a.avgPrecision, 0) / (areas.length || 1),
        avgRecall: areas.reduce((s, a) => s + a.avgRecall, 0) / (areas.length || 1),
    };
    // Global invisible docs
    const allRetrieved = new Set(taskMetrics.flatMap((t) => t.retrieved));
    const allExpected = new Set(taskMetrics.flatMap((t) => t.expected));
    const globalInvisibleDocs = [...allExpected]
        .filter((s) => !allRetrieved.has(s))
        .sort();
    return {
        areas,
        globalInvisibleDocs,
        overall,
    };
}
/**
 * Compute retrieval metrics for a single task.
 */
export function computeTaskMetrics(taskId, area, retrieved, canonical) {
    const retrievedSet = new Set(retrieved);
    const hits = [...retrievedSet].filter((s) => canonical.has(s));
    const missed = [...canonical].filter((s) => !retrievedSet.has(s));
    const unexpected = [...retrievedSet].filter((s) => !canonical.has(s));
    const precision = retrievedSet.size > 0 ? hits.length / retrievedSet.size : 0;
    const recall = canonical.size > 0 ? hits.length / canonical.size : 0;
    const f1 = precision + recall > 0 ? (2 * precision * recall) / (precision + recall) : 0;
    return {
        area,
        expected: [...canonical],
        f1,
        hits,
        missed,
        precision,
        recall,
        retrieved: [...retrievedSet],
        taskId,
        unexpected,
    };
}
/**
 * Load canonical docs from *.task.ts files in tasks/literacy/.
 * Returns a map of taskId → { slugs: Set<string>, area: string }.
 */
export function loadCanonicalDocs(rootDir) {
    const tasksDir = resolveVendoredSubdir(rootDir, "tasks/literacy");
    if (!existsSync(tasksDir))
        return new Map();
    const result = new Map();
    const files = discoverTsTaskFiles(tasksDir);
    for (const file of files) {
        const loaded = loadTsTaskFileSync(file);
        for (const task of loaded.tasks) {
            const t = task;
            const id = typeof t.id === "string" ? t.id : undefined;
            const area = typeof t.area === "string" ? t.area : undefined;
            if (!id || !area)
                continue;
            // Extract slugs from context.docs
            const context = t.context;
            if (!context?.docs || !Array.isArray(context.docs))
                continue;
            const slugs = new Set();
            for (const doc of context.docs) {
                const d = doc;
                if (typeof d.slug === "string")
                    slugs.add(d.slug);
            }
            if (slugs.size > 0) {
                result.set(id, { area, slugs });
            }
        }
    }
    return result;
}
