/**
 * `dql eval` — Routing-accuracy harness for the DQL answer agent.
 *
 * Replays a golden set of questions through the EXISTING agent router
 * (`planAgentAnswer`) and scores how well DQL routes each question:
 * certified vs generated vs missing-context (refusal), which certified block
 * it selected, and whether the answer grain matches.
 *
 * The golden set comes from two sources:
 *   1. Every certified block's `examples[].question` in the compiled manifest.
 *      These expect route = certified and expect that same block to be selected.
 *   2. Optional `eval/*.yaml` files in the project, each a list (or { cases: [] })
 *      of `{ question, expectRoute, expectBlock?, expectGrain?, expectRefuse?, name? }`.
 *      These let a repo express the harder categories the block examples cannot:
 *      generated (Tier-2), insufficient-context (refusal), conflict, wrong-grain.
 *
 * This command READS the router output only. It never executes SQL against a
 * warehouse and never changes routing, the manifest schema, or the DQL language.
 *
 * Usage:
 *   dql eval [path]                          Replay the golden set, print a report
 *   dql eval [path] --format json            Emit a machine-readable JSON report
 *   dql eval [path] --min-route-accuracy 0.9 Fail (exit 1) below this route accuracy
 *   dql eval [path] --min-refusal 0.8        Fail (exit 1) below this refusal recall
 *   dql eval [path] --no-examples            Skip manifest block examples (yaml only)
 */
import { existsSync, readdirSync, readFileSync } from 'node:fs';
import { join, resolve } from 'node:path';
import { load as loadYaml } from 'js-yaml';
import { buildManifest, resolveDbtManifestPath, } from '@duckcodeailabs/dql-core';
import { planAgentAnswer } from '@duckcodeailabs/dql-agent';
const ZERO_RESULTS = [];
/** Map the router's route vocabulary onto the spec's route labels. */
export function mapRoute(route) {
    switch (route) {
        case 'certified':
            return 'certified';
        case 'generated_sql':
            return 'generated';
        case 'clarify':
            return 'missing_context';
        case 'research':
            return 'research';
        default:
            return 'missing_context';
    }
}
function blockNameFromObjectKey(objectKey) {
    if (!objectKey)
        return undefined;
    // Certified block keys are minted as `dql:block:<name>`.
    return objectKey.replace(/^dql:block:/, '');
}
function isCertified(block) {
    return String(block.status ?? '').toLowerCase() === 'certified';
}
function normalizeGrain(grain) {
    return String(grain ?? '').trim().toLowerCase();
}
/**
 * Build the golden set from a compiled manifest plus optional eval/*.yaml.
 * Each certified block contributes one case per example question.
 */
export function collectEvalCases(manifest, yamlCases, includeBlockExamples) {
    const cases = [];
    if (includeBlockExamples) {
        for (const block of Object.values(manifest.blocks ?? {})) {
            if (!isCertified(block))
                continue;
            for (const example of block.examples ?? []) {
                const question = example.question?.trim();
                if (!question)
                    continue;
                cases.push({
                    name: `${block.name} · ${question}`,
                    question,
                    source: 'block_example',
                    category: 'certified',
                    expectRoute: 'certified',
                    expectBlock: block.name,
                    expectGrain: block.grain,
                });
            }
        }
    }
    cases.push(...yamlCases);
    return cases;
}
/** Score a single case against the router's decision. Pure + unit-testable. */
export function scoreCase(testCase, plan, manifest) {
    const actualRoute = mapRoute(plan.routeDecision.route);
    const actualBlock = blockNameFromObjectKey(plan.routeDecision.exactObjectKey);
    const actualGrain = actualBlock ? manifest.blocks?.[actualBlock]?.grain : undefined;
    const hasBlockingMissingContext = (plan.missingContext ?? []).some((item) => item.severity === 'blocking');
    const failures = [];
    // Refusal expectation takes precedence: a refusal means route = missing_context.
    const expectRoute = testCase.expectRefuse ? 'missing_context' : testCase.expectRoute;
    let routeMatch;
    if (expectRoute !== undefined) {
        routeMatch = actualRoute === expectRoute;
        if (!routeMatch)
            failures.push(`route expected ${expectRoute}, got ${actualRoute}`);
    }
    let refusalMatch;
    if (testCase.expectRefuse !== undefined) {
        const actuallyRefused = actualRoute === 'missing_context';
        refusalMatch = actuallyRefused === testCase.expectRefuse;
        if (!refusalMatch) {
            failures.push(testCase.expectRefuse
                ? `expected a safe refusal, got route ${actualRoute}`
                : `expected no refusal, but router refused`);
        }
    }
    let blockMatch;
    if (testCase.expectBlock !== undefined) {
        blockMatch = actualBlock === testCase.expectBlock;
        if (!blockMatch) {
            failures.push(`block expected ${testCase.expectBlock}, got ${actualBlock ?? 'none'}`);
        }
    }
    let grainMatch;
    if (testCase.expectGrain !== undefined) {
        grainMatch = normalizeGrain(actualGrain) === normalizeGrain(testCase.expectGrain);
        if (!grainMatch) {
            failures.push(`grain expected ${testCase.expectGrain}, got ${actualGrain ?? 'none'}`);
        }
    }
    return {
        name: testCase.name,
        question: testCase.question,
        source: testCase.source,
        category: testCase.category,
        passed: failures.length === 0,
        actualRoute,
        actualBlock,
        actualGrain,
        hasBlockingMissingContext,
        expectRoute,
        expectBlock: testCase.expectBlock,
        expectGrain: testCase.expectGrain,
        expectRefuse: testCase.expectRefuse,
        routeMatch,
        blockMatch,
        grainMatch,
        refusalMatch,
        failures,
    };
}
function ratio(numerator, denominator) {
    if (denominator === 0)
        return null;
    return Number((numerator / denominator).toFixed(4));
}
/** Aggregate per-case results into the spec's score set. Pure + unit-testable. */
export function computeScores(results) {
    const routeCases = results.filter((r) => r.expectRoute !== undefined);
    const blockCases = results.filter((r) => r.expectBlock !== undefined);
    const grainCases = results.filter((r) => r.expectGrain !== undefined);
    const refusalExpected = results.filter((r) => r.expectRefuse === true);
    // Refusal precision: of every case where the router refused, how many should have?
    const routerRefused = results.filter((r) => r.actualRoute === 'missing_context');
    const routerRefusedCorrectly = routerRefused.filter((r) => r.expectRefuse === true);
    return {
        total: results.length,
        passed: results.filter((r) => r.passed).length,
        routeAccuracy: ratio(routeCases.filter((r) => r.routeMatch).length, routeCases.length),
        blockSelectionAccuracy: ratio(blockCases.filter((r) => r.blockMatch).length, blockCases.length),
        grainMatchPrecision: ratio(grainCases.filter((r) => r.grainMatch).length, grainCases.length),
        refusalPrecision: ratio(routerRefusedCorrectly.length, routerRefused.length),
        refusalRecall: ratio(refusalExpected.filter((r) => r.refusalMatch).length, refusalExpected.length),
    };
}
function normalizeYamlRoute(value) {
    if (!value)
        return undefined;
    const v = value.trim().toLowerCase();
    if (v === 'certified')
        return 'certified';
    if (v === 'generated' || v === 'generated_sql')
        return 'generated';
    if (v === 'missing_context' || v === 'missing-context' || v === 'clarify' || v === 'refuse') {
        return 'missing_context';
    }
    if (v === 'research')
        return 'research';
    return undefined;
}
function inferCategory(raw, route) {
    const explicit = raw.category?.trim().toLowerCase();
    if (explicit === 'certified')
        return 'certified';
    if (explicit === 'generated')
        return 'generated';
    if (explicit === 'insufficient_context' || explicit === 'refusal')
        return 'insufficient_context';
    if (explicit === 'conflict')
        return 'conflict';
    if (explicit === 'wrong_grain')
        return 'wrong_grain';
    // Infer from expectations when the author did not label it.
    if (raw.expectRefuse)
        return 'insufficient_context';
    if (route === 'missing_context')
        return 'insufficient_context';
    if (route === 'generated')
        return 'generated';
    return 'certified';
}
/** Load and normalize all `eval/*.yaml` cases for a project. */
export function loadYamlEvalCases(projectRoot) {
    const evalDir = join(projectRoot, 'eval');
    if (!existsSync(evalDir))
        return [];
    const files = readdirSync(evalDir).filter((f) => f.endsWith('.yaml') || f.endsWith('.yml'));
    const cases = [];
    for (const file of files.sort()) {
        const raw = loadYaml(readFileSync(join(evalDir, file), 'utf-8'));
        const rawCases = Array.isArray(raw) ? raw : raw?.cases ?? [];
        for (const rc of rawCases) {
            const question = rc.question?.trim();
            if (!question)
                continue;
            const route = normalizeYamlRoute(rc.expectRoute);
            cases.push({
                name: rc.name?.trim() || `${file} · ${question}`,
                question,
                source: 'yaml',
                category: inferCategory(rc, route),
                expectRoute: route,
                expectBlock: rc.expectBlock?.trim() || undefined,
                expectGrain: rc.expectGrain?.trim() || undefined,
                expectRefuse: typeof rc.expectRefuse === 'boolean' ? rc.expectRefuse : undefined,
            });
        }
    }
    return cases;
}
/**
 * Run the full eval: build (or read) the manifest, collect cases, replay each
 * through the router, score, and assemble the report. Side-effect free except
 * for invoking the (read-only) router. Returns a structured report.
 */
export async function runEvalHarness(projectRoot, options) {
    const manifest = buildManifest({
        projectRoot,
        dbtManifestPath: resolveDbtManifestPath(projectRoot) ?? undefined,
    });
    const yamlCases = loadYamlEvalCases(projectRoot);
    const cases = collectEvalCases(manifest, yamlCases, options.includeBlockExamples);
    const results = [];
    for (const testCase of cases) {
        const plan = await planAgentAnswer(projectRoot, {
            question: testCase.question,
            surface: 'cli',
        });
        results.push(scoreCase(testCase, plan, manifest));
    }
    const scores = computeScores(results.length > 0 ? results : ZERO_RESULTS);
    const ok = meetsThresholds(scores, options.minRouteAccuracy, options.minRefusal);
    return {
        ok,
        scores,
        thresholds: { minRouteAccuracy: options.minRouteAccuracy, minRefusal: options.minRefusal },
        results,
    };
}
/** A threshold gate only fails when a configured threshold has measurable data below it. */
export function meetsThresholds(scores, minRouteAccuracy, minRefusal) {
    if (minRouteAccuracy !== null &&
        scores.routeAccuracy !== null &&
        scores.routeAccuracy < minRouteAccuracy) {
        return false;
    }
    if (minRefusal !== null && scores.refusalRecall !== null && scores.refusalRecall < minRefusal) {
        return false;
    }
    return true;
}
function formatRate(value) {
    return value === null ? 'n/a' : `${Math.round(value * 1000) / 10}%`;
}
export async function runEval(pathArg, rest, flags) {
    const allArgs = [...(pathArg ? [pathArg] : []), ...rest];
    const positional = allArgs.filter((a) => !a.startsWith('-'));
    const projectRoot = resolve(positional[0] ?? '.');
    if (!existsSync(join(projectRoot, 'dql.config.json'))) {
        console.error('No DQL project found (missing dql.config.json). Run from a project root or pass a project path.');
        process.exitCode = 1;
        return;
    }
    const includeBlockExamples = !flags.noExamples;
    const minRouteAccuracy = flags.minRouteAccuracy ?? null;
    const minRefusal = flags.minRefusal ?? null;
    const report = await runEvalHarness(projectRoot, {
        includeBlockExamples,
        minRouteAccuracy,
        minRefusal,
    });
    if (flags.format === 'json') {
        console.log(JSON.stringify(report, null, 2));
        if (!report.ok)
            process.exitCode = 1;
        return;
    }
    printReport(report);
    if (!report.ok)
        process.exitCode = 1;
}
function printReport(report) {
    const { results, scores } = report;
    console.log('\n  DQL Eval — routing accuracy');
    console.log('  ' + '='.repeat(50));
    if (results.length === 0) {
        console.log('\n  No eval cases found.');
        console.log('  Add `examples = [{ question = "..." }]` to certified blocks,');
        console.log('  or create eval/*.yaml cases, then run `dql compile` and retry.\n');
        return;
    }
    console.log('');
    for (const result of results) {
        const icon = result.passed ? '✓' : '✕';
        const route = `[${result.actualRoute}]`;
        console.log(`  ${icon} ${route} ${result.name}`);
        for (const failure of result.failures) {
            console.log(`      - ${failure}`);
        }
    }
    console.log('\n  Scores');
    console.log('  ' + '-'.repeat(50));
    console.log(`  Cases passed:            ${scores.passed}/${scores.total}`);
    console.log(`  Route accuracy:          ${formatRate(scores.routeAccuracy)}`);
    console.log(`  Block-selection accuracy:${formatRate(scores.blockSelectionAccuracy).padStart(7)}`);
    console.log(`  Grain-match precision:   ${formatRate(scores.grainMatchPrecision)}`);
    console.log(`  Refusal precision:       ${formatRate(scores.refusalPrecision)}`);
    console.log(`  Refusal recall:          ${formatRate(scores.refusalRecall)}`);
    const t = report.thresholds;
    if (t.minRouteAccuracy !== null || t.minRefusal !== null) {
        console.log('\n  Thresholds');
        console.log('  ' + '-'.repeat(50));
        if (t.minRouteAccuracy !== null) {
            console.log(`  --min-route-accuracy ${t.minRouteAccuracy}  (route accuracy ${formatRate(scores.routeAccuracy)})`);
        }
        if (t.minRefusal !== null) {
            console.log(`  --min-refusal ${t.minRefusal}  (refusal recall ${formatRate(scores.refusalRecall)})`);
        }
        console.log(`\n  ${report.ok ? '✓ PASS — thresholds met.' : '✕ FAIL — below configured thresholds.'}`);
    }
    console.log('');
}
//# sourceMappingURL=eval.js.map