/**
 * `dql eval` — Routing-accuracy harness for the DQL answer agent.
 *
 * Replays a golden set of questions through the EXISTING agent router
 * (`planAgentAnswer`) and scores how well DQL routes each question:
 * certified vs generated vs missing-context (refusal), which certified block
 * it selected, and whether the answer grain matches.
 *
 * The golden set comes from two sources:
 *   1. Every certified block's `examples[].question` in the compiled manifest.
 *      These expect route = certified and expect that same block to be selected.
 *   2. Optional `eval/*.yaml` files in the project, each a list (or { cases: [] })
 *      of `{ question, expectRoute, expectBlock?, expectGrain?, expectRefuse?, name? }`.
 *      These let a repo express the harder categories the block examples cannot:
 *      generated (Tier-2), insufficient-context (refusal), conflict, wrong-grain.
 *
 * This command READS the router output only. It never executes SQL against a
 * warehouse and never changes routing, the manifest schema, or the DQL language.
 *
 * Usage:
 *   dql eval [path]                          Replay the golden set, print a report
 *   dql eval [path] --format json            Emit a machine-readable JSON report
 *   dql eval [path] --min-route-accuracy 0.9 Fail (exit 1) below this route accuracy
 *   dql eval [path] --min-refusal 0.8        Fail (exit 1) below this refusal recall
 *   dql eval [path] --no-examples            Skip manifest block examples (yaml only)
 */
import { type DQLManifest } from '@duckcodeailabs/dql-core';
import { type PlanAgentAnswerResult } from '@duckcodeailabs/dql-agent';
import type { CLIFlags } from '../args.js';
/** Spec-facing route labels. The router speaks certified/generated_sql/research/clarify. */
export type EvalRoute = 'certified' | 'generated' | 'missing_context' | 'research';
/** The five categories the harness can express and score. */
export type EvalCategory = 'certified' | 'generated' | 'insufficient_context' | 'conflict' | 'wrong_grain';
export interface EvalCase {
    /** Stable label for the report. Defaults to the question. */
    name: string;
    question: string;
    /** Where this case came from. */
    source: 'block_example' | 'yaml';
    /** Which scoring category this case exercises. */
    category: EvalCategory;
    expectRoute?: EvalRoute;
    /** Block name we expect the router to select (only meaningful for certified). */
    expectBlock?: string;
    /** Grain we expect the selected block to carry. */
    expectGrain?: string;
    /** When true, we expect a safe refusal (route = missing_context). */
    expectRefuse?: boolean;
}
export interface EvalCaseResult {
    name: string;
    question: string;
    source: EvalCase['source'];
    category: EvalCategory;
    passed: boolean;
    /** What the router actually did. */
    actualRoute: EvalRoute;
    actualBlock?: string;
    actualGrain?: string;
    hasBlockingMissingContext: boolean;
    /** Per-check expectations + outcomes for the human report and JSON. */
    expectRoute?: EvalRoute;
    expectBlock?: string;
    expectGrain?: string;
    expectRefuse?: boolean;
    routeMatch?: boolean;
    blockMatch?: boolean;
    grainMatch?: boolean;
    refusalMatch?: boolean;
    failures: string[];
}
export interface EvalScores {
    total: number;
    passed: number;
    /** route accuracy: cases whose expected route matched / cases with a route expectation. */
    routeAccuracy: number | null;
    /** block-selection accuracy: correct block / cases that expected a specific block. */
    blockSelectionAccuracy: number | null;
    /** grain-match precision: correct grain / cases that expected a specific grain. */
    grainMatchPrecision: number | null;
    /** refusal precision: true refusals / all router refusals across the set. */
    refusalPrecision: number | null;
    /** refusal recall: refusals correctly produced / cases that expected a refusal. */
    refusalRecall: number | null;
}
export interface EvalReport {
    ok: boolean;
    scores: EvalScores;
    thresholds: {
        minRouteAccuracy: number | null;
        minRefusal: number | null;
    };
    results: EvalCaseResult[];
}
/** Map the router's route vocabulary onto the spec's route labels. */
export declare function mapRoute(route: PlanAgentAnswerResult['routeDecision']['route']): EvalRoute;
/**
 * Build the golden set from a compiled manifest plus optional eval/*.yaml.
 * Each certified block contributes one case per example question.
 */
export declare function collectEvalCases(manifest: DQLManifest, yamlCases: EvalCase[], includeBlockExamples: boolean): EvalCase[];
/** Score a single case against the router's decision. Pure + unit-testable. */
export declare function scoreCase(testCase: EvalCase, plan: PlanAgentAnswerResult, manifest: DQLManifest): EvalCaseResult;
/** Aggregate per-case results into the spec's score set. Pure + unit-testable. */
export declare function computeScores(results: EvalCaseResult[]): EvalScores;
/** Load and normalize all `eval/*.yaml` cases for a project. */
export declare function loadYamlEvalCases(projectRoot: string): EvalCase[];
/**
 * Run the full eval: build (or read) the manifest, collect cases, replay each
 * through the router, score, and assemble the report. Side-effect free except
 * for invoking the (read-only) router. Returns a structured report.
 */
export declare function runEvalHarness(projectRoot: string, options: {
    includeBlockExamples: boolean;
    minRouteAccuracy: number | null;
    minRefusal: number | null;
}): Promise<EvalReport>;
/** A threshold gate only fails when a configured threshold has measurable data below it. */
export declare function meetsThresholds(scores: EvalScores, minRouteAccuracy: number | null, minRefusal: number | null): boolean;
export declare function runEval(pathArg: string | null, rest: string[], flags: CLIFlags): Promise<void>;
//# sourceMappingURL=eval.d.ts.map