/**
 * Eval scenario runner — hits AI endpoints and scores outputs using the judge engine.
 *
 * Eval scenarios store their config in scenario.metadata as EvalScenarioConfig.
 * Each test case runs against the endpoint and is scored by one or more rubrics.
 * Results are stored in the standard results table with per-case scores in metadata.
 */
import type { Scenario, Result } from "../types/index.js";
import type { JudgeRubric, JudgeResult } from "./judge.js";
export interface EvalTestCase {
    input: string;
    context?: string;
    rubrics: JudgeRubric[];
}
export interface EvalScenarioConfig {
    endpoint: string;
    method?: string;
    headers?: Record<string, string>;
    inputField?: string;
    outputField?: string;
    testCases: EvalTestCase[];
    judgeModel?: string;
    judgeProvider?: string;
    baseUrl?: string;
}
export interface EvalCaseResult {
    input: string;
    output: string | null;
    rubricResults: Array<{
        rubricType: string;
        pass: boolean;
        score: number;
        reason: string;
    }>;
    passed: boolean;
    score: number;
    error?: string;
}
export interface EvalRunResult {
    passed: boolean;
    totalCases: number;
    passedCases: number;
    avgScore: number;
    caseResults: EvalCaseResult[];
    tokensUsed: number;
    durationMs: number;
}
export declare function runEvalScenario(scenario: Scenario, options: {
    runId: string;
    baseUrl: string;
}): Promise<Result>;
export interface RagTestCase {
    question: string;
    sourceDocs: string[];
    expectedFacts?: string[];
    forbiddenClaims?: string[];
}
export interface RagEvalConfig extends EvalScenarioConfig {
    ragTestCases: RagTestCase[];
}
export interface RagCaseResult {
    question: string;
    output: string | null;
    faithfulnessScore: number;
    faithfulnessPass: boolean;
    factualCompletenessScore: number;
    factualCompletenessPass: boolean;
    forbiddenClaimViolations: string[];
    passed: boolean;
    error?: string;
    judgeResults: JudgeResult[];
}
export interface RagEvalResult {
    passed: boolean;
    totalCases: number;
    passedCases: number;
    avgFaithfulnessScore: number;
    avgFactualCompletenessScore: number;
    totalForbiddenViolations: number;
    caseResults: RagCaseResult[];
    tokensUsed: number;
    durationMs: number;
}
export declare function runRagEval(scenario: Scenario, options: {
    runId: string;
    baseUrl: string;
}): Promise<Result>;
export declare function runPipelineScenario(scenario: Scenario, options: {
    runId: string;
    baseUrl: string;
}): Promise<Result>;
export { runPipeline } from "./pipeline-runner.js";
export type { PipelineConfig, PipelineStep, PipelineRunResult, PipelineStepResult } from "./pipeline-runner.js";
//# sourceMappingURL=eval-runner.d.ts.map