/**
 * Behavioral eval framework — EvalScenario + runEval + LLM scorer.
 *
 * Sprint-563: B6 — EvalScenario + LLM scorer + Vitest integration.
 *
 * Setup an agent scenario with assertions on the output events.
 * Optionally score the result with an LLM judge.
 * Integrates with Vitest for regression gating in CI.
 */
// ─── Runner ──────────────────────────────────────────────────────────────────
/**
 * Run a single eval scenario. Triggers one cycle via streamCycle() and
 * validates assertions + optional LLM scorer.
 */
export async function runEval(scenario, opts) {
    const start = Date.now();
    const events = [];
    await scenario.setup?.();
    // Stream one cycle in dry-run mode
    const stream = scenario.agent.streamCycle({ dryRun: true });
    for await (const event of stream) {
        events.push(event);
    }
    const durationMs = Date.now() - start;
    let passed = true;
    let assertionFailure;
    // Status check
    const finalStatus = events.find((e) => e.type === "cycle_complete" || e.type === "cycle_error");
    const status = finalStatus?.type === "cycle_complete"
        ? finalStatus.status
        : "error";
    if (scenario.expectedStatus && status !== scenario.expectedStatus) {
        passed = false;
        assertionFailure = `Expected status "${scenario.expectedStatus}", got "${status}"`;
    }
    // Custom assertions
    if (passed && scenario.assertions) {
        try {
            const result = await scenario.assertions(events);
            if (!result) {
                passed = false;
                assertionFailure = "Custom assertions failed";
            }
        }
        catch (err) {
            passed = false;
            assertionFailure = `Assertion error: ${err instanceof Error ? err.message : String(err)}`;
        }
    }
    // LLM scorer
    let llmScore;
    if (passed && opts?.llmScorer && scenario.minScore !== undefined) {
        llmScore = await opts.llmScorer.score(events, scenario);
        if (llmScore < scenario.minScore) {
            passed = false;
            assertionFailure = `LLM score ${llmScore} below minimum ${scenario.minScore}`;
        }
    }
    return {
        name: scenario.name,
        passed,
        events,
        status,
        assertionFailure,
        llmScore,
        durationMs,
    };
}
/**
 * Run a suite of eval scenarios. Returns aggregate results.
 */
export async function runEvalSuite(scenarios, opts) {
    const results = [];
    for (const scenario of scenarios) {
        results.push(await runEval(scenario, opts));
    }
    return {
        results,
        passed: results.filter((r) => r.passed).length,
        failed: results.filter((r) => !r.passed).length,
        total: results.length,
    };
}
// ─── Simple pass/fail LLM scorer (no LLM dependency in core) ─────────────────
/**
 * A basic scorer that checks if the expected number of phase events occurred.
 * Production use should inject a real LLM judge via the LLMScorer interface.
 */
export function createPhaseCountScorer(expectedPhases) {
    return {
        async score(events) {
            const phaseStarts = events.filter((e) => e.type === "phase_start");
            const matchCount = expectedPhases.filter((p) => phaseStarts.some((e) => "phase" in e && e.phase === p)).length;
            return Math.round((matchCount / expectedPhases.length) * 100);
        },
    };
}
//# sourceMappingURL=eval.js.map