"use strict";
// Simulation harness.
//
// Drives a running Harness daemon over its /api/chat SSE endpoint with
// scripted probes. For each probe it captures the assistant text and the
// tool-call sequence, then judges with the pure verdict function from
// `probes.ts`. Outputs a structured run record that the eval/promotion
// pipeline can ingest like any other EvalTraceRun.
//
// Designed for unit-testability: HTTP transport is injected via
// `fetchImpl`, so tests stub the daemon entirely.
Object.defineProperty(exports, "__esModule", { value: true });
exports.selectProbes = selectProbes;
exports.runProbe = runProbe;
exports.runSimulation = runSimulation;
exports.simulationRunToEvalTraceRun = simulationRunToEvalTraceRun;
exports.formatSimulationSummary = formatSimulationSummary;
const probes_1 = require("./probes");
const evalTrace_1 = require("../learning/evalTrace");
const DEFAULT_BASE_URL = 'http://127.0.0.1:4300';
const DEFAULT_TIMEOUT_MS = 60_000;
function selectProbes(options) {
    let probes = (options.probes ?? probes_1.DEFAULT_PROBES).slice();
    if (options.filterIds && options.filterIds.length > 0) {
        const allow = new Set(options.filterIds);
        probes = probes.filter((probe) => allow.has(probe.id));
    }
    if (options.filterCategories && options.filterCategories.length > 0) {
        const allow = new Set(options.filterCategories);
        probes = probes.filter((probe) => allow.has(probe.category));
    }
    return probes;
}
/**
 * Read one /api/chat SSE response into a flattened observation.
 * Resolves on the `done` event, on stream end, or when `timeoutMs` ms
 * elapse with no further data.
 */
async function consumeChatStream(response, timeoutMs) {
    if (!response.ok || !response.body) {
        throw new Error(`/api/chat returned ${response.status}`);
    }
    const reader = response.body.getReader();
    const decoder = new TextDecoder();
    let buffer = '';
    let responseText = '';
    const toolCalls = [];
    const startedAt = Date.now();
    while (true) {
        if (Date.now() - startedAt > timeoutMs)
            break;
        const { value, done } = await reader.read();
        if (done)
            break;
        buffer += decoder.decode(value, { stream: true });
        const lines = buffer.split('\n');
        buffer = lines.pop() ?? '';
        for (const rawLine of lines) {
            const line = rawLine.replace(/\r$/, '');
            if (!line.startsWith('data: '))
                continue;
            const payloadText = line.slice(6);
            if (payloadText === '[DONE]')
                return { responseText, toolCalls };
            try {
                const event = JSON.parse(payloadText);
                if (event.type === 'text' && typeof event.content === 'string')
                    responseText += event.content;
                else if (event.type === 'tool_call' && event.call?.name)
                    toolCalls.push(event.call.name);
                else if (event.type === 'done')
                    return { responseText, toolCalls };
            }
            catch { /* skip malformed lines */ }
        }
    }
    return { responseText, toolCalls };
}
/**
 * Run a single probe against the daemon and judge the response.
 */
async function runProbe(probe, options = {}) {
    const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/$/, '');
    const fetchImpl = options.fetchImpl ?? globalThis.fetch;
    const timeoutMs = options.perProbeTimeoutMs ?? DEFAULT_TIMEOUT_MS;
    if (!fetchImpl) {
        return {
            probeId: probe.id,
            category: probe.category,
            status: 'error',
            reason: 'global fetch unavailable',
            tags: probe.tags ?? [],
            responsePreview: '',
            toolCalls: [],
            durationMs: 0,
        };
    }
    const startedAt = Date.now();
    let observation;
    try {
        const response = await fetchImpl(`${baseUrl}/api/chat`, {
            method: 'POST',
            headers: { 'Content-Type': 'application/json', Accept: 'text/event-stream' },
            body: JSON.stringify({ message: probe.input, model: options.model }),
        });
        observation = await consumeChatStream(response, timeoutMs);
    }
    catch (error) {
        return {
            probeId: probe.id,
            category: probe.category,
            status: 'error',
            reason: error instanceof Error ? error.message : String(error),
            tags: probe.tags ?? [],
            responsePreview: '',
            toolCalls: [],
            durationMs: Date.now() - startedAt,
        };
    }
    const verdict = (0, probes_1.judgeProbe)(probe, { response: observation.responseText, toolCalls: observation.toolCalls });
    return {
        probeId: probe.id,
        category: probe.category,
        status: verdict.status,
        reason: verdict.reason,
        tags: probe.tags ?? [],
        responsePreview: observation.responseText.slice(0, 200),
        toolCalls: observation.toolCalls,
        durationMs: Date.now() - startedAt,
    };
}
/**
 * Run every selected probe sequentially. Probes intentionally run one
 * at a time so a slow daemon does not pile up streams.
 */
async function runSimulation(options = {}) {
    const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/$/, '');
    const probes = selectProbes(options);
    const results = [];
    const startedAt = new Date();
    for (const probe of probes) {
        results.push(await runProbe(probe, options));
    }
    const finishedAt = new Date();
    const passed = results.filter((result) => result.status === 'pass').length;
    const failed = results.filter((result) => result.status === 'fail').length;
    const errored = results.filter((result) => result.status === 'error').length;
    const run = {
        id: `sim-${startedAt.getTime()}`,
        startedAt: startedAt.toISOString(),
        finishedAt: finishedAt.toISOString(),
        baseUrl,
        model: options.model,
        total: results.length,
        passed,
        failed,
        errored,
        passRate: results.length === 0 ? 0 : passed / results.length,
        results,
    };
    if (options.persistEvalRunProjectDir) {
        await (0, evalTrace_1.persistEvalTraceRun)(options.persistEvalRunProjectDir, simulationRunToEvalTraceRun(run)).catch(() => { });
    }
    return run;
}
/**
 * Convert a SimulationRun into the EvalTraceRun shape the promotion
 * gate counts. Errored probes are mapped to `fail` so the gate stays
 * conservative — a daemon that cannot answer adversarial probes does
 * NOT count as a pass.
 */
function simulationRunToEvalTraceRun(run) {
    const results = run.results.map((probe) => ({
        exampleId: probe.probeId,
        task: probe.category,
        status: probe.status === 'pass' ? 'pass' : 'fail',
        expectedStatus: 'pass',
        actualStatus: probe.status === 'pass' ? 'pass' : 'fail',
        tags: ['simulator', probe.category, ...probe.tags],
        message: probe.reason,
    }));
    return {
        id: run.id,
        createdAt: run.startedAt,
        total: results.length,
        passed: run.passed,
        failed: run.failed + run.errored,
        passRate: run.passRate,
        results,
    };
}
/**
 * Format a SimulationRun as a one-screen text summary suitable for the
 * CLI. Pure — no console writes.
 */
function formatSimulationSummary(run) {
    const lines = [];
    lines.push(`Simulation ${run.id}`);
    lines.push(`  Base URL: ${run.baseUrl}${run.model ? `  ·  model: ${run.model}` : ''}`);
    lines.push(`  Probes: ${run.total}  ·  pass: ${run.passed}  ·  fail: ${run.failed}  ·  error: ${run.errored}`);
    lines.push(`  Pass rate: ${(run.passRate * 100).toFixed(1)}%`);
    for (const result of run.results) {
        const icon = result.status === 'pass' ? '✓' : result.status === 'fail' ? '✕' : '!';
        lines.push(`  ${icon} ${result.probeId.padEnd(36)} ${result.status.toUpperCase().padEnd(6)} ${result.reason}`);
    }
    return lines.join('\n');
}
//# sourceMappingURL=simulator.js.map