// ---------------------------------------------------------------------------
// benchmark-runner.ts -- SSE benchmark execution (single-case + bulk)
// ---------------------------------------------------------------------------
import { estimateDurationSec } from "../eval/llm.js";
import { judgeAssertion } from "../eval/judge.js";
import { writeHistoryEntry } from "../eval/benchmark-history.js";
import { sendSSE, sendSSEDone, withHeartbeat } from "./sse-helpers.js";
import { classifyError } from "./error-classifier.js";
import { Semaphore } from "./concurrency.js";
// ---------------------------------------------------------------------------
// CLI providers that spawn child processes — default concurrency 1
// ---------------------------------------------------------------------------
const CLI_PROVIDERS = new Set(["claude-cli", "codex-cli", "gemini-cli"]);
function defaultConcurrency(provider) {
    return CLI_PROVIDERS.has(provider) ? 1 : 5;
}
// ---------------------------------------------------------------------------
// 429 retry helper — exponential backoff (1s, 2s, 4s), max 3 retries
// ---------------------------------------------------------------------------
function is429(err) {
    if (err instanceof Error) {
        const msg = err.message;
        if (msg.includes("429") || msg.includes("rate limit") || msg.includes("Rate limit"))
            return true;
        if ("status" in err && err.status === 429)
            return true;
    }
    return false;
}
async function sleep(ms) {
    return new Promise((resolve) => setTimeout(resolve, ms));
}
// ---------------------------------------------------------------------------
// Weak-model judge warning heuristic
// ---------------------------------------------------------------------------
const MODEL_STRENGTH = {
    haiku: 1,
    flash: 1,
    "mini": 1,
    sonnet: 2,
    pro: 2,
    opus: 3,
};
function estimateStrength(model) {
    const lower = model.toLowerCase();
    for (const [key, strength] of Object.entries(MODEL_STRENGTH)) {
        if (lower.includes(key))
            return strength;
    }
    return 2; // unknown = medium
}
export function checkWeakJudgeWarning(generatorModel, judgeModel) {
    const genStrength = estimateStrength(generatorModel);
    const judgeStrength = estimateStrength(judgeModel);
    if (judgeStrength < genStrength) {
        return `Warning: Judge model "${judgeModel}" appears weaker than generator "${generatorModel}". Judge scores may be less reliable.`;
    }
    return null;
}
export async function runSingleCaseSSE(opts) {
    const { res, evalCase, systemPrompt, client, judgeClient, judgeCache, isAborted, totalCases = 1, provider } = opts;
    const effectiveJudgeClient = judgeClient ?? client;
    let sequence = 0;
    const emitSSE = (event, data) => {
        sendSSE(res, event, { ...data, caseId: evalCase.id, sequence: sequence++ });
    };
    emitSSE("case_start", {
        eval_id: evalCase.id,
        eval_name: evalCase.name,
        total: totalCases,
    });
    emitSSE("progress", {
        eval_id: evalCase.id,
        phase: "generating",
        message: `Generating LLM response for "${evalCase.name}"...`,
    });
    try {
        // Retry with exponential backoff on 429
        let genResult;
        const MAX_RETRIES = 3;
        const RETRY_DELAYS = [1000, 2000, 4000];
        for (let attempt = 0;; attempt++) {
            try {
                genResult = await withHeartbeat(res, evalCase.id, "generating", `Generating LLM response for "${evalCase.name}"...`, () => client.generate(systemPrompt, evalCase.prompt));
                break;
            }
            catch (err) {
                if (is429(err) && attempt < MAX_RETRIES) {
                    const delayMs = RETRY_DELAYS[attempt];
                    emitSSE("warning", {
                        eval_id: evalCase.id,
                        message: `Rate limited (429). Retrying in ${delayMs / 1000}s (attempt ${attempt + 1}/${MAX_RETRIES})...`,
                        attempt: attempt + 1,
                        delayMs,
                    });
                    await sleep(delayMs);
                    continue;
                }
                throw err;
            }
        }
        const totalTokens = genResult.inputTokens != null && genResult.outputTokens != null
            ? genResult.inputTokens + genResult.outputTokens
            : null;
        emitSSE("output_ready", {
            eval_id: evalCase.id,
            output: genResult.text,
            durationMs: genResult.durationMs,
            tokens: totalTokens,
            inputTokens: genResult.inputTokens ?? null,
            outputTokens: genResult.outputTokens ?? null,
        });
        emitSSE("progress", {
            eval_id: evalCase.id,
            phase: "judging",
            message: `Evaluating ${evalCase.assertions.length} assertion${evalCase.assertions.length !== 1 ? "s" : ""}...`,
            total: evalCase.assertions.length,
        });
        // T-003: Parallelize intra-case assertion judges via Promise.all
        const assertionResults = await Promise.all(evalCase.assertions.map(async (assertion, ai) => {
            if (isAborted()) {
                return { id: assertion.id, text: assertion.text, pass: false, reasoning: "aborted" };
            }
            const judgeCall = () => judgeAssertion(genResult.text, assertion, client, effectiveJudgeClient);
            let result;
            if (judgeCache) {
                result = await judgeCache.getOrCompute(assertion.text, genResult.text, effectiveJudgeClient.model, judgeCall);
            }
            else {
                result = await judgeCall();
            }
            emitSSE("assertion_result", {
                eval_id: evalCase.id,
                assertion_id: result.id,
                text: result.text,
                pass: result.pass,
                reasoning: result.reasoning,
                assertion_index: ai,
            });
            return result;
        }));
        const passRate = assertionResults.length > 0
            ? assertionResults.filter((a) => a.pass).length / assertionResults.length
            : 0;
        const status = assertionResults.length > 0 && assertionResults.every((a) => a.pass) ? "pass" : "fail";
        const benchCase = {
            eval_id: evalCase.id,
            eval_name: evalCase.name,
            status: status,
            error_message: null,
            pass_rate: passRate,
            durationMs: genResult.durationMs,
            tokens: totalTokens,
            inputTokens: genResult.inputTokens,
            outputTokens: genResult.outputTokens,
            cost: genResult.cost,
            billingMode: genResult.billingMode,
            output: genResult.text,
            assertions: assertionResults,
        };
        emitSSE("case_complete", {
            eval_id: evalCase.id,
            status,
            pass_rate: passRate,
            durationMs: genResult.durationMs,
            tokens: totalTokens,
            cost: genResult.cost,
            billing_mode: genResult.billingMode,
        });
        return benchCase;
    }
    catch (err) {
        const errorMsg = err instanceof Error ? err.message : String(err);
        const classified = classifyError(err, provider);
        const errorCase = {
            eval_id: evalCase.id,
            eval_name: evalCase.name,
            status: "error",
            error_message: errorMsg,
            pass_rate: 0,
            assertions: [],
        };
        emitSSE("case_complete", {
            eval_id: evalCase.id,
            status: "error",
            error_message: errorMsg,
            classified_error: classified,
        });
        return errorCase;
    }
}
// ---------------------------------------------------------------------------
// Assemble bulk result from individual case results
// ---------------------------------------------------------------------------
export function assembleBulkResult(cases, meta) {
    const totalAssertions = cases.reduce((s, c) => s + c.assertions.length, 0);
    const passedAssertions = cases.reduce((s, c) => s + c.assertions.filter((a) => a.pass).length, 0);
    const totalDurationMs = cases.reduce((s, c) => s + (c.durationMs ?? 0), 0);
    const hasTokens = cases.some((c) => c.inputTokens != null);
    const hasCost = cases.some((c) => c.cost != null);
    const totalCost = hasCost ? cases.reduce((s, c) => s + (c.cost ?? 0), 0) : null;
    return {
        timestamp: new Date().toISOString(),
        model: meta.model,
        skill_name: meta.skillName,
        cases,
        overall_pass_rate: totalAssertions > 0 ? passedAssertions / totalAssertions : 0,
        type: meta.runType,
        provider: meta.provider,
        totalDurationMs,
        totalInputTokens: hasTokens ? cases.reduce((s, c) => s + (c.inputTokens ?? 0), 0) : null,
        totalOutputTokens: hasTokens ? cases.reduce((s, c) => s + (c.outputTokens ?? 0), 0) : null,
        totalCost,
        scope: "bulk",
    };
}
export async function runBenchmarkSSE(opts) {
    const { res, skillDir, skillName, systemPrompt, runType, provider, evalCases: allCases, filterIds, client, judgeClient, judgeCache, isAborted, concurrency: concurrencyOverride, } = opts;
    const evalCases = filterIds
        ? allCases.filter((e) => filterIds.has(e.id))
        : allCases;
    const concurrency = concurrencyOverride ?? defaultConcurrency(provider);
    // Emit duration estimate so the UI can warn about long runs
    const totalAssertions = evalCases.reduce((s, e) => s + e.assertions.length, 0);
    const estimate = estimateDurationSec(provider, evalCases.length, totalAssertions);
    sendSSE(res, "estimate", {
        totalCases: evalCases.length,
        totalAssertions,
        estimatedMinSec: estimate.minSec,
        estimatedMaxSec: estimate.maxSec,
        estimatedLabel: estimate.label,
        concurrency,
    });
    // Emit weak-model judge warning if applicable
    if (judgeClient) {
        const warning = checkWeakJudgeWarning(client.model, judgeClient.model);
        if (warning) {
            console.warn(warning);
            sendSSE(res, "warning", { message: warning });
        }
    }
    // T-001: Parallel case execution via Semaphore + Promise.allSettled
    const semaphore = new Semaphore(concurrency);
    const caseTasks = evalCases.map((evalCase) => async () => {
        if (isAborted())
            return null;
        await semaphore.acquire();
        try {
            return await runSingleCaseSSE({
                res,
                evalCase,
                systemPrompt,
                client,
                judgeClient,
                judgeCache,
                isAborted,
                totalCases: evalCases.length,
                provider,
            });
        }
        finally {
            semaphore.release();
        }
    });
    const settled = await Promise.allSettled(caseTasks.map((fn) => fn()));
    const cases = [];
    for (const result of settled) {
        if (result.status === "fulfilled" && result.value != null) {
            cases.push(result.value);
        }
        else if (result.status === "rejected") {
            // Shouldn't normally happen since runSingleCaseSSE catches errors,
            // but handle as safety net
            cases.push({
                eval_id: -1,
                eval_name: "unknown",
                status: "error",
                error_message: result.reason instanceof Error ? result.reason.message : String(result.reason),
                pass_rate: 0,
                assertions: [],
            });
        }
    }
    const bulkResult = assembleBulkResult(cases, { model: client.model, skillName, runType, provider });
    if (!isAborted()) {
        // Save history for bulk runs (single-case runs save via per-case endpoint)
        if (!filterIds) {
            await writeHistoryEntry(skillDir, bulkResult);
        }
        sendSSEDone(res, bulkResult);
    }
}
//# sourceMappingURL=benchmark-runner.js.map