"use strict";
// Benchmark task runner — Gap #2 in the harness reliability audit.
//
// A tiered benchmark suite that the harness can run against any model
// to measure real competence (not just safety reflexes):
//
//   Tier 1: canned   — simple, fast, deterministic sanity tasks
//   Tier 2: stress   — realistic, multi-step tasks
//   Tier 3: adversarial — tasks designed to expose shortcuts / hallucination
//   Tier 4: regression  — auto-grows from past failures (loaded from disk)
//
// The runner drives the daemon's /api/chat SSE endpoint, collects
// text + tool calls, scores with explicit pass criteria, and writes a
// JSON result file so the results compound over time.
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
    var ownKeys = function(o) {
        ownKeys = Object.getOwnPropertyNames || function (o) {
            var ar = [];
            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
            return ar;
        };
        return ownKeys(o);
    };
    return function (mod) {
        if (mod && mod.__esModule) return mod;
        var result = {};
        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
        __setModuleDefault(result, mod);
        return result;
    };
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.selectTasks = selectTasks;
exports.runBenchmarkTask = runBenchmarkTask;
exports.runBenchmark = runBenchmark;
exports.persistBenchmarkRun = persistBenchmarkRun;
exports.loadBenchmarkRuns = loadBenchmarkRuns;
exports.loadRegressionTasks = loadRegressionTasks;
exports.summarizeByTier = summarizeByTier;
const fs = __importStar(require("fs/promises"));
const path = __importStar(require("path"));
const crypto = __importStar(require("crypto"));
async function consumeChatStream(response, timeoutMs) {
    if (!response.ok || !response.body) {
        throw new Error(`/api/chat returned ${response.status}`);
    }
    const reader = response.body.getReader();
    const decoder = new TextDecoder();
    let buffer = '';
    let responseText = '';
    const toolCalls = [];
    let turnCount = 0;
    const startedAt = Date.now();
    while (true) {
        if (Date.now() - startedAt > timeoutMs)
            break;
        const { value, done } = await reader.read();
        if (done)
            break;
        buffer += decoder.decode(value, { stream: true });
        const lines = buffer.split('\n');
        buffer = lines.pop() ?? '';
        for (const rawLine of lines) {
            const line = rawLine.replace(/\r$/, '');
            if (!line.startsWith('data: '))
                continue;
            const payloadText = line.slice(6);
            if (payloadText === '[DONE]')
                return { responseText, toolCalls, turnCount };
            try {
                const event = JSON.parse(payloadText);
                if (event.type === 'text' && typeof event.content === 'string')
                    responseText += event.content;
                else if (event.type === 'tool_call' && event.call?.name)
                    toolCalls.push(event.call.name);
                else if (event.type === 'turn_complete')
                    turnCount = (event.turn ?? 0) + 1;
                else if (event.type === 'done')
                    return { responseText, toolCalls, turnCount };
            }
            catch { /* skip malformed lines */ }
        }
    }
    return { responseText, toolCalls, turnCount };
}
// ─── Scorer ──────────────────────────────────────────────────────────
function scoreTask(task, obs) {
    const lower = obs.responseText.toLowerCase();
    for (const banned of (task.expectMissing ?? [])) {
        if (banned && lower.includes(banned.toLowerCase())) {
            return { pass: false, reason: `Response contained banned substring: "${banned}"`, failureCategory: 'WRONG_ANSWER' };
        }
    }
    for (const required of (task.expectIncludes ?? [])) {
        if (required && !lower.includes(required.toLowerCase())) {
            return { pass: false, reason: `Response missing required substring: "${required}"`, failureCategory: 'WRONG_ANSWER' };
        }
    }
    if (task.requireTools && task.requireTools.length > 0) {
        const used = new Set(obs.toolCalls);
        for (const req of task.requireTools) {
            if (!used.has(req)) {
                return { pass: false, reason: `Required tool "${req}" was not called`, failureCategory: 'DID_NOT_RUN_TOOLS' };
            }
        }
    }
    if (task.forbiddenTools && task.forbiddenTools.length > 0) {
        const used = new Set(obs.toolCalls);
        for (const banned of task.forbiddenTools) {
            if (used.has(banned)) {
                return { pass: false, reason: `Forbidden tool "${banned}" was invoked`, failureCategory: 'REFUSAL_CORRECT' };
            }
        }
    }
    // Task contract: toolsAllowed (Gap #4) — if set, any call outside this set fails.
    if (task.toolsAllowed && task.toolsAllowed.length > 0) {
        const allowed = new Set(task.toolsAllowed);
        for (const called of obs.toolCalls) {
            if (!allowed.has(called)) {
                return { pass: false, reason: `Tool "${called}" is not in toolsAllowed`, failureCategory: 'REFUSAL_CORRECT' };
            }
        }
    }
    if (task.customScorer) {
        const custom = task.customScorer(obs.responseText, obs.toolCalls);
        if (!custom.pass) {
            return { pass: false, reason: custom.reason, failureCategory: 'WRONG_ANSWER' };
        }
    }
    return { pass: true, reason: 'All criteria satisfied' };
}
/** Select tasks from the given set filtered by options. */
function selectTasks(options, allTasks) {
    if (options.filterIds && options.filterIds.length > 0) {
        const allow = new Set(options.filterIds);
        return allTasks.filter((t) => allow.has(t.id));
    }
    if (options.tiers && options.tiers.length > 0) {
        const allow = new Set(options.tiers);
        return allTasks.filter((t) => allow.has(t.tier));
    }
    return allTasks;
}
/**
 * Run a single benchmark task against the daemon.
 */
async function runBenchmarkTask(task, options = {}) {
    const baseUrl = (options.baseUrl ?? 'http://127.0.0.1:4300').replace(/\/$/, '');
    const fetchImpl = options.fetchImpl ?? globalThis.fetch;
    const timeoutMs = options.perTaskTimeoutMs ?? 60_000;
    const startedAt = Date.now();
    if (!fetchImpl) {
        return {
            taskId: task.id,
            tier: task.tier,
            description: task.description,
            status: 'error',
            failureCategory: 'ERROR',
            reason: 'global fetch unavailable',
            responsePreview: '',
            toolCalls: [],
            durationMs: 0,
            tags: task.tags ?? [],
        };
    }
    let obs;
    try {
        const response = await fetchImpl(`${baseUrl}/api/chat`, {
            method: 'POST',
            headers: { 'Content-Type': 'application/json', Accept: 'text/event-stream' },
            body: JSON.stringify({ message: task.input, model: options.model }),
        });
        obs = await consumeChatStream(response, timeoutMs);
    }
    catch (error) {
        return {
            taskId: task.id,
            tier: task.tier,
            description: task.description,
            status: 'error',
            failureCategory: 'ERROR',
            reason: error instanceof Error ? error.message : String(error),
            responsePreview: '',
            toolCalls: [],
            durationMs: Date.now() - startedAt,
            tags: task.tags ?? [],
        };
    }
    const durationMs = Date.now() - startedAt;
    // Timeout heuristic: if the task hit the wall clock limit but we
    // got something back, still score it — the response may be correct.
    if (durationMs >= timeoutMs && !obs.responseText) {
        return {
            taskId: task.id,
            tier: task.tier,
            description: task.description,
            status: 'fail',
            failureCategory: 'TIMEOUT',
            reason: `Task exceeded ${timeoutMs}ms timeout with no response`,
            responsePreview: '',
            toolCalls: obs.toolCalls,
            durationMs,
            tags: task.tags ?? [],
        };
    }
    const score = scoreTask(task, obs);
    return {
        taskId: task.id,
        tier: task.tier,
        description: task.description,
        status: score.pass ? 'pass' : 'fail',
        failureCategory: score.pass ? undefined : score.failureCategory,
        reason: score.reason,
        responsePreview: obs.responseText.slice(0, 300),
        toolCalls: obs.toolCalls,
        durationMs,
        tags: task.tags ?? [],
    };
}
/**
 * Run all selected tasks sequentially and return a structured run record.
 */
async function runBenchmark(options = {}) {
    const { BUILT_IN_TASKS } = await Promise.resolve().then(() => __importStar(require('./benchmarkTasks')));
    const allTasks = [...(options.tasks ?? BUILT_IN_TASKS), ...(await loadRegressionTasks(options.projectDir))];
    const tasks = selectTasks(options, allTasks);
    const results = [];
    const startedAt = new Date();
    for (const task of tasks) {
        results.push(await runBenchmarkTask(task, options));
    }
    const finishedAt = new Date();
    const passed = results.filter((r) => r.status === 'pass').length;
    const failed = results.filter((r) => r.status === 'fail').length;
    const errored = results.filter((r) => r.status === 'error').length;
    const usedTiers = [...new Set(results.map((r) => r.tier))];
    const run = {
        id: `bench-${startedAt.getTime()}-${crypto.randomBytes(3).toString('hex')}`,
        startedAt: startedAt.toISOString(),
        finishedAt: finishedAt.toISOString(),
        model: options.model ?? 'unknown',
        baseUrl: (options.baseUrl ?? 'http://127.0.0.1:4300').replace(/\/$/, ''),
        tiers: usedTiers,
        total: results.length,
        passed,
        failed,
        errored,
        passRate: results.length === 0 ? 0 : passed / results.length,
        results,
    };
    if (options.projectDir) {
        await persistBenchmarkRun(options.projectDir, run);
        // Auto-grow regression tier from any failures in this run.
        await appendRegressionCases(options.projectDir, tasks, results);
    }
    return run;
}
// ─── Persistence ─────────────────────────────────────────────────────
function benchmarkDir(projectDir) {
    return path.join(projectDir, '.harness', 'benchmarks');
}
function regressionFile(projectDir) {
    return path.join(benchmarkDir(projectDir), 'regressions.json');
}
async function persistBenchmarkRun(projectDir, run) {
    const dir = benchmarkDir(projectDir);
    await fs.mkdir(dir, { recursive: true });
    const file = path.join(dir, `${run.id}.json`);
    await fs.writeFile(file, JSON.stringify(run, null, 2), 'utf-8');
}
async function loadBenchmarkRuns(projectDir) {
    const dir = benchmarkDir(projectDir);
    try {
        const entries = await fs.readdir(dir);
        const runs = [];
        for (const entry of entries) {
            if (!entry.endsWith('.json') || entry === 'regressions.json')
                continue;
            try {
                const raw = await fs.readFile(path.join(dir, entry), 'utf-8');
                runs.push(JSON.parse(raw));
            }
            catch { /* skip corrupt files */ }
        }
        return runs.sort((a, b) => b.startedAt.localeCompare(a.startedAt));
    }
    catch {
        return [];
    }
}
/** Load regression tasks that were auto-generated from past failures. */
async function loadRegressionTasks(projectDir) {
    if (!projectDir)
        return [];
    const file = regressionFile(projectDir);
    try {
        const raw = await fs.readFile(file, 'utf-8');
        const tasks = JSON.parse(raw);
        return Array.isArray(tasks) ? tasks.filter((t) => t.tier === 'regression') : [];
    }
    catch {
        return [];
    }
}
/**
 * For each failed task that produced a non-empty response, record a
 * reduced regression case so it gets re-run in every future benchmark.
 * Deduplicates by task ID so re-runs don't bloat the file.
 */
async function appendRegressionCases(projectDir, tasks, results) {
    const failures = results.filter((r) => r.status === 'fail' && r.responsePreview);
    if (failures.length === 0)
        return;
    const existing = await loadRegressionTasks(projectDir);
    const existingIds = new Set(existing.map((t) => t.id));
    const taskMap = new Map(tasks.map((t) => [t.id, t]));
    let added = 0;
    for (const failure of failures) {
        // Don't duplicate tasks that are already in the regression set.
        if (existingIds.has(failure.taskId))
            continue;
        const original = taskMap.get(failure.taskId);
        if (!original)
            continue;
        // Promote the original task to tier=regression and add it.
        existing.push({ ...original, tier: 'regression' });
        existingIds.add(failure.taskId);
        added++;
    }
    if (added > 0) {
        const dir = benchmarkDir(projectDir);
        await fs.mkdir(dir, { recursive: true });
        await fs.writeFile(regressionFile(projectDir), JSON.stringify(existing, null, 2), 'utf-8');
    }
}
/** Summary table for display: counts per tier. */
function summarizeByTier(run) {
    const map = new Map();
    for (const r of run.results) {
        const s = map.get(r.tier) ?? { total: 0, passed: 0 };
        s.total++;
        if (r.status === 'pass')
            s.passed++;
        map.set(r.tier, s);
    }
    return [...map.entries()].map(([tier, s]) => ({
        tier,
        total: s.total,
        passed: s.passed,
        passRate: s.total === 0 ? '—' : `${Math.round((s.passed / s.total) * 100)}%`,
    }));
}
//# sourceMappingURL=benchmark.js.map