import { _ as alwaysQuote } from "./tools-NxnEmzYg.js";
import { r as createProcessContext } from "./contexts-BJVgG0LY.js";
import { a as headlessEventToJsonl, c as runHeadless } from "./headless-CPaunZsU.js";
import { i as createMemoryStore, t as createSession } from "./session-CtAWwwkn.js";
import { join, relative, resolve } from "node:path";
import { tmpdir } from "node:os";
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import chalk from "chalk";
//#region src/eval.ts
/**
* Declare an eval's metric set. Returns the same map, typed; pass it to
* `EvalCaseOptions.metrics`. Declared metrics that are not emitted are recorded
* as missing with a normalized score of 0; emitting an undeclared metric is an
* authoring error and throws.
*/
function defineMetrics(spec) {
	return spec;
}
/** Always-available efficiency metrics derived from `HeadlessResult`. */
const EFFICIENCY_METRICS = {
	"execution-time": {
		min: 0,
		max: 3e5,
		direction: "lower-is-better",
		tags: ["@efficiency"],
		description: "Wall-clock run time (ms)."
	},
	"provider-tokens": {
		min: 0,
		max: 2e5,
		direction: "lower-is-better",
		tags: ["@efficiency"],
		description: "Input + output tokens."
	},
	"cache-read-rate": {
		min: 0,
		max: 1,
		direction: "higher-is-better",
		tags: ["@efficiency"],
		description: "Cache-read tokens / input tokens."
	}
};
function normalizeMetric(raw, spec) {
	const span = spec.max - spec.min;
	if (span === 0) return 1;
	const normalized = (Math.min(spec.max, Math.max(spec.min, raw)) - spec.min) / span;
	return spec.direction === "lower-is-better" ? 1 - normalized : normalized;
}
/**
* Thrown by `runEvalCase` when a scorer emits a metric id that was not declared
* in `defineMetrics`. This is an authoring error (a typo'd or stale metric id),
* not a low score — it fails the eval test rather than being recorded as a
* failed scorer. Exported so downstream harnesses can type-narrow on it.
*/
var EvalMetricError = class extends Error {
	constructor(message) {
		super(message);
		this.name = "EvalMetricError";
	}
};
/**
* Wrap an execution context so repeated agent runs share one handle until the
* caller disposes it. This is useful for eval fixtures that want per-test
* Docker/process lifetime while still using `runHeadless()` per turn.
*/
function createReusableExecutionContext(base) {
	let handle;
	let spawnPromise;
	let disposed = false;
	let disposePromise;
	const ensureHandle = (config) => {
		if (disposed) throw new Error("Reusable execution context has been disposed");
		if (handle) return Promise.resolve(handle);
		spawnPromise ??= base.spawn(config).then((next) => {
			handle = next;
			return next;
		}, (err) => {
			spawnPromise = void 0;
			throw err;
		});
		return spawnPromise;
	};
	return {
		execution: {
			...base,
			async spawn(config) {
				return ensureHandle(config);
			},
			async destroy() {}
		},
		dispose() {
			disposePromise ??= (async () => {
				disposed = true;
				const active = handle ?? await spawnPromise?.catch(() => void 0);
				handle = void 0;
				spawnPromise = void 0;
				if (active) await base.destroy(active);
			})();
			return disposePromise;
		},
		handle: () => handle
	};
}
/**
* Multi-turn eval agent over the low-level headless runner. It keeps session
* and execution lifetime outside any Playwright/Bolt-specific fixture layer,
* so downstream platforms can wrap it with their own `agent.run(...)` shape.
*/
function createEvalAgent(options) {
	const { cwd, execution, session: initialSession, store: initialStore, mcpServers: baseMcpServers, onEvent: baseOnEvent, ...headlessDefaults } = options;
	const executionOwner = createReusableExecutionContext(execution ?? createProcessContext({ cwd }));
	const store = initialStore ?? createMemoryStore();
	let sessionPromise = initialSession ? Promise.resolve(initialSession) : void 0;
	let disposed = false;
	let runInFlight = false;
	let disposePromise;
	const stats = {
		runs: 0,
		input: 0,
		output: 0,
		cacheRead: 0,
		cacheCreation: 0,
		cost: 0,
		turns: 0,
		toolCalls: 0,
		durationMs: 0
	};
	async function getSession() {
		if (disposed) throw new Error("Eval agent has been disposed");
		sessionPromise ??= createSession({ store });
		return sessionPromise;
	}
	async function resolveMcpServers(runServers) {
		const merged = [...(typeof baseMcpServers === "function" ? await baseMcpServers() : baseMcpServers) ?? [], ...runServers ?? []];
		return merged.length > 0 ? [...merged] : void 0;
	}
	async function run(runOptions) {
		if (runInFlight) throw new Error("Eval agent already has a run in progress");
		runInFlight = true;
		try {
			const session = await getSession();
			const { mcpServers: runMcpServers, onEvent: runOnEvent, ...restRunOptions } = runOptions;
			const mcpServers = await resolveMcpServers(runMcpServers);
			const result = await runHeadless({
				...headlessDefaults,
				...restRunOptions,
				session,
				execution: executionOwner.execution,
				...mcpServers ? { mcpServers } : {},
				onEvent(event) {
					baseOnEvent?.(event);
					runOnEvent?.(event);
				}
			});
			const newTranscript = result.transcript.slice();
			const runStats = {
				input: result.usage.input,
				output: result.usage.output,
				cacheRead: result.usage.cacheRead,
				cacheCreation: result.usage.cacheCreation,
				cost: result.usage.cost ?? 0,
				durationMs: result.durationMs,
				turns: result.turns,
				toolCalls: countToolCallsInTurns(newTranscript)
			};
			accumulateEvalAgentStats(stats, runStats);
			return {
				result,
				stats: runStats,
				newTranscript
			};
		} finally {
			runInFlight = false;
		}
	}
	function dispose() {
		disposePromise ??= (async () => {
			disposed = true;
			await executionOwner.dispose();
		})();
		return disposePromise;
	}
	return {
		run,
		stats,
		session: getSession,
		dispose,
		[Symbol.asyncDispose]: dispose
	};
}
/**
* Build an ordered, grouped trajectory from a transcript. Walks assistant
* turns in order, emitting `think` / `text` / `tool:<name>` steps, grouping
* only consecutive identical kinds. Per-step duration is approximated from the
* owning turn's `createdAt` delta to the next turn.
*/
function buildTrajectory(transcript) {
	const steps = [];
	let total = 0;
	const push = (kind, name, durationMs) => {
		total++;
		const last = steps[steps.length - 1];
		if (last && last.kind === kind && last.name === name) {
			last.count++;
			if (durationMs > 0) last.durationMs = (last.durationMs ?? 0) + durationMs;
			return;
		}
		steps.push({
			kind,
			...name ? { name } : {},
			count: 1,
			...durationMs > 0 ? { durationMs } : {}
		});
	};
	for (let i = 0; i < transcript.length; i++) {
		const turn = transcript[i];
		if (turn.role !== "assistant") continue;
		const next = transcript[i + 1];
		const turnDuration = next ? Math.max(0, next.createdAt - turn.createdAt) : 0;
		const blocks = turn.content.filter((b) => b.type === "thinking" || b.type === "redacted_thinking" || b.type === "text" || b.type === "tool_call");
		const toolBlocks = blocks.filter((b) => b.type === "tool_call").length;
		const perTool = toolBlocks > 0 ? Math.round(turnDuration / toolBlocks) : 0;
		for (const block of blocks) if (block.type === "thinking" || block.type === "redacted_thinking") push("think", void 0, 0);
		else if (block.type === "text") push("text", void 0, 0);
		else if (block.type === "tool_call") push("tool", block.name, perTool);
	}
	return {
		steps,
		totalBlocks: total
	};
}
/** Render a trajectory as a one-line timeline string (no color). */
function formatTrajectoryLine(trajectory) {
	return trajectory.steps.map(stepLabel).join(" → ");
}
function stepLabel(step) {
	const base = step.kind === "tool" ? step.name ?? "tool" : step.kind;
	return step.count > 1 ? `${base} ×${step.count}` : base;
}
const evalRegistry = [];
const STUB_PROVIDER = {
	name: "stub",
	meta: { defaultModel: "stub" },
	formatTools: (tools) => tools,
	userMessage: () => ({
		role: "user",
		content: []
	}),
	assistantMessage: () => ({
		role: "assistant",
		content: []
	}),
	toolResultsMessage: () => ({
		role: "user",
		content: []
	}),
	stream: async () => {
		throw new Error("stub provider is not runnable");
	}
};
/**
* Register an eval definition. Eval files call this at module load; the harness
* (`buildRegisteredEvals`) materializes them with a provider at run time. The
* factory keeps definitions lazy so a single registry serves any provider/judge
* pairing without re-importing eval files.
*
* Returns the same factory so a file can also `export default` it for direct,
* registry-free use (e.g. hermetic unit tests).
*/
function defineEval(define) {
	const id = safeRegistryId(define);
	if (!evalRegistry.some((entry) => entry.id === id)) evalRegistry.push({
		id,
		define
	});
	return define;
}
/** Snapshot of every registered eval, materialized for one provider/judge pair. */
function buildRegisteredEvals(ctx) {
	return evalRegistry.map((entry) => entry.define(ctx));
}
/** Drop every registered eval — test isolation helper. */
function clearRegisteredEvals() {
	evalRegistry.length = 0;
}
function safeRegistryId(define) {
	try {
		const stub = define({
			provider: STUB_PROVIDER,
			judge: STUB_PROVIDER
		});
		return `${stub.suite ?? "eval"}/${stub.id}`;
	} catch {
		return `eval/${evalRegistry.length}`;
	}
}
const DEFAULT_JUDGE_SYSTEM = [
	"You are a strict software eval judge.",
	"Grade only the supplied output against the rubric.",
	"Call the `submit_grade` tool with your score and reasoning.",
	"The score must be between 0 and 1."
].join("\n");
/** Forced-output tool the judge must call — the schema-enforcement path. */
const JUDGE_TOOL = {
	name: "submit_grade",
	description: "Submit the grade for the output under evaluation.",
	inputSchema: {
		type: "object",
		properties: {
			score: {
				type: "number",
				minimum: 0,
				maximum: 1,
				description: "Quality score from 0 (worst) to 1 (best)."
			},
			reasoning: {
				type: "string",
				description: "Brief justification for the score."
			},
			feedback: {
				type: "string",
				description: "Optional actionable feedback."
			}
		},
		required: ["score", "reasoning"],
		additionalProperties: false
	}
};
async function runEvalCase(options) {
	const { id, suite, variant, tags = [], artifactDir, workspace, scorers = [], onEvent, ...headless } = options;
	const { metrics: declaredMetrics, sourceFile, ...headlessRest } = headless;
	const events = [];
	const caseArtifactDir = artifactDir ? join(artifactDir, ...variant ? [safeSegment(variant)] : [], safeSegment(suite ?? "eval"), safeSegment(id)) : void 0;
	const workspaceState = await prepareWorkspace(headlessRest.execution, workspace);
	const caseMetricIds = new Set(Object.keys(declaredMetrics ?? {}));
	const hasCaseMetrics = caseMetricIds.size > 0;
	const metricSpecs = {
		...EFFICIENCY_METRICS,
		...declaredMetrics
	};
	const rawMetrics = /* @__PURE__ */ new Map();
	const emitMetric = (metricId, raw) => {
		if (!(metricId in metricSpecs)) throw new EvalMetricError(`Eval ${id}: metric "${metricId}" emitted but not declared in defineMetrics`);
		if (!Number.isFinite(raw)) throw new EvalMetricError(`Eval ${id}: metric "${metricId}" emitted a non-finite value (${raw}); scores require finite numbers`);
		rawMetrics.set(metricId, raw);
	};
	try {
		const result = await runHeadless({
			...headlessRest,
			...workspaceState.execution ? { execution: workspaceState.execution } : {},
			onEvent(event) {
				events.push(event);
				onEvent?.(event);
			}
		});
		emitEfficiencyMetrics(emitMetric, result);
		const workspaceSnapshot = workspaceState.snapshot();
		const workspaceError = workspaceState.error();
		const scores = await runScorers({
			id,
			...suite ? { suite } : {},
			tags,
			result,
			events,
			metric: emitMetric,
			...workspaceSnapshot ? { workspace: workspaceSnapshot } : {},
			...caseArtifactDir ? { artifactDir: caseArtifactDir } : {}
		}, scorers);
		const metrics = finalizeEvalMetrics(metricSpecs, rawMetrics);
		const tagScores = computeEvalTagScores(metrics);
		const caseMetrics = metrics.filter((metric) => caseMetricIds.has(metric.id));
		const score = hasCaseMetrics && caseMetrics.length > 0 ? meanNormalized(caseMetrics) : aggregateScore(scores);
		const evalResult = {
			id,
			...suite ? { suite } : {},
			...variant ? { variant } : {},
			tags,
			result,
			score,
			passed: result.status === "completed" && scores.every((score) => score.passed),
			scores,
			metrics,
			tagScores,
			trajectory: buildTrajectory(result.transcript),
			events,
			...sourceFile ? { sourceFile } : {},
			...workspaceSnapshot ? { workspace: workspaceSnapshot } : {},
			...workspaceError ? { workspaceError } : {}
		};
		const artifacts = caseArtifactDir ? await writeEvalArtifacts(caseArtifactDir, evalResult) : void 0;
		return {
			...evalResult,
			...artifacts ? { artifacts } : {}
		};
	} finally {
		await workspaceState.cleanup();
	}
}
function statusCompleted(name = "status.completed") {
	return ({ result }) => ({
		name,
		passed: result.status === "completed",
		score: result.status === "completed" ? 1 : 0,
		details: result.status === "completed" ? void 0 : {
			status: result.status,
			error: result.error
		}
	});
}
function fileExists(path, name = `file.exists:${path}`) {
	return ({ workspace }) => {
		const found = workspace?.files.some((file) => file.path === path) ?? false;
		return {
			name,
			passed: found,
			score: found ? 1 : 0,
			details: found ? void 0 : `Missing ${path}`
		};
	};
}
function fileExistsOneOf(paths, name = `file.existsOneOf:${paths.join("|")}`) {
	return ({ workspace }) => {
		const found = paths.find((path) => workspace?.files.some((file) => file.path === path));
		return {
			name,
			passed: Boolean(found),
			score: found ? 1 : 0,
			details: found ? void 0 : `Missing one of: ${paths.join(", ")}`
		};
	};
}
function fileContains(path, expected, name = `file.contains:${path}`) {
	return ({ workspace }) => {
		const file = workspace?.files.find((file) => file.path === path);
		const content = file?.content ?? "";
		const passed = typeof expected === "string" ? content.includes(expected) : expected.test(content);
		return {
			name,
			passed,
			score: passed ? 1 : 0,
			details: passed ? void 0 : file ? `Expected ${path} to contain ${String(expected)}` : `Missing ${path}`
		};
	};
}
function fileContentQuality(path, expected, name = `file.quality:${path}`) {
	return ({ workspace }) => {
		const content = (workspace?.files.find((file) => file.path === path))?.content;
		if (content === void 0) return {
			name,
			passed: false,
			score: 0,
			details: `Missing ${path}`
		};
		const normalized = content.trim();
		const expectedNormalized = expected.trim();
		const score = normalized === expectedNormalized ? 1 : normalized.includes(expectedNormalized) ? .75 : content.includes(expected) ? .5 : 0;
		return {
			name,
			passed: score >= .75,
			score,
			details: score >= .75 ? void 0 : `Expected ${path} to closely match ${JSON.stringify(expectedNormalized)}`
		};
	};
}
function llmJudge(options) {
	const name = options.name ?? "llm.judge";
	const model = options.model ?? options.provider.meta.defaultModel;
	return async (ctx) => {
		const userMessage = options.provider.userMessage(renderJudgePrompt({
			rubric: options.rubric,
			input: options.input?.(ctx) ?? defaultJudgeInput(ctx)
		}));
		let text = "";
		const controller = new AbortController();
		const timeoutMs = options.timeoutMs ?? 6e4;
		let timedOut = false;
		const timer = timeoutMs > 0 ? setTimeout(() => {
			timedOut = true;
			controller.abort(/* @__PURE__ */ new Error(`LLM judge timed out after ${timeoutMs}ms`));
		}, timeoutMs) : void 0;
		try {
			const streamPromise = options.provider.stream({
				model,
				system: options.system ?? DEFAULT_JUDGE_SYSTEM,
				tools: options.provider.formatTools([JUDGE_TOOL]),
				messages: [userMessage],
				maxTokens: options.maxTokens ?? 600,
				cache: false,
				signal: controller.signal,
				toolChoice: {
					type: "tool",
					name: JUDGE_TOOL.name
				}
			}, { onText(delta) {
				text += delta;
			} });
			const timeoutPromise = timeoutMs > 0 ? new Promise((_resolve, reject) => {
				controller.signal.addEventListener("abort", () => {
					if (timedOut) reject(/* @__PURE__ */ new Error(`LLM judge timed out after ${timeoutMs}ms`));
				}, { once: true });
			}) : void 0;
			const result = timeoutPromise ? await Promise.race([streamPromise, timeoutPromise]) : await streamPromise;
			const parsed = readJudgeResult(result.toolCalls, text || result.text);
			if (options.metric) ctx.metric(options.metric, parsed.score);
			return {
				name,
				passed: parsed.score >= .7,
				score: parsed.score,
				details: {
					reasoning: parsed.reasoning,
					...parsed.feedback ? { feedback: parsed.feedback } : {},
					...parsed.raw ? { raw: parsed.raw } : {},
					usage: result.usage
				}
			};
		} finally {
			if (timer) clearTimeout(timer);
		}
	};
}
function readJudgeResult(toolCalls, fallbackText) {
	const call = toolCalls.find((tc) => tc.name === JUDGE_TOOL.name);
	if (call) {
		const input = call.input;
		return {
			score: clampScore(input.score),
			reasoning: typeof input.reasoning === "string" ? input.reasoning : "",
			...typeof input.feedback === "string" ? { feedback: input.feedback } : {}
		};
	}
	return parseJudgeOutput(fallbackText);
}
/**
* Aggregate a group of boolean scorers into a single `0..1` functionality
* metric (fraction passed) and emit it. Useful for "N of M files present".
*/
function functionalityMetric(metricId, scorers, name = metricId) {
	return async (ctx) => {
		const results = await runScorers(ctx, scorers);
		const passed = results.filter((r) => r.passed).length;
		const fraction = results.length === 0 ? 1 : passed / results.length;
		ctx.metric(metricId, fraction);
		return {
			name,
			passed: passed === results.length,
			score: fraction,
			details: {
				passed,
				total: results.length,
				checks: results.map((r) => ({
					name: r.name,
					passed: r.passed
				}))
			}
		};
	};
}
function formatEvalCaseSummary(result) {
	const lines = [
		`Eval ${result.suite ? `${result.suite}/` : ""}${result.id}`,
		...result.variant ? [`variant: ${result.variant}`] : [],
		`status: ${result.result.status}`,
		`passed: ${result.passed}`,
		`score: ${result.score.toFixed(2)}`,
		`duration: ${result.result.durationMs}ms`,
		`turns: ${result.result.turns}`,
		`tool calls: ${result.result.numToolCalls}`,
		`usage: input=${result.result.usage.input} output=${result.result.usage.output} cacheRead=${result.result.usage.cacheRead} cacheCreation=${result.result.usage.cacheCreation}${result.result.usage.cost !== void 0 ? ` cost=$${result.result.usage.cost.toFixed(6)}` : ""}`
	];
	if (result.workspace) lines.push(`workspace files: ${result.workspace.files.length}`);
	if (result.workspaceError) lines.push(`workspace error: ${result.workspaceError}`);
	if (result.artifacts) lines.push(`artifacts: ${result.artifacts.dir}`);
	if (result.scores.length > 0) {
		lines.push("scores:");
		for (const score of result.scores) {
			const value = score.score !== void 0 ? score.score.toFixed(2) : score.passed ? "1.00" : "0.00";
			lines.push(`  - ${score.name}: ${value} ${score.passed ? "pass" : "fail"}`);
			if (typeof score.details === "string" && score.details.length > 0) lines.push(`    ${score.details}`);
			else if (isRecord(score.details) && typeof score.details.reasoning === "string") lines.push(`    ${score.details.reasoning}`);
		}
	}
	return lines.join("\n");
}
function formatEvalRunSummary(results) {
	return formatEvalRunSummaryWithOptions(results);
}
/**
* Register one test per eval case against a test runner, funnel every result
* into a shared reporter, and print the aggregated summary once after all cases.
*
* A failing test means the agent run itself broke (provider/tool/timeout). Low
* scores are reported, not asserted — see {@link RegisterEvalTestsOptions.failOnIncomplete}.
*/
function registerEvalTests(options) {
	const reporter = options.reporter ?? createEvalRunReporter();
	const failOnIncomplete = options.failOnIncomplete ?? true;
	const printSummary = options.printSummary ?? true;
	const repeat = Math.max(1, options.repeat ?? 1);
	const concurrency = Math.max(1, options.concurrency ?? 1);
	options.runner.afterAll(async () => {
		await reporter.flush();
		if (printSummary) console.log(`\n${reporter.format()}\n`);
		if (options.dispose) await options.dispose();
	});
	const work = [];
	for (const evalCase of options.cases) {
		const caseLabel = `${evalCase.suite ? `${evalCase.suite}/` : ""}${evalCase.id}`;
		const baseLabel = evalCase.variant ? `[${evalCase.variant}] ${caseLabel}` : caseLabel;
		for (let run = 1; run <= repeat; run++) {
			const label = repeat > 1 ? `${baseLabel} #${run}` : baseLabel;
			const caseId = repeat > 1 ? `${evalCase.id}-repeat-${run}` : evalCase.id;
			work.push({
				label,
				run: () => runEvalCase({
					...evalCase,
					id: caseId,
					...options.artifactDir ? { artifactDir: options.artifactDir } : {}
				})
			});
		}
	}
	const schedule = createLazyPool(concurrency);
	work.forEach((w) => {
		options.runner.it(w.label, async () => {
			const result = await schedule(w.run);
			await reporter.record(result);
			if (failOnIncomplete && result.result.status !== "completed") throw new Error(`Eval ${w.label} did not complete: ${JSON.stringify(result.result.error ?? { status: result.result.status }, null, 2)}`);
		});
	});
	return reporter;
}
/**
* Lazily run tasks with a bounded concurrency `limit`. Nothing starts until a
* runner invokes the corresponding test callback, so filtered/skipped tests do
* not leak model or Docker work in the background.
*/
function createLazyPool(limit) {
	const queue = [];
	let active = 0;
	const drain = () => {
		const available = Math.min(limit - active, queue.length);
		for (let i = 0; i < available; i++) queue.shift()?.();
	};
	return (task) => new Promise((resolve, reject) => {
		const run = async () => {
			active++;
			try {
				resolve(await task());
			} catch (err) {
				reject(err);
			} finally {
				active--;
				drain();
			}
		};
		queue.push(() => void run());
		drain();
	});
}
function createEvalRunReporter(options = {}) {
	const results = [];
	return {
		get results() {
			return results;
		},
		async record(result) {
			results.push(result);
			if (options.outputDir) {
				const segments = [
					...result.variant ? [safeSegment(result.variant)] : [],
					safeSegment(result.suite ?? "eval"),
					safeSegment(result.id)
				];
				const casePath = join(options.outputDir, "cases", `${segments.join("--")}.json`);
				await mkdir(join(options.outputDir, "cases"), { recursive: true });
				await writeFile(casePath, `${JSON.stringify(result, null, 2)}\n`);
			}
		},
		async flush() {
			const summary = buildEvalRunSummary(results);
			if (options.outputDir) {
				await mkdir(options.outputDir, { recursive: true });
				await writeFile(join(options.outputDir, "run-summary.json"), `${JSON.stringify(summary, null, 2)}\n`);
			}
			return summary;
		},
		format() {
			return formatEvalRunSummaryWithOptions(results, { color: options.color });
		}
	};
}
/** Stable ordering by `variant`, then `suite/id`, so parallel completion order doesn't shuffle output. */
function sortCases(results) {
	return [...results].sort((a, b) => {
		const va = a.variant ?? "";
		const vb = b.variant ?? "";
		if (va !== vb) return va.localeCompare(vb);
		const ka = `${a.suite ?? ""}/${a.id}`;
		const kb = `${b.suite ?? ""}/${b.id}`;
		return ka.localeCompare(kb);
	});
}
/** Distinct variant labels across results, in sorted order. */
function distinctVariants(results) {
	return [...new Set(results.map((r) => r.variant).filter((v) => Boolean(v)))].sort();
}
function buildEvalRunSummary(input) {
	const results = sortCases(input);
	const usage = results.reduce((acc, result) => {
		acc.input += result.result.usage.input;
		acc.output += result.result.usage.output;
		acc.cacheRead += result.result.usage.cacheRead;
		acc.cacheCreation += result.result.usage.cacheCreation;
		acc.cost += result.result.usage.cost ?? 0;
		return acc;
	}, {
		input: 0,
		output: 0,
		cacheRead: 0,
		cacheCreation: 0,
		cost: 0
	});
	const totalScore = results.reduce((sum, result) => sum + result.score, 0);
	return {
		count: results.length,
		passed: results.filter((result) => result.passed).length,
		score: results.length > 0 ? totalScore / results.length : 0,
		durationMs: results.reduce((sum, result) => sum + result.result.durationMs, 0),
		usage,
		cases: results.map((result) => ({
			id: result.id,
			...result.suite ? { suite: result.suite } : {},
			...result.variant ? { variant: result.variant } : {},
			passed: result.passed,
			score: result.score,
			status: result.result.status,
			durationMs: result.result.durationMs,
			scores: result.scores,
			metrics: result.metrics,
			tagScores: result.tagScores,
			trajectory: result.trajectory
		})),
		metrics: aggregateMetrics(results),
		tagScores: aggregateTagScores(results),
		...distinctVariants(results).length > 0 ? { variants: aggregateVariants(results) } : {}
	};
}
function aggregateVariants(results) {
	return distinctVariants(results).map((variant) => {
		const group = results.filter((r) => r.variant === variant);
		const usage = group.reduce((acc, result) => {
			acc.input += result.result.usage.input;
			acc.output += result.result.usage.output;
			acc.cacheRead += result.result.usage.cacheRead;
			acc.cacheCreation += result.result.usage.cacheCreation;
			acc.cost += result.result.usage.cost ?? 0;
			return acc;
		}, {
			input: 0,
			output: 0,
			cacheRead: 0,
			cacheCreation: 0,
			cost: 0
		});
		return {
			variant,
			count: group.length,
			passed: group.filter((r) => r.passed).length,
			score: mean(group.map((r) => r.score)),
			durationMs: group.reduce((sum, r) => sum + r.result.durationMs, 0),
			usage
		};
	});
}
function aggregateMetrics(results) {
	const byId = /* @__PURE__ */ new Map();
	for (const result of results) for (const metric of result.metrics) {
		const list = byId.get(metric.id) ?? [];
		list.push(metric);
		byId.set(metric.id, list);
	}
	return [...byId.entries()].map(([id, metrics]) => ({
		id,
		direction: metrics[0].direction,
		tags: metrics[0].tags,
		raw: metricStats(metrics.map((m) => m.raw)),
		normalized: metricStats(metrics.map((m) => m.normalized))
	}));
}
function aggregateTagScores(results) {
	const byTag = /* @__PURE__ */ new Map();
	for (const result of results) for (const [tag, value] of Object.entries(result.tagScores)) {
		const list = byTag.get(tag) ?? [];
		list.push(value);
		byTag.set(tag, list);
	}
	const out = {};
	for (const [tag, values] of byTag) out[tag] = mean(values);
	return out;
}
function metricStats(values) {
	const sorted = [...values].sort((a, b) => a - b);
	return {
		mean: mean(values),
		min: sorted[0] ?? 0,
		max: sorted[sorted.length - 1] ?? 0,
		p50: percentile(sorted, .5),
		p90: percentile(sorted, .9),
		zeroCount: values.filter((v) => v === 0).length,
		values
	};
}
function percentile(sorted, q) {
	if (sorted.length === 0) return 0;
	return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(q * sorted.length) - 1))];
}
function formatEvalRunSummaryWithOptions(input, options = {}) {
	const results = sortCases(input);
	const color = createEvalColors(options.color);
	if (results.length === 0) return `${color.heading("Eval run summary")}\n${color.muted("no evals ran")}`;
	const summary = buildEvalRunSummary(results);
	const out = [];
	const variants = distinctVariants(results);
	out.push(color.heading("Eval run summary"));
	if (variants.length > 1) {
		out.push(renderComparisonTable(results, variants, color));
		for (const variant of variants) {
			const group = results.filter((r) => r.variant === variant);
			out.push("");
			out.push(color.heading(variant));
			out.push(renderResultsTable(group, buildEvalRunSummary(group), color));
		}
	} else out.push(renderResultsTable(results, summary, color));
	const tagTable = renderTagTable(summary.tagScores, color);
	if (tagTable) {
		out.push("");
		out.push(tagTable);
	}
	out.push("");
	const hyperlinks = options.color !== false;
	const linkText = (target) => {
		const abs = absPath(target);
		return hyperlinks ? oscLink(abs, abs) : abs;
	};
	for (const result of results) {
		const caseLabel = `${result.suite ? `${result.suite}/` : ""}${result.id}`;
		const label = result.variant ? `[${result.variant}] ${caseLabel}` : caseLabel;
		out.push(`${color.dot(result.passed)} ${color.caseStatus(result.passed, label)}`);
		out.push(`${color.muted("score")} ${color.score(result.score)}`);
		const tagEntries = Object.entries(result.tagScores);
		if (tagEntries.length > 0) out.push(`${color.muted("tags:")} ${tagEntries.map(([tag, value]) => `${color.tag(tag)} ${color.score(value)}`).join("  ")}`);
		for (const metric of result.metrics) {
			const note = metric.missing ? color.fail(" (not emitted)") : color.muted(` (${formatMetricRaw(metric)})`);
			out.push(`${color.muted(padEnd(metric.id, 28))} ${color.score(metric.normalized)}${note}`);
		}
		for (const score of result.scores) {
			const detail = scoreDetailLine(score);
			if (!score.passed && detail) out.push(`${color.dot(score.passed)} ${color.muted(`${score.name}: ${truncate(detail, 140)}`)}`);
		}
		if (result.sourceFile) out.push(`${color.muted("case:")} ${color.muted(linkText(result.sourceFile))}`);
		const resultTarget = caseLinkTarget(result);
		if (resultTarget) out.push(`${color.muted("result:")} ${color.muted(linkText(resultTarget))}`);
		if (result.trajectory.steps.length > 0) out.push(`${color.muted(`turns: ${result.trajectory.totalBlocks}`)} ${formatTrajectoryColored(result.trajectory, color)}`);
		out.push("");
	}
	return out.join("\n").replace(/\n+$/, "");
}
function renderTagTable(tagScores, color) {
	const entries = Object.entries(tagScores);
	if (entries.length === 0) return void 0;
	const tagWidth = Math.max(3, ...entries.map(([tag]) => tag.length));
	const top = `┌${"─".repeat(tagWidth + 2)}┬${"─".repeat(9)}┐`;
	const mid = `├${"─".repeat(tagWidth + 2)}┼${"─".repeat(9)}┤`;
	const bot = `└${"─".repeat(tagWidth + 2)}┴${"─".repeat(9)}┘`;
	return [
		top,
		`│ ${color.heading(padEnd("TAG", tagWidth))} │ ${color.heading("SCORE")}   │`,
		mid,
		...entries.map(([tag, value]) => `│ ${color.muted(padEnd(tag, tagWidth))} │ ${color.score(value)}    │`),
		bot
	].join("\n");
}
function formatMetricRaw(metric) {
	return `raw ${Number.isInteger(metric.raw) ? formatInt(metric.raw) : metric.raw.toFixed(3)} ${metric.direction === "lower-is-better" ? "↓" : "↑"}`;
}
/**
* Resolve a filesystem target to link an eval case to. Prefers the written
* artifact (`result.json` → its dir), so the link opens the case's output.
* Returns `undefined` when no artifacts were written.
*/
function caseLinkTarget(result) {
	return result.artifacts?.result ?? result.artifacts?.dir;
}
/** Wrap text in an OSC 8 terminal hyperlink to a local path or URL. */
function oscLink(target, text) {
	const url = /^(?:file|https?):\/\//.test(target) ? target : `file://${resolve(target)}`;
	const OSC = "\x1B]8;;";
	const ST = "\x1B\\";
	return `${OSC}${url}${ST}${text}${OSC}${ST}`;
}
/** Absolute filesystem path (strips any `file://` prefix, resolves relatives). */
function absPath(target) {
	return resolve(target.replace(/^file:\/\//, ""));
}
/**
* Case key used to line up the same eval across variants in the comparison
* matrix. Strips the `-repeat-N` suffix `registerEvalTests` appends so a
* repeated case aggregates into one row (mean score, x/y passed).
*/
function comparisonCaseKey(result) {
	const id = result.id.replace(/-repeat-\d+$/, "");
	return `${result.suite ? `${result.suite}/` : ""}${id}`;
}
/**
* Cross-variant comparison matrix: one row per eval case, one column per
* variant (provider/model target), plus rollup rows (score / passed / cost /
* time) so two models can be read side by side at a glance.
*/
function renderComparisonTable(input, variants, color) {
	const results = sortCases(input);
	const caseKeys = [...new Set(results.map(comparisonCaseKey))].sort();
	const byCell = /* @__PURE__ */ new Map();
	for (const result of results) {
		const key = `${comparisonCaseKey(result)}\u0000${result.variant ?? ""}`;
		const list = byCell.get(key) ?? [];
		list.push(result);
		byCell.set(key, list);
	}
	const cellFor = (caseKey, variant) => {
		const group = byCell.get(`${caseKey}\u0000${variant}`);
		if (!group || group.length === 0) return { text: "-" };
		const score = mean(group.map((r) => r.score));
		const passedCount = group.filter((r) => r.passed).length;
		const allPassed = passedCount === group.length;
		const status = group.length > 1 ? `${passedCount}/${group.length}` : allPassed ? "pass" : "fail";
		return {
			text: `${score.toFixed(2)} ${status}`,
			score,
			passed: allPassed
		};
	};
	const variantSummaries = aggregateVariants(results);
	const summaryOf = (variant) => variantSummaries.find((s) => s.variant === variant);
	const rows = caseKeys.map((key) => ({
		label: key,
		cells: variants.map((variant) => cellFor(key, variant))
	}));
	const rollups = [
		{
			label: "SCORE",
			rollup: true,
			cells: variants.map((variant) => {
				const s = summaryOf(variant);
				return s ? {
					text: s.score.toFixed(2),
					score: s.score
				} : { text: "-" };
			})
		},
		{
			label: "PASSED",
			rollup: true,
			cells: variants.map((variant) => {
				const s = summaryOf(variant);
				return s ? {
					text: `${s.passed}/${s.count}`,
					passed: s.passed === s.count
				} : { text: "-" };
			})
		},
		{
			label: "COST",
			rollup: true,
			cells: variants.map((variant) => {
				return { text: formatCost(summaryOf(variant)?.usage.cost ?? 0) };
			})
		},
		{
			label: "TIME",
			rollup: true,
			cells: variants.map((variant) => {
				const s = summaryOf(variant);
				return { text: s ? formatDuration(s.durationMs) : "-" };
			})
		}
	];
	const header = ["EVAL", ...variants];
	const allRows = [...rows, ...rollups];
	const widths = header.map((h, i) => Math.max(h.length, ...allRows.map((row) => (i === 0 ? row.label : row.cells[i - 1].text).length)));
	const top = `┌${widths.map((w) => "─".repeat(w + 2)).join("┬")}┐`;
	const mid = `├${widths.map((w) => "─".repeat(w + 2)).join("┼")}┤`;
	const bot = `└${widths.map((w) => "─".repeat(w + 2)).join("┴")}┘`;
	const paintComparisonCell = (cell, padded) => {
		if (cell.score === void 0 && cell.passed === void 0) return color.muted(padded);
		if (cell.score !== void 0) {
			const scoreText = cell.score.toFixed(2);
			const rest = cell.text.slice(scoreText.length);
			const coloredRest = cell.passed === void 0 ? color.muted(rest) : cell.passed ? color.pass(rest) : color.fail(rest);
			return padded.replace(cell.text, `${color.score(cell.score)}${coloredRest}`);
		}
		return padded.replace(cell.text, cell.passed ? color.pass(cell.text) : color.fail(cell.text));
	};
	const renderRow = (row) => {
		const labelPadded = padEnd(row.label, widths[0]);
		return `│ ${row.rollup ? color.heading(labelPadded) : color.muted(labelPadded)} │${row.cells.map((cell, i) => {
			return ` ${paintComparisonCell(cell, padStart(cell.text, widths[i + 1]))} `;
		}).join("│")}│`;
	};
	return [
		top,
		`│ ${color.heading(padEnd(header[0], widths[0]))} │${variants.map((v, i) => ` ${color.heading(padStart(v, widths[i + 1]))} `).join("│")}│`,
		mid,
		...rows.map(renderRow),
		mid,
		...rollups.map(renderRow),
		bot
	].join("\n");
}
function renderResultsTable(results, summary, color) {
	const header = [
		"EVAL",
		"STATUS",
		"SCORE",
		"IN",
		"OUT",
		"CACHE R",
		"CACHE W",
		"COST",
		"TIME"
	];
	const rows = results.map((result) => {
		const label = `${result.suite ? `${result.suite}/` : ""}${result.id}`;
		const u = result.result.usage;
		return {
			passed: result.passed,
			cells: [
				label,
				result.passed ? "pass" : "fail",
				result.score.toFixed(2),
				formatInt(u.input),
				formatInt(u.output),
				formatInt(u.cacheRead),
				formatInt(u.cacheCreation),
				formatCost(u.cost ?? 0),
				formatDuration(result.result.durationMs)
			],
			score: result.score
		};
	});
	const totals = {
		cells: [
			`TOTAL (${summary.passed}/${summary.count})`,
			summary.passed === summary.count ? "pass" : "fail",
			summary.score.toFixed(2),
			formatInt(summary.usage.input),
			formatInt(summary.usage.output),
			formatInt(summary.usage.cacheRead),
			formatInt(summary.usage.cacheCreation),
			formatCost(summary.usage.cost),
			formatDuration(summary.durationMs)
		],
		passed: summary.passed === summary.count,
		score: summary.score
	};
	const aligns = [
		"left",
		"left",
		"right",
		"right",
		"right",
		"right",
		"right",
		"right",
		"right"
	];
	const widths = header.map((h, i) => Math.max(h.length, ...rows.map((r) => r.cells[i].length), totals.cells[i].length));
	const top = `┌${widths.map((w) => "─".repeat(w + 2)).join("┬")}┐`;
	const mid = `├${widths.map((w) => "─".repeat(w + 2)).join("┼")}┤`;
	const bot = `└${widths.map((w) => "─".repeat(w + 2)).join("┴")}┘`;
	const renderRow = (cells, paint) => {
		return `│${cells.map((cell, i) => {
			const padded = aligns[i] === "right" ? padStart(cell, widths[i]) : padEnd(cell, widths[i]);
			return ` ${paint ? paint(i, cell, padded) : padded} `;
		}).join("│")}│`;
	};
	return [
		top,
		renderRow(header, (_i, _raw, padded) => color.heading(padded)),
		mid,
		...rows.map((r) => renderRow(r.cells, (i, raw, padded) => paintCell(i, raw, padded, r.passed, r.score, color))),
		mid,
		renderRow(totals.cells, (i, raw, padded) => paintCell(i, raw, padded, totals.passed, totals.score, color, true)),
		bot
	].join("\n");
}
function paintCell(col, raw, padded, passed, score, color, bold = false) {
	if (col === 0) return bold ? color.heading(padded) : color.caseStatus(passed, padded);
	if (col === 1) return padded.replace(raw, color.status(passed));
	if (col === 2) return padded.replace(raw, color.score(score));
	if (col === 7) return color.cost(padded);
	return color.muted(padded);
}
function scoreDetailLine(score) {
	if (typeof score.details === "string") return score.details;
	if (isRecord(score.details) && typeof score.details.reasoning === "string") return score.details.reasoning;
}
function formatInt(value) {
	return value.toLocaleString("en-US");
}
function formatCost(value) {
	return value > 0 ? `$${value.toFixed(4)}` : "-";
}
function formatDuration(ms) {
	const rounded = Math.round(ms);
	if (rounded < 1e3) return `${rounded}ms`;
	const s = rounded / 1e3;
	return `${Number.isInteger(s) ? s : s.toFixed(1)}s`;
}
function padEnd(value, width) {
	return value.length >= width ? value : value + " ".repeat(width - value.length);
}
function padStart(value, width) {
	return value.length >= width ? value : " ".repeat(width - value.length) + value;
}
function truncate(value, max) {
	const oneLine = value.replace(/\s+/g, " ").trim();
	return oneLine.length > max ? `${oneLine.slice(0, max - 1)}…` : oneLine;
}
async function prepareWorkspace(execution, workspace) {
	if (!workspace) return {
		snapshot: () => void 0,
		error: () => void 0,
		cleanup: async () => {}
	};
	let tempDir;
	let baseExecution = execution;
	if (!baseExecution) {
		const cwd = workspace.cwd ?? await mkdtemp(join(tmpdir(), "zidane-eval-"));
		if (!workspace.cwd) tempDir = cwd;
		baseExecution = createProcessContext({ cwd });
	}
	const captured = {};
	return {
		execution: withWorkspaceLifecycle(baseExecution, workspace, captured),
		snapshot: () => captured.snapshot,
		error: () => captured.error,
		cleanup: async () => {
			if (tempDir && !workspace.retain) await rm(tempDir, {
				recursive: true,
				force: true
			});
		}
	};
}
function withWorkspaceLifecycle(execution, workspace, captured) {
	return {
		...execution,
		async spawn(config) {
			const handle = await execution.spawn({
				...config,
				...workspace.cwd && !config?.cwd ? { cwd: workspace.cwd } : {}
			});
			if (workspace.seedDir) await seedWorkspace(execution, handle, workspace);
			return handle;
		},
		async destroy(handle) {
			try {
				captured.snapshot = await captureWorkspace(execution, handle, workspace);
			} catch (err) {
				captured.error = err instanceof Error ? err.message : String(err);
			} finally {
				await execution.destroy(handle);
			}
		}
	};
}
async function seedWorkspace(execution, handle, workspace) {
	const target = workspace.seedTarget ?? ".";
	const result = await execution.exec(handle, `mkdir -p ${alwaysQuote(target)} && cp -R ${alwaysQuote(`${workspace.seedDir}/.`)} ${alwaysQuote(target)}`);
	if (result.exitCode !== 0) throw new Error(`Failed to seed eval workspace: ${result.stderr || result.stdout}`);
}
async function captureWorkspace(execution, handle, workspace) {
	const capturePaths = workspace.capture?.length ? workspace.capture : ["."];
	const command = [
		`cd ${alwaysQuote(handle.cwd)}`,
		"&&",
		"for p in",
		capturePaths.map(alwaysQuote).join(" "),
		"; do",
		"if [ -e \"$p\" ]; then",
		"find \"$p\" \\( -path \"*/node_modules/*\" -o -path \"*/.git/*\" \\) -prune -o -type f -print0",
		"; fi",
		"done"
	].join(" ");
	const listed = await execution.exec(handle, command);
	if (listed.exitCode !== 0) throw new Error(`Failed to list eval workspace: ${listed.stderr || listed.stdout}`);
	const maxFileChars = workspace.maxFileChars ?? 256 * 1024;
	const files = [];
	const seen = /* @__PURE__ */ new Set();
	for (const rawPath of listed.stdout.split("\0").filter(Boolean)) {
		const path = normalizeSnapshotPath(rawPath);
		if (seen.has(path)) continue;
		seen.add(path);
		const content = await execution.readFile(handle, path);
		const binary = content.includes("\0");
		const truncated = content.length > maxFileChars;
		files.push({
			path,
			size: content.length,
			...binary ? { binary: true } : { content: truncated ? content.slice(0, maxFileChars) : content },
			...truncated ? { truncated: true } : {}
		});
	}
	return {
		cwd: handle.cwd,
		files
	};
}
async function runScorers(ctx, scorers) {
	const scores = [];
	for (const scorer of scorers) try {
		scores.push(await scorer(ctx));
	} catch (err) {
		if (err instanceof EvalMetricError) throw err;
		scores.push({
			name: scorer.name || "anonymous-scorer",
			passed: false,
			score: 0,
			details: err instanceof Error ? err.message : String(err)
		});
	}
	return scores;
}
function aggregateScore(scores) {
	if (scores.length === 0) return 1;
	return scores.reduce((sum, score) => sum + (score.score ?? (score.passed ? 1 : 0)), 0) / scores.length;
}
function efficiencyMetricValues(result) {
	const u = result.usage;
	const readTotal = u.input + u.cacheRead;
	return {
		"execution-time": result.durationMs,
		"provider-tokens": u.input + u.output,
		"cache-read-rate": readTotal > 0 ? u.cacheRead / readTotal : 0
	};
}
function emitEfficiencyMetrics(emit, result) {
	const values = efficiencyMetricValues(result);
	for (const [id, raw] of Object.entries(values)) emit(id, raw);
}
function finalizeEvalMetrics(specs, raw) {
	return Object.entries(specs).map(([metricId, spec]) => {
		const emitted = raw.has(metricId);
		const value = emitted ? raw.get(metricId) : spec.min;
		return {
			id: metricId,
			raw: emitted ? value : 0,
			normalized: emitted ? normalizeMetric(value, spec) : 0,
			direction: spec.direction,
			min: spec.min,
			max: spec.max,
			tags: spec.tags ?? [],
			...spec.description ? { description: spec.description } : {},
			...emitted ? {} : { missing: true }
		};
	});
}
function computeEvalTagScores(metrics) {
	const byTag = /* @__PURE__ */ new Map();
	for (const metric of metrics) for (const tag of metric.tags) {
		const list = byTag.get(tag) ?? [];
		list.push(metric.normalized);
		byTag.set(tag, list);
	}
	const out = {};
	for (const [tag, values] of byTag) out[tag] = mean(values);
	return out;
}
function meanNormalized(metrics) {
	return metrics.length === 0 ? 0 : mean(metrics.map((m) => m.normalized));
}
function accumulateEvalAgentStats(acc, turn) {
	acc.runs += 1;
	acc.input += turn.input;
	acc.output += turn.output;
	acc.cacheRead += turn.cacheRead;
	acc.cacheCreation += turn.cacheCreation;
	acc.cost += turn.cost;
	acc.turns += turn.turns;
	acc.toolCalls += turn.toolCalls;
	acc.durationMs += turn.durationMs;
}
function countToolCallsInTurns(turns) {
	let count = 0;
	for (const turn of turns) {
		if (turn.role !== "assistant") continue;
		count += turn.content.filter((block) => block.type === "tool_call").length;
	}
	return count;
}
function mean(values) {
	return values.length === 0 ? 0 : values.reduce((a, b) => a + b, 0) / values.length;
}
function isRecord(value) {
	return typeof value === "object" && value !== null;
}
function createEvalColors(enabled = shouldUseColor()) {
	if (!enabled) return {
		heading: (value) => value,
		pass: (value) => value,
		fail: (value) => value,
		muted: (value) => value,
		cost: (value) => value,
		status: (passed) => passed ? "pass" : "fail",
		score: (value) => value.toFixed(2),
		caseStatus: (_passed, value) => value,
		dot: (passed) => passed ? "+" : "x",
		tag: (value) => value,
		step: (_kind, value) => value
	};
	return {
		heading: (value) => chalk.bold(value),
		pass: (value) => chalk.green(value),
		fail: (value) => chalk.red(value),
		muted: (value) => chalk.gray(value),
		cost: (value) => chalk.cyan(value),
		status: (passed) => passed ? chalk.green("pass") : chalk.red("fail"),
		score: (value) => value >= .9 ? chalk.green(value.toFixed(2)) : value >= .7 ? chalk.yellow(value.toFixed(2)) : chalk.red(value.toFixed(2)),
		caseStatus: (passed, value) => passed ? chalk.green(value) : chalk.red(value),
		dot: (passed) => passed ? chalk.green("●") : chalk.red("●"),
		tag: (value) => chalk.bold(chalk.whiteBright(value)),
		step: (kind, value) => paintStep(kind, value)
	};
}
/** Per-kind trajectory colors: think=magenta, text=blue, tools by family. */
function paintStep(kind, value) {
	if (kind === "think") return chalk.magenta(value);
	if (kind === "text") return chalk.blue(value);
	if (/write|edit|create/.test(value)) return chalk.green(value);
	if (/read|list|glob|grep|search/.test(value)) return chalk.cyan(value);
	if (/shell|bash|exec|run/.test(value)) return chalk.yellow(value);
	if (/spawn|task|agent/.test(value)) return chalk.magentaBright(value);
	return chalk.white(value);
}
function formatTrajectoryColored(trajectory, color) {
	return trajectory.steps.map((step) => {
		const base = step.kind === "tool" ? step.name ?? "tool" : step.kind;
		const labelWithCount = step.count > 1 ? `${base} ×${step.count}` : base;
		const painted = color.step(step.kind, labelWithCount);
		const timing = step.durationMs && step.durationMs >= 100 ? color.muted(`(${formatDuration(step.durationMs)})`) : "";
		return timing ? `${painted} ${timing}` : painted;
	}).join(color.muted(" → "));
}
function shouldUseColor() {
	return Boolean(process.stdout.isTTY && !process.env.NO_COLOR);
}
function renderJudgePrompt(input) {
	return [
		"Rubric:",
		input.rubric,
		"",
		"Output to grade:",
		input.input,
		"",
		"Return JSON only."
	].join("\n");
}
function defaultJudgeInput(ctx) {
	const chunks = [`Final answer:\n${ctx.result.finalText}`];
	if (ctx.workspace) chunks.push(`Workspace files:\n${ctx.workspace.files.map((file) => [`--- ${file.path}${file.truncated ? " (truncated)" : ""} ---`, file.binary ? "[binary omitted]" : file.content ?? ""].join("\n")).join("\n")}`);
	return chunks.join("\n\n");
}
function parseJudgeOutput(raw) {
	const trimmed = stripCodeFences(raw.trim());
	const jsonText = extractJsonObject(trimmed);
	if (jsonText) try {
		const parsed = JSON.parse(jsonText);
		if (parsed.score !== void 0 || parsed.reasoning !== void 0) return {
			score: clampScore(parsed.score),
			reasoning: typeof parsed.reasoning === "string" ? parsed.reasoning : "",
			...typeof parsed.feedback === "string" ? { feedback: parsed.feedback } : {},
			...jsonText === trimmed ? {} : { raw: trimmed }
		};
	} catch {}
	const scoreFragment = trimmed.match(/score["']?\s*[:=]\s*(-?\d+(?:\.\d+)?)/i);
	if (scoreFragment) return {
		score: clampScore(Number(scoreFragment[1])),
		reasoning: trimmed.slice(0, 500),
		raw: trimmed
	};
	const bareFraction = trimmed.match(/^(-?\d+(?:\.\d+)?)\s*\/\s*(\d+(?:\.\d+)?)$/);
	if (bareFraction) {
		const denom = Number(bareFraction[2]);
		return {
			score: clampScore(denom === 0 ? 0 : Number(bareFraction[1]) / denom),
			reasoning: trimmed,
			raw: trimmed
		};
	}
	if (trimmed.match(/^-?\d+(?:\.\d+)?$/)) return {
		score: clampScore(Number(trimmed)),
		reasoning: "",
		raw: trimmed
	};
	return {
		score: 0,
		reasoning: "Judge did not return a parseable score.",
		raw: trimmed
	};
}
/** Strip a leading/trailing markdown code fence (``` or ```json). */
function stripCodeFences(text) {
	const fence = text.match(/^```(?:json)?[ \t]*\n([\s\S]*?)\n```$/i);
	return fence ? fence[1].trim() : text;
}
/** Extract the first balanced `{...}` object, ignoring braces inside strings. */
function extractJsonObject(text) {
	const start = text.indexOf("{");
	if (start === -1) return void 0;
	let depth = 0;
	let inString = false;
	let escaped = false;
	for (let i = start; i < text.length; i++) {
		const ch = text[i];
		if (inString) {
			if (escaped) escaped = false;
			else if (ch === "\\") escaped = true;
			else if (ch === "\"") inString = false;
			continue;
		}
		if (ch === "\"") inString = true;
		else if (ch === "{") depth++;
		else if (ch === "}") {
			depth--;
			if (depth === 0) return text.slice(start, i + 1);
		}
	}
}
function clampScore(value) {
	const n = typeof value === "number" ? value : Number(value);
	if (!Number.isFinite(n)) return 0;
	return Math.max(0, Math.min(1, n));
}
async function writeEvalArtifacts(dir, result) {
	await mkdir(dir, { recursive: true });
	const resultPath = join(dir, "result.json");
	const eventsPath = join(dir, "events.jsonl");
	const transcriptPath = join(dir, "transcript.json");
	const workspacePath = result.workspace ? join(dir, "workspace.json") : void 0;
	await writeFile(resultPath, `${JSON.stringify(result, null, 2)}\n`);
	await writeFile(eventsPath, result.events.map(headlessEventToJsonl).join(""));
	await writeFile(transcriptPath, `${JSON.stringify(result.result.transcript, null, 2)}\n`);
	if (workspacePath) await writeFile(workspacePath, `${JSON.stringify(result.workspace, null, 2)}\n`);
	return {
		dir,
		result: resultPath,
		events: eventsPath,
		transcript: transcriptPath,
		...workspacePath ? { workspace: workspacePath } : {}
	};
}
function normalizeSnapshotPath(path) {
	const normalized = path.startsWith("./") ? path.slice(2) : path;
	return normalized === "" ? "." : normalized;
}
function safeSegment(value) {
	const sanitized = value.replace(/[^\w.-]+/g, "-").replace(/^-+|-+$/g, "");
	if (sanitized === value && sanitized.length > 0) return sanitized;
	return `${sanitized || "eval"}-${shortHash(value)}`;
}
function shortHash(value) {
	let hash = 2166136261;
	for (let i = 0; i < value.length; i++) {
		hash ^= value.charCodeAt(i);
		hash = Math.imul(hash, 16777619);
	}
	return (hash >>> 0).toString(36).padStart(7, "0").slice(0, 7);
}
function artifactPath(root, result) {
	return resolve(root, safeSegment(result.suite ?? "eval"), safeSegment(result.id));
}
function relativeArtifactPath(root, path) {
	return relative(root, path);
}
//#endregion
export { EFFICIENCY_METRICS, EvalMetricError, artifactPath, buildEvalRunSummary, buildRegisteredEvals, buildTrajectory, clearRegisteredEvals, computeEvalTagScores, createEvalAgent, createEvalRunReporter, createReusableExecutionContext, defineEval, defineMetrics, efficiencyMetricValues, emitEfficiencyMetrics, fileContains, fileContentQuality, fileExists, fileExistsOneOf, finalizeEvalMetrics, formatEvalCaseSummary, formatEvalRunSummary, formatTrajectoryLine, functionalityMetric, llmJudge, normalizeMetric, registerEvalTests, relativeArtifactPath, runEvalCase, statusCompleted };

//# sourceMappingURL=eval.js.map