import {
  confidenceInterval
} from "./chunk-56GC6VYO.js";
import {
  AgentEvalError
} from "./chunk-3BFEG2F6.js";

// src/campaign/run-campaign.ts
import { createHash } from "crypto";
import { join } from "path";

// src/integrity/backend-integrity.ts
var BackendIntegrityError = class extends AgentEvalError {
  constructor(message, report) {
    super("backend_integrity", message);
    this.report = report;
  }
  report;
};
function isStubRecord(rec) {
  return rec.tokenUsage.input === 0 && rec.tokenUsage.output === 0;
}
function isUncostedRecord(rec) {
  return rec.tokenUsage.output > 0 && rec.costUsd === 0;
}
function summarizeBackendIntegrity(records) {
  const totalRecords = records.length;
  let stubRecords = 0;
  let realRecords = 0;
  let uncostedRecords = 0;
  let totalInputTokens = 0;
  let totalOutputTokens = 0;
  let totalCostUsd = 0;
  for (const rec of records) {
    totalInputTokens += rec.tokenUsage.input;
    totalOutputTokens += rec.tokenUsage.output;
    totalCostUsd += rec.costUsd;
    if (isStubRecord(rec)) stubRecords++;
    else realRecords++;
    if (isUncostedRecord(rec)) uncostedRecords++;
  }
  const verdict = totalRecords === 0 ? "stub" : stubRecords === totalRecords ? "stub" : stubRecords === 0 ? "real" : "mixed";
  const diagnosis = buildDiagnosis({
    totalRecords,
    stubRecords,
    realRecords,
    uncostedRecords,
    totalInputTokens,
    totalOutputTokens,
    totalCostUsd,
    verdict
  });
  return {
    totalRecords,
    stubRecords,
    realRecords,
    uncostedRecords,
    totalInputTokens,
    totalOutputTokens,
    totalCostUsd,
    verdict,
    diagnosis
  };
}
function buildDiagnosis(r) {
  if (r.totalRecords === 0) {
    return "no records \u2014 eval produced zero runs; backend likely failed before first turn";
  }
  if (r.verdict === "stub") {
    return [
      `all ${r.totalRecords} records have zero token usage \u2014 the LLM backend was never called.`,
      "common causes: --backend sandbox without a sandbox bridge running; stub model returning hard-coded strings;",
      "auth misconfigured so requests were silently dropped before the LLM. Re-run with --backend tcloud and TANGLE_API_KEY set,",
      "or boot the cli-bridge / sandbox before invoking the eval."
    ].join(" ");
  }
  if (r.verdict === "mixed") {
    const pct = (r.stubRecords / r.totalRecords * 100).toFixed(0);
    return [
      `${r.stubRecords}/${r.totalRecords} records (${pct}%) have zero token usage \u2014 the backend partially failed.`,
      "common causes: rate-limit cascade (429s after the first N personas);",
      "transient auth expiry mid-run; provider outage. Treat the affected records as missing data, not agent failures."
    ].join(" ");
  }
  if (r.uncostedRecords > 0) {
    const pct = (r.uncostedRecords / r.totalRecords * 100).toFixed(0);
    return [
      `${r.totalRecords} records with real LLM activity (in=${r.totalInputTokens}, out=${r.totalOutputTokens} tokens).`,
      `${r.uncostedRecords} (${pct}%) have output tokens but costUsd=0. Two distinct roots:`,
      "(a) cost ledger mis-wired \u2014 no usage propagation from the runtime stream into RunRecord; or",
      "(b) the model is unpriced at the source (sandbox/router returned $0 despite real tokens).",
      "For (b), price the measured tokens against the substrate table (estimateCost) instead of leaving $0."
    ].join(" ");
  }
  return `${r.totalRecords} records with real LLM activity (in=${r.totalInputTokens}, out=${r.totalOutputTokens} tokens, $${r.totalCostUsd.toFixed(4)}).`;
}
function assertRealBackend(records, opts = {}) {
  const report = summarizeBackendIntegrity(records);
  const allowMixed = opts.allowMixed ?? true;
  if (report.verdict === "stub") {
    throw new BackendIntegrityError(
      `backend-integrity: ran against a stub or unconfigured backend \u2014 ${report.diagnosis}`,
      report
    );
  }
  if (!allowMixed && report.verdict === "mixed") {
    throw new BackendIntegrityError(
      `backend-integrity: partial backend failure rejected \u2014 ${report.diagnosis}`,
      report
    );
  }
  return report;
}

// src/campaign/storage.ts
import { createRequire } from "module";
function fsCampaignStorage() {
  const nodeRequire = createRequire(import.meta.url);
  const { existsSync, mkdirSync, readFileSync, writeFileSync } = nodeRequire(
    "node:fs"
  );
  return {
    ensureDir(dir) {
      if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
    },
    exists(path) {
      return existsSync(path);
    },
    read(path) {
      try {
        return readFileSync(path, "utf8");
      } catch {
        return void 0;
      }
    },
    write(path, content) {
      writeFileSync(path, content);
    }
  };
}
function inMemoryCampaignStorage() {
  const files = /* @__PURE__ */ new Map();
  const dirs = /* @__PURE__ */ new Set();
  return {
    ensureDir(dir) {
      dirs.add(dir);
    },
    exists(path) {
      return files.has(path) || dirs.has(path);
    },
    read(path) {
      const value = files.get(path);
      if (value === void 0) return void 0;
      return typeof value === "string" ? value : new TextDecoder().decode(value);
    },
    write(path, content) {
      files.set(path, content);
    }
  };
}

// src/campaign/run-campaign.ts
async function runCampaign(opts) {
  const seed = opts.seed ?? 42;
  const reps = opts.reps ?? 1;
  const resumable = opts.resumable ?? true;
  const maxConcurrency = opts.maxConcurrency ?? 2;
  const now = opts.now ?? (() => /* @__PURE__ */ new Date());
  const judges = opts.judges ?? [];
  const storage = opts.storage ?? fsCampaignStorage();
  if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) {
    throw new Error("runCampaign: runDir is required and must be a non-empty string");
  }
  storage.ensureDir(opts.runDir);
  const manifestHash = computeManifestHash({
    scenarios: opts.scenarios,
    judges,
    dispatchRef: opts.dispatch.name || "anonymous",
    seed,
    reps
  });
  const startedAt = now();
  const cells = [];
  const artifactsByPath = {};
  const schedule = [];
  let cellIndex = 0;
  for (const scenario of opts.scenarios) {
    for (let rep = 0; rep < reps; rep++) {
      const cellId = `${scenario.id}:${rep}`;
      const cellSeed = seed + cellIndex;
      schedule.push({ scenario, rep, cellId, cellSeed });
      cellIndex += 1;
    }
  }
  let totalCostUsd = 0;
  let costCeilingReached = false;
  const abortController = new AbortController();
  const lanes = [];
  let nextIdx = 0;
  const cellsRef = cells;
  for (let i = 0; i < maxConcurrency; i++) {
    lanes.push(
      (async () => {
        while (true) {
          const myIdx = nextIdx++;
          if (myIdx >= schedule.length) return;
          const slot = schedule[myIdx];
          if (costCeilingReached) {
            cellsRef.push(skippedCell(slot, "cost_ceiling_reached"));
            continue;
          }
          const result = await executeCell({
            slot,
            opts,
            manifestHash,
            resumable,
            now,
            storage,
            buildTraceWriter: opts.buildTraceWriter ?? defaultBuildTraceWriter(storage),
            signal: abortController.signal,
            dispatchTimeoutMs: opts.dispatchTimeoutMs
          });
          cellsRef.push(result.cell);
          enforceCellUsage(result.cell, opts.expectUsage ?? "warn");
          totalCostUsd += result.cell.costUsd;
          Object.assign(artifactsByPath, result.artifactsByPath);
          if (opts.costCeiling !== void 0 && totalCostUsd >= opts.costCeiling) {
            costCeilingReached = true;
          }
          if (opts.labeledStore && opts.labeledStore !== "off" && !result.cell.error) {
            await captureToStore({
              store: opts.labeledStore,
              cell: result.cell,
              scenario: slot.scenario,
              opts,
              now
            }).catch((err) => {
              console.warn(
                `[runCampaign] capture failed for ${result.cell.cellId}: ${err instanceof Error ? err.message : String(err)}`
              );
            });
          }
        }
      })()
    );
  }
  await Promise.all(lanes);
  const endedAt = now();
  cellsRef.sort((a, b) => a.cellId.localeCompare(b.cellId));
  const aggregates = computeAggregates(
    cellsRef,
    judges,
    seed
  );
  return {
    manifestHash,
    seed,
    startedAt: startedAt.toISOString(),
    endedAt: endedAt.toISOString(),
    durationMs: endedAt.getTime() - startedAt.getTime(),
    cells: cellsRef,
    aggregates,
    runDir: opts.runDir,
    artifactsByPath,
    scenarios: opts.scenarios.map((s) => ({ id: s.id, kind: s.kind }))
  };
}
async function executeCell(args) {
  const storage = args.storage;
  const cellDir = join(args.opts.runDir, args.slot.cellId.replace(/[^a-zA-Z0-9_-]/g, "_"));
  storage.ensureDir(cellDir);
  const cachePath = join(cellDir, "cached-result.json");
  if (args.resumable) {
    const raw = storage.read(cachePath);
    if (raw !== void 0) {
      try {
        const cached = JSON.parse(raw);
        if (cached.cellId === args.slot.cellId) {
          return { cell: { ...cached, cached: true }, artifactsByPath: {} };
        }
      } catch {
      }
    }
  }
  const startMs = Date.now();
  const trace = args.buildTraceWriter(args.slot.cellId, cellDir);
  const artifactsByPath = {};
  const artifacts = {
    async write(path, content) {
      const fullPath = join(cellDir, path);
      storage.ensureDir(join(fullPath, ".."));
      storage.write(fullPath, content);
      artifactsByPath[`${args.slot.cellId}/${path}`] = fullPath;
      return fullPath;
    },
    async writeJson(path, value) {
      return artifacts.write(path, JSON.stringify(value, null, 2));
    }
  };
  let costSoFar = 0;
  const tokensSoFar = { input: 0, output: 0 };
  const cost = {
    observe(amount, source) {
      costSoFar += amount;
      trace.span(`cost.${source}`, { amountUsd: amount }).end();
    },
    observeTokens(usage) {
      tokensSoFar.input += usage.input;
      tokensSoFar.output += usage.output;
      if (usage.cached) tokensSoFar.cached = (tokensSoFar.cached ?? 0) + usage.cached;
    },
    current() {
      return costSoFar;
    },
    tokens() {
      return { ...tokensSoFar };
    }
  };
  const placement = args.opts.cellPlacement?.({
    scenario: args.slot.scenario,
    rep: args.slot.rep
  });
  const cellAbort = new AbortController();
  const onCampaignAbort = () => cellAbort.abort(args.signal.reason);
  if (args.signal.aborted) cellAbort.abort(args.signal.reason);
  else args.signal.addEventListener("abort", onCampaignAbort, { once: true });
  const ctx = {
    cellId: args.slot.cellId,
    rep: args.slot.rep,
    seed: args.slot.cellSeed,
    signal: cellAbort.signal,
    trace,
    artifacts,
    cost,
    placement
  };
  let artifact;
  let errorMessage;
  const timeoutMs = args.dispatchTimeoutMs;
  let timeoutTimer;
  try {
    const dispatched = args.opts.dispatch(args.slot.scenario, ctx);
    if (timeoutMs !== void 0 && timeoutMs > 0) {
      artifact = await Promise.race([
        dispatched,
        new Promise((_, reject) => {
          timeoutTimer = setTimeout(() => {
            cellAbort.abort(new Error("dispatch timeout"));
            reject(
              new Error(
                `dispatch exceeded ${timeoutMs}ms for cell '${args.slot.cellId}' \u2014 aborted and failed loud (no silent hang)`
              )
            );
          }, timeoutMs);
          if (typeof timeoutTimer.unref === "function")
            timeoutTimer.unref();
        })
      ]);
    } else {
      artifact = await dispatched;
    }
  } catch (err) {
    errorMessage = err instanceof Error ? err.message : String(err);
  } finally {
    if (timeoutTimer) clearTimeout(timeoutTimer);
    args.signal.removeEventListener("abort", onCampaignAbort);
  }
  const judgeScores = {};
  if (artifact !== void 0) {
    for (const judge of args.opts.judges ?? []) {
      if (judge.appliesTo && !judge.appliesTo(args.slot.scenario)) continue;
      try {
        judgeScores[judge.name] = await runJudgeCell(judge, {
          artifact,
          scenario: args.slot.scenario,
          signal: args.signal
        });
      } catch (err) {
        errorMessage = `judge '${judge.name}' failed: ${err instanceof Error ? err.message : String(err)}`;
        break;
      }
    }
  }
  await trace.flush();
  const cell = {
    cellId: args.slot.cellId,
    scenarioId: args.slot.scenario.id,
    rep: args.slot.rep,
    artifact: artifact ?? null,
    judgeScores,
    costUsd: costSoFar,
    tokenUsage: { ...tokensSoFar },
    durationMs: Date.now() - startMs,
    seed: args.slot.cellSeed,
    cached: false,
    error: errorMessage
  };
  if (!errorMessage && args.resumable) {
    storage.write(cachePath, JSON.stringify(cell));
  }
  return { cell, artifactsByPath };
}
function enforceCellUsage(cell, mode) {
  if (mode === "off" || cell.error) return;
  if (cell.artifact === null || cell.artifact === void 0) return;
  const zeroTokens = cell.tokenUsage.input === 0 && cell.tokenUsage.output === 0;
  if (cell.costUsd !== 0 || !zeroTokens) return;
  const msg = `cell '${cell.cellId}' produced an artifact but reported zero cost and zero tokens \u2014 the dispatch never reported LLM usage via ctx.cost.observe/observeTokens (a stub cell)`;
  if (mode === "assert") {
    const report = {
      totalRecords: 1,
      stubRecords: 1,
      realRecords: 0,
      uncostedRecords: 0,
      totalInputTokens: 0,
      totalOutputTokens: 0,
      totalCostUsd: 0,
      verdict: "stub",
      diagnosis: msg
    };
    throw new BackendIntegrityError(`expectUsage: ${msg}`, report);
  }
  console.warn(`[runCampaign] expectUsage: ${msg}`);
}
async function runJudgeCell(judge, input) {
  return judge.score(input);
}
function defaultBuildTraceWriter(storage) {
  return (cellId, dir) => {
    const spans = [];
    return {
      span(name, attributes) {
        const startMs = Date.now();
        const record = { name, cellId, startMs, ...attributes ?? {} };
        const finish = {
          end(endAttrs) {
            record.durationMs = Date.now() - startMs;
            if (endAttrs) Object.assign(record, endAttrs);
            spans.push(record);
          },
          setAttribute(key, value) {
            record[key] = value;
          }
        };
        return finish;
      },
      async flush() {
        storage.write(join(dir, "spans.jsonl"), spans.map((s) => JSON.stringify(s)).join("\n"));
      }
    };
  };
}
function skippedCell(slot, reason) {
  return {
    cellId: slot.cellId,
    scenarioId: slot.scenario.id,
    rep: slot.rep,
    artifact: null,
    judgeScores: {},
    costUsd: 0,
    tokenUsage: { input: 0, output: 0 },
    durationMs: 0,
    seed: slot.cellSeed,
    cached: false,
    error: `skipped: ${reason}`
  };
}
async function captureToStore(args) {
  await args.store.observe({
    scenario: args.scenario,
    artifact: args.cell.artifact,
    judgeScores: args.cell.judgeScores,
    source: args.opts.captureSource ?? "eval-run",
    sourceVersionHash: args.opts.captureSourceVersionHash ?? "unknown",
    capturedAt: args.now().toISOString(),
    redactionStatus: "raw"
  });
}
function computeManifestHash(input) {
  const canonical = {
    scenarios: input.scenarios.map((s) => ({ id: s.id, kind: s.kind })),
    judges: input.judges.map((j) => ({ name: j.name, dims: j.dimensions.map((d) => d.key) })),
    dispatch: input.dispatchRef,
    seed: input.seed,
    reps: input.reps
  };
  return createHash("sha256").update(JSON.stringify(canonical)).digest("hex");
}
function computeAggregates(cells, judges, seed) {
  const byJudge = {};
  for (const judge of judges) {
    const scores = [];
    for (const cell of cells) {
      const s = cell.judgeScores[judge.name];
      if (s !== void 0) scores.push(s.composite);
    }
    byJudge[judge.name] = aggregate(scores, seed);
  }
  const byScenario = {};
  const scenarioGroups = /* @__PURE__ */ new Map();
  for (const cell of cells) {
    const composites = Object.values(cell.judgeScores).map((s) => s.composite);
    if (composites.length === 0) continue;
    const mean = composites.reduce((a, b) => a + b, 0) / composites.length;
    const arr = scenarioGroups.get(cell.scenarioId) ?? [];
    arr.push(mean);
    scenarioGroups.set(cell.scenarioId, arr);
  }
  for (const [scenarioId, samples] of scenarioGroups) {
    const ag = aggregate(samples, seed);
    byScenario[scenarioId] = { meanComposite: ag.mean, ci95: ag.ci95, n: ag.n };
  }
  return {
    byJudge,
    byScenario,
    totalCostUsd: cells.reduce((a, c) => a + c.costUsd, 0),
    cellsExecuted: cells.filter((c) => !c.error).length,
    cellsSkipped: cells.filter((c) => c.error?.startsWith("skipped:")).length,
    cellsCached: cells.filter((c) => c.cached).length,
    cellsFailed: cells.filter((c) => c.error && !c.error.startsWith("skipped:")).length
  };
}
function aggregate(samples, seed) {
  const n = samples.length;
  if (n === 0) return { mean: 0, stdev: 0, ci95: [0, 0], n: 0 };
  const mean = samples.reduce((a, b) => a + b, 0) / n;
  const variance = samples.reduce((a, b) => a + (b - mean) ** 2, 0) / Math.max(1, n - 1);
  const stdev = Math.sqrt(variance);
  const ci = confidenceInterval(samples, 0.95, { seed, resamples: 1e3 });
  return { mean, stdev, ci95: [ci.lower, ci.upper], n };
}

export {
  BackendIntegrityError,
  summarizeBackendIntegrity,
  assertRealBackend,
  fsCampaignStorage,
  inMemoryCampaignStorage,
  runCampaign
};
//# sourceMappingURL=chunk-2T4EZACH.js.map