import {
  buildAgentProfileCell,
  validateRunRecord,
  verifyAgentProfileCell
} from "./chunk-NCK5QLGT.js";
import {
  researchReport
} from "./chunk-OLIBRKRD.js";
import {
  RunIntegrityError,
  assertRunCaptured
} from "./chunk-UBPIXOC4.js";
import {
  TraceEmitter
} from "./chunk-TVVP3ZZQ.js";
import {
  canonicalize,
  hashJson
} from "./chunk-VSMTAMNK.js";
import {
  assertLlmRoute
} from "./chunk-VXNVVBZO.js";
import {
  FileSystemRawProviderSink
} from "./chunk-PC4UYEBM.js";

// src/eval-campaign.ts
var DEFAULT_INTEGRITY = {
  llmSpansMin: 1,
  requireRawCoverageOfLlmSpans: true,
  requireOutcome: true
};
var DEFAULT_ROUTE = {
  requireExplicitBaseUrl: true,
  requireAuth: true
};
async function runEvalCampaign(opts) {
  assertLlmRoute(opts.llmOpts, opts.routeRequirements ?? DEFAULT_ROUTE);
  if (opts.variants.length === 0) {
    throw new Error("runEvalCampaign: variants must be non-empty.");
  }
  if (opts.scenarios.length === 0) {
    throw new Error("runEvalCampaign: scenarios must be non-empty.");
  }
  const variantIds = /* @__PURE__ */ new Set();
  for (const v of opts.variants) {
    if (variantIds.has(v.id)) {
      throw new Error(`runEvalCampaign: duplicate variant id "${v.id}".`);
    }
    variantIds.add(v.id);
  }
  const scenarioIds = /* @__PURE__ */ new Set();
  for (const s of opts.scenarios) {
    if (scenarioIds.has(s.scenarioId)) {
      throw new Error(`runEvalCampaign: duplicate scenarioId "${s.scenarioId}".`);
    }
    scenarioIds.add(s.scenarioId);
  }
  if (opts.report?.comparator && !variantIds.has(opts.report.comparator)) {
    throw new Error(
      `runEvalCampaign: report.comparator "${opts.report.comparator}" is not a configured variantId.`
    );
  }
  if (!opts.commitSha) {
    throw new Error("runEvalCampaign: commitSha is required (every RunRecord needs it).");
  }
  const seeds = opts.seeds ?? [0, 1, 2];
  const splitTag = opts.splitTag ?? "holdout";
  const concurrency = Math.max(1, opts.concurrency ?? 1);
  const integrity = { ...DEFAULT_INTEGRITY, ...opts.integrity ?? {} };
  const onIntegrityFailure = opts.onIntegrityFailure ?? "mark_failed";
  const now = opts.now ?? (() => Date.now());
  const baseUrl = (opts.llmOpts.baseUrl ?? "").replace(/\/+$/, "");
  const provider = opts.llmOpts.provider ?? null;
  const preregistrationHash = opts.preregistrationHash ?? null;
  const rawSinkFactory = opts.rawSinkFactory ?? defaultRawSinkFactory(opts.workDir);
  const campaignFingerprint = await hashJson(
    canonicalize({
      campaignId: opts.campaignId,
      variants: opts.variants.map((v) => v.id).sort(),
      scenarios: opts.scenarios.map((s) => s.scenarioId).sort(),
      seeds: [...seeds].sort((a, b) => a - b),
      splitTag,
      comparator: opts.report?.comparator ?? null,
      baseUrl,
      provider,
      preregistrationHash
    })
  );
  const cells = [];
  for (const variant of opts.variants) {
    for (const scenario of opts.scenarios) {
      for (const seed of seeds) {
        cells.push({ variant, scenario, seed });
      }
    }
  }
  const startedAt = new Date(now()).toISOString();
  const runs = [];
  const integrityReports = [];
  const failedRuns = [];
  let cursor = 0;
  async function worker() {
    while (true) {
      const i = cursor++;
      if (i >= cells.length) return;
      const cell = cells[i];
      try {
        const result = await runOneCell(cell);
        runs.push(result.record);
        integrityReports.push(result.integrity);
      } catch (err) {
        if (err instanceof CellExecutionError) {
          failedRuns.push(err.failed);
          if (err.integrity) integrityReports.push(err.integrity);
        } else {
          throw err;
        }
      }
    }
  }
  async function runOneCell(cell) {
    const runId = (opts.runId ?? defaultRunId)({
      campaignId: opts.campaignId,
      runId: "",
      // unused by default generator
      variantId: cell.variant.id,
      scenarioId: cell.scenario.scenarioId,
      seed: cell.seed
    });
    const factoryParams = {
      campaignId: opts.campaignId,
      runId,
      variantId: cell.variant.id,
      scenarioId: cell.scenario.scenarioId,
      seed: cell.seed
    };
    const store = opts.storeFactory(factoryParams);
    const rawSink = rawSinkFactory(factoryParams);
    const emitter = new TraceEmitter(store, {
      runId,
      now: opts.now,
      onRunComplete: opts.onRunComplete
    });
    const llmOpts = {
      ...opts.llmOpts,
      rawSink,
      traceContext: { runId }
    };
    const ctx = {
      runId,
      experimentId: opts.campaignId,
      variant: cell.variant.payload,
      variantId: cell.variant.id,
      scenarioId: cell.scenario.scenarioId,
      scenarioTags: cell.scenario.tags ?? {},
      seed: cell.seed,
      splitTag,
      emitter,
      store,
      rawSink,
      llmOpts
    };
    const wallStart = now();
    let outcome;
    try {
      outcome = await opts.runner(ctx);
    } catch (err) {
      const message = err instanceof Error ? err.message : String(err);
      try {
        await emitter.abortRun(message);
      } catch {
      }
      throw new CellExecutionError({
        runId,
        variantId: cell.variant.id,
        scenarioId: cell.scenario.scenarioId,
        seed: cell.seed,
        reason: "runner_threw",
        error: message
      });
    }
    const wallMs = now() - wallStart;
    const integrityReport = await assertRunCaptured(store, runId, { ...integrity, rawSink });
    if (!integrityReport.ok) {
      switch (onIntegrityFailure) {
        case "throw":
          throw new RunIntegrityError(integrityReport);
        case "mark_failed":
          throw new CellExecutionError(
            {
              runId,
              variantId: cell.variant.id,
              scenarioId: cell.scenario.scenarioId,
              seed: cell.seed,
              reason: "integrity_failed",
              error: integrityReport.issues.map((i) => i.code).join(", ")
            },
            integrityReport
          );
        case "log":
          break;
      }
    }
    const recordOutcome = {
      raw: outcome.raw ?? {}
    };
    if (splitTag === "holdout") recordOutcome.holdoutScore = outcome.score;
    else recordOutcome.searchScore = outcome.score;
    if (outcome.judgeScores !== void 0) recordOutcome.judgeScores = outcome.judgeScores;
    const record = {
      runId,
      experimentId: opts.campaignId,
      candidateId: cell.variant.id,
      seed: cell.seed,
      model: outcome.model,
      promptHash: outcome.promptHash,
      configHash: outcome.configHash,
      commitSha: opts.commitSha,
      wallMs,
      costUsd: outcome.costUsd,
      tokenUsage: outcome.tokenUsage,
      judgeMetadata: outcome.judgeMetadata,
      outcome: recordOutcome,
      ...outcome.failureClass ? { failureClass: outcome.failureClass } : {},
      failureMode: outcome.failureMode,
      splitTag,
      scenarioId: cell.scenario.scenarioId
    };
    const profileSource = outcome.agentProfile ?? (typeof opts.agentProfile === "function" ? await opts.agentProfile({
      campaignId: opts.campaignId,
      runId,
      variantId: cell.variant.id,
      scenarioId: cell.scenario.scenarioId,
      seed: cell.seed,
      variant: cell.variant.payload,
      scenarioTags: cell.scenario.tags ?? {}
    }) : opts.agentProfile);
    if (profileSource !== void 0) {
      const agentProfile = await resolveAgentProfileCell(profileSource);
      assertAgentProfileMatchesRun(agentProfile, outcome.model, outcome.promptHash);
      record.agentProfile = agentProfile;
    }
    return { record: validateRunRecord(record), integrity: integrityReport };
  }
  const workers = Array.from({ length: Math.min(concurrency, cells.length) }, () => worker());
  await Promise.all(workers);
  let report;
  if (opts.report) {
    const reportOpts = {
      ...opts.report,
      comparator: opts.report.comparator,
      split: splitTag === "dev" ? "search" : splitTag,
      generatedAt: new Date(now()).toISOString(),
      preregistrationHash: preregistrationHash ?? void 0
    };
    report = await researchReport(runs, reportOpts);
  }
  const endedAt = new Date(now()).toISOString();
  return {
    campaignId: opts.campaignId,
    campaignFingerprint,
    preregistrationHash,
    runs,
    integrityReports,
    failedRuns,
    report,
    startedAt,
    endedAt
  };
}
var CellExecutionError = class extends Error {
  failed;
  integrity;
  constructor(failed, integrity) {
    super(`cell ${failed.variantId}/${failed.scenarioId}@${failed.seed} failed: ${failed.reason}`);
    this.failed = failed;
    this.integrity = integrity;
  }
};
function defaultRawSinkFactory(workDir) {
  return (params) => {
    if (!workDir) {
      throw new Error(
        "runEvalCampaign: rawSinkFactory not supplied and workDir not set. Pass either to enable raw provider capture, or pass `new NoopRawProviderSink()` via rawSinkFactory to opt out explicitly."
      );
    }
    return new FileSystemRawProviderSink({
      dir: `${workDir}/raw-events/${params.runId}`
    });
  };
}
async function resolveAgentProfileCell(input) {
  if (isAgentProfileCell(input)) {
    if (!await verifyAgentProfileCell(input)) {
      throw new Error(`runEvalCampaign: agentProfile.cellId does not match its content`);
    }
    return input;
  }
  return buildAgentProfileCell(input);
}
function isAgentProfileCell(input) {
  return "schemaVersion" in input && "cellId" in input;
}
function assertAgentProfileMatchesRun(profile, model, promptHash) {
  if (profile.model !== void 0 && profile.model !== model) {
    throw new Error(
      `runEvalCampaign: agentProfile.model "${profile.model}" does not match outcome.model "${model}"`
    );
  }
  if (profile.promptHash !== void 0 && profile.promptHash !== promptHash) {
    throw new Error(
      `runEvalCampaign: agentProfile.promptHash "${profile.promptHash}" does not match outcome.promptHash "${promptHash}"`
    );
  }
}
function defaultRunId(params) {
  const base = `${params.campaignId}::${params.variantId}::${params.scenarioId}::${params.seed}`;
  let h1 = 2166136261;
  let h2 = 305419896;
  for (let i = 0; i < base.length; i++) {
    const c = base.charCodeAt(i);
    h1 = Math.imul(h1 ^ c, 16777619) >>> 0;
    h2 = Math.imul(h2 ^ c, 2654435761) >>> 0;
  }
  return `run-${h1.toString(16).padStart(8, "0")}${h2.toString(16).padStart(8, "0")}`;
}

export {
  runEvalCampaign
};
//# sourceMappingURL=chunk-GM476SZU.js.map