import {
  ValidationError
} from "./chunk-3BFEG2F6.js";

// src/product-benchmark/index.ts
import { existsSync, readFileSync, statSync } from "fs";
import { dirname, join } from "path";
var productBenchmarkSplits = ["practice", "dev", "holdout", "safety", "sentinel"];
function isObject(value) {
  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
}
function fail(path, message) {
  throw new ValidationError(`${path}: ${message}`);
}
function wrapValidationError(path, err) {
  if (err instanceof ValidationError) {
    throw new ValidationError(`${path}: ${err.message}`, { cause: err });
  }
  throw new ValidationError(`${path}: ${err instanceof Error ? err.message : String(err)}`, {
    cause: err
  });
}
function expectObject(value, path) {
  if (!isObject(value)) fail(path, "must be an object");
  return value;
}
function expectString(value, path) {
  if (typeof value !== "string" || value.trim().length === 0)
    fail(path, "must be a non-empty string");
  return value;
}
function expectBoolean(value, path) {
  if (typeof value !== "boolean") fail(path, "must be a boolean");
  return value;
}
function expectNumber(value, path, opts = {}) {
  if (typeof value !== "number" || !Number.isFinite(value)) fail(path, "must be a finite number");
  if (opts.integer && !Number.isInteger(value)) fail(path, "must be an integer");
  if (opts.min !== void 0 && value < opts.min) fail(path, `must be >= ${opts.min}`);
  if (opts.max !== void 0 && value > opts.max) fail(path, `must be <= ${opts.max}`);
  return value;
}
function expectStringArray(value, path) {
  if (!Array.isArray(value)) fail(path, "must be an array");
  return value.map((entry, index) => expectString(entry, `${path}[${index}]`));
}
function expectObjectArray(value, path) {
  if (!Array.isArray(value)) fail(path, "must be an array");
  return value.map((entry, index) => expectObject(entry, `${path}[${index}]`));
}
function expectSplit(value, path) {
  const split = expectString(value, path);
  if (!productBenchmarkSplits.includes(split)) {
    fail(path, `must be one of ${productBenchmarkSplits.join(", ")}`);
  }
  return split;
}
function optionalString(value, path) {
  if (value === void 0) return void 0;
  return expectString(value, path);
}
function expectDimensions(value, path) {
  const obj = expectObject(value, path);
  const out = {};
  for (const [key, raw] of Object.entries(obj)) out[key] = expectNumber(raw, `${path}.${key}`);
  return out;
}
function expectRuntimeResolution(value, path) {
  const obj = expectObject(value, path);
  return {
    model: expectString(obj.model, `${path}.model`),
    harness: expectString(obj.harness, `${path}.harness`),
    backend: expectString(obj.backend, `${path}.backend`),
    ...obj.reasoningEffort !== void 0 ? { reasoningEffort: optionalString(obj.reasoningEffort, `${path}.reasoningEffort`) } : {}
  };
}
function validateProductBenchmarkManifest(value) {
  const obj = expectObject(value, "manifest");
  if (obj.schemaVersion !== 1) fail("manifest.schemaVersion", "must be 1");
  const repo = expectObject(obj.repo, "manifest.repo");
  const substrate = expectObject(obj.substrate, "manifest.substrate");
  const profiles = expectObjectArray(obj.profiles, "manifest.profiles").map((profile, index) => ({
    id: expectString(profile.id, `manifest.profiles[${index}].id`),
    profileHash: expectString(profile.profileHash, `manifest.profiles[${index}].profileHash`),
    agentProfilePath: expectString(
      profile.agentProfilePath,
      `manifest.profiles[${index}].agentProfilePath`
    )
  }));
  const arms = expectObjectArray(obj.arms, "manifest.arms").map((arm, index) => ({
    id: expectString(arm.id, `manifest.arms[${index}].id`),
    profileId: expectString(arm.profileId, `manifest.arms[${index}].profileId`),
    mutableSurfaces: expectStringArray(
      arm.mutableSurfaces,
      `manifest.arms[${index}].mutableSurfaces`
    ),
    policyAxes: expectObject(arm.policyAxes, `manifest.arms[${index}].policyAxes`)
  }));
  const scenarios = expectObjectArray(obj.scenarios, "manifest.scenarios").map(
    (scenario, index) => ({
      id: expectString(scenario.id, `manifest.scenarios[${index}].id`),
      split: expectSplit(scenario.split, `manifest.scenarios[${index}].split`),
      tags: expectStringArray(scenario.tags, `manifest.scenarios[${index}].tags`),
      sourceAllowedForSynthesis: expectBoolean(
        scenario.sourceAllowedForSynthesis,
        `manifest.scenarios[${index}].sourceAllowedForSynthesis`
      )
    })
  );
  const budgets = expectObject(obj.budgets, "manifest.budgets");
  if (profiles.length === 0) fail("manifest.profiles", "must contain at least one profile");
  if (arms.length === 0) fail("manifest.arms", "must contain at least one arm");
  if (scenarios.length === 0) fail("manifest.scenarios", "must contain at least one scenario");
  assertUnique(
    profiles.map((profile) => profile.id),
    "manifest.profiles.id"
  );
  assertUnique(
    arms.map((arm) => arm.id),
    "manifest.arms.id"
  );
  assertUnique(
    scenarios.map((scenario) => scenario.id),
    "manifest.scenarios.id"
  );
  const profileIds = new Set(profiles.map((profile) => profile.id));
  for (const arm of arms) {
    if (!profileIds.has(arm.profileId))
      fail(`manifest.arms.${arm.id}.profileId`, `unknown profile ${arm.profileId}`);
  }
  return {
    schemaVersion: 1,
    projectId: expectString(obj.projectId, "manifest.projectId"),
    benchmarkId: expectString(obj.benchmarkId, "manifest.benchmarkId"),
    repo: {
      url: expectString(repo.url, "manifest.repo.url"),
      commit: expectString(repo.commit, "manifest.repo.commit"),
      branch: expectString(repo.branch, "manifest.repo.branch")
    },
    substrate: {
      agentEval: expectString(substrate.agentEval, "manifest.substrate.agentEval"),
      agentRuntime: expectString(substrate.agentRuntime, "manifest.substrate.agentRuntime"),
      agentInterface: expectString(substrate.agentInterface, "manifest.substrate.agentInterface"),
      sandbox: expectString(substrate.sandbox, "manifest.substrate.sandbox"),
      ...substrate.agentBench !== void 0 ? { agentBench: expectString(substrate.agentBench, "manifest.substrate.agentBench") } : {}
    },
    profiles,
    arms,
    scenarios,
    budgets: {
      maxUsd: expectNumber(budgets.maxUsd, "manifest.budgets.maxUsd", { min: 0 }),
      maxCells: expectNumber(budgets.maxCells, "manifest.budgets.maxCells", {
        min: 0,
        integer: true
      }),
      maxWallMs: expectNumber(budgets.maxWallMs, "manifest.budgets.maxWallMs", {
        min: 0,
        integer: true
      })
    },
    expectedArtifactDir: expectString(obj.expectedArtifactDir, "manifest.expectedArtifactDir")
  };
}
function validateProductBenchmarkRecord(value) {
  const obj = expectObject(value, "record");
  if (obj.schemaVersion !== 1) fail("record.schemaVersion", "must be 1");
  const agentProfile = expectObject(obj.agentProfile, "record.agentProfile");
  const model = expectObject(obj.model, "record.model");
  const backend = expectObject(obj.backend, "record.backend");
  const outcome = expectObject(obj.outcome, "record.outcome");
  const usage = expectObject(obj.usage, "record.usage");
  const integrity = expectObject(obj.integrity, "record.integrity");
  const artifacts = expectObject(obj.artifacts, "record.artifacts");
  const record = {
    schemaVersion: 1,
    projectId: expectString(obj.projectId, "record.projectId"),
    benchmarkId: expectString(obj.benchmarkId, "record.benchmarkId"),
    runId: expectString(obj.runId, "record.runId"),
    scenarioId: expectString(obj.scenarioId, "record.scenarioId"),
    split: expectSplit(obj.split, "record.split"),
    armId: expectString(obj.armId, "record.armId"),
    rep: expectNumber(obj.rep, "record.rep", { min: 1, integer: true }),
    agentProfile: {
      id: expectString(agentProfile.id, "record.agentProfile.id"),
      hash: expectString(agentProfile.hash, "record.agentProfile.hash"),
      path: expectString(agentProfile.path, "record.agentProfile.path"),
      declared: expectRuntimeResolution(agentProfile.declared, "record.agentProfile.declared"),
      resolved: expectRuntimeResolution(agentProfile.resolved, "record.agentProfile.resolved")
    },
    model: {
      provider: expectString(model.provider, "record.model.provider"),
      id: expectString(model.id, "record.model.id")
    },
    backend: {
      kind: expectString(backend.kind, "record.backend.kind"),
      version: expectString(backend.version, "record.backend.version")
    },
    outcome: {
      pass: expectBoolean(outcome.pass, "record.outcome.pass"),
      score: expectNumber(outcome.score, "record.outcome.score", { min: 0, max: 1 }),
      dimensions: expectDimensions(outcome.dimensions, "record.outcome.dimensions"),
      failureMode: outcome.failureMode === null ? null : expectString(outcome.failureMode, "record.outcome.failureMode")
    },
    usage: {
      inputTokens: expectNumber(usage.inputTokens, "record.usage.inputTokens", {
        min: 0,
        integer: true
      }),
      outputTokens: expectNumber(usage.outputTokens, "record.usage.outputTokens", {
        min: 0,
        integer: true
      }),
      costUsd: expectNumber(usage.costUsd, "record.usage.costUsd", { min: 0 }),
      wallMs: expectNumber(usage.wallMs, "record.usage.wallMs", { min: 0, integer: true }),
      toolCalls: expectNumber(usage.toolCalls, "record.usage.toolCalls", { min: 0, integer: true })
    },
    integrity: {
      realBackend: expectBoolean(integrity.realBackend, "record.integrity.realBackend"),
      rawCapture: expectBoolean(integrity.rawCapture, "record.integrity.rawCapture"),
      traceCapture: expectBoolean(integrity.traceCapture, "record.integrity.traceCapture"),
      noStubRows: expectBoolean(integrity.noStubRows, "record.integrity.noStubRows"),
      priced: expectBoolean(integrity.priced, "record.integrity.priced"),
      profileMaterialized: expectBoolean(
        integrity.profileMaterialized,
        "record.integrity.profileMaterialized"
      )
    },
    artifacts: {
      records: expectString(artifacts.records, "record.artifacts.records"),
      traces: expectString(artifacts.traces, "record.artifacts.traces"),
      raws: expectString(artifacts.raws, "record.artifacts.raws"),
      scores: expectString(artifacts.scores, "record.artifacts.scores"),
      workspace: expectString(artifacts.workspace, "record.artifacts.workspace")
    }
  };
  if (record.integrity.realBackend && record.usage.inputTokens + record.usage.outputTokens === 0) {
    fail("record.usage", "realBackend rows must carry non-zero token usage");
  }
  return record;
}
function productBenchmarkIntegrityFailures(record) {
  const failures = [];
  for (const [key, value] of Object.entries(record.integrity)) {
    if (!value) failures.push(`${record.runId}:${key}=false`);
  }
  return failures;
}
function readProductBenchmarkRecords(path) {
  const lines = readFileSync(path, "utf8").split("\n").map((line) => line.trim()).filter(Boolean);
  const records = [];
  for (const [index, line] of lines.entries()) {
    try {
      records.push(validateProductBenchmarkRecord(JSON.parse(line)));
    } catch (err) {
      wrapValidationError(`${path}:${index + 1}`, err);
    }
  }
  return records;
}
function readProductBenchmarkManifest(path) {
  try {
    return validateProductBenchmarkManifest(JSON.parse(readFileSync(path, "utf8")));
  } catch (err) {
    wrapValidationError(path, err);
  }
}
function validateProductBenchmarkRun(input) {
  const manifest = readProductBenchmarkManifest(input.manifestPath);
  const records = readProductBenchmarkRecords(input.recordsPath);
  const manifestProjectBench = `${manifest.projectId}/${manifest.benchmarkId}`;
  const integrityFailures = records.flatMap(productBenchmarkIntegrityFailures);
  const missingArtifacts = input.checkArtifacts === false ? [] : records.flatMap(
    (record) => missingArtifactsForRecord(record, input.artifactRoot ?? dirname(input.recordsPath))
  );
  for (const [index, record] of records.entries()) {
    const recordProjectBench = `${record.projectId}/${record.benchmarkId}`;
    if (recordProjectBench !== manifestProjectBench) {
      fail(
        `records[${index}]`,
        `project/benchmark ${recordProjectBench} does not match manifest ${manifestProjectBench}`
      );
    }
    if (!manifest.arms.some((arm) => arm.id === record.armId))
      fail(`records[${index}].armId`, `unknown arm ${record.armId}`);
    if (!manifest.scenarios.some((scenario) => scenario.id === record.scenarioId)) {
      fail(`records[${index}].scenarioId`, `unknown scenario ${record.scenarioId}`);
    }
  }
  return {
    manifestPath: input.manifestPath,
    recordsPath: input.recordsPath,
    records: records.length,
    projects: sortedUnique(records.map((record) => record.projectId)),
    benchmarks: sortedUnique(records.map((record) => record.benchmarkId)),
    arms: sortedUnique(records.map((record) => record.armId)),
    scenarios: sortedUnique(records.map((record) => record.scenarioId)),
    passed: records.filter((record) => record.outcome.pass).length,
    failed: records.filter((record) => !record.outcome.pass).length,
    inputTokens: sum(records, (record) => record.usage.inputTokens),
    outputTokens: sum(records, (record) => record.usage.outputTokens),
    costUsd: sum(records, (record) => record.usage.costUsd),
    wallMs: sum(records, (record) => record.usage.wallMs),
    integrityFailures,
    missingArtifacts
  };
}
function missingArtifactsForRecord(record, artifactRoot) {
  const missing = [];
  for (const [key, value] of Object.entries(record.artifacts)) {
    const path = resolveArtifactPath(value, artifactRoot);
    if (!existsSync(path)) missing.push(`${record.runId}:${key}:${value}`);
  }
  return missing;
}
function resolveArtifactPath(value, artifactRoot) {
  return value.startsWith("/") ? value : join(artifactRoot, value);
}
function assertUnique(values, path) {
  const seen = /* @__PURE__ */ new Set();
  for (const value of values) {
    if (seen.has(value)) fail(path, `duplicate ${value}`);
    seen.add(value);
  }
}
function sortedUnique(values) {
  return [...new Set(values)].sort();
}
function sum(items, fn) {
  return items.reduce((total, item) => total + fn(item), 0);
}
function findProductBenchmarkArtifacts(runDir) {
  const manifestPath = join(runDir, "product-benchmark-manifest.json");
  const recordsPath = join(runDir, "product-benchmark-records.jsonl");
  if (existsSync(manifestPath) && statSync(manifestPath).isFile() && existsSync(recordsPath) && statSync(recordsPath).isFile()) {
    return { manifestPath, recordsPath };
  }
  return null;
}

export {
  productBenchmarkSplits,
  validateProductBenchmarkManifest,
  validateProductBenchmarkRecord,
  productBenchmarkIntegrityFailures,
  readProductBenchmarkRecords,
  readProductBenchmarkManifest,
  validateProductBenchmarkRun,
  findProductBenchmarkArtifacts
};
//# sourceMappingURL=chunk-63MBSQTX.js.map