import {
  fsCampaignStorage,
  runCampaign
} from "./chunk-N3QPYIVG.js";
import {
  __export
} from "./chunk-PZ5AY32C.js";

// src/benchmarks/index.ts
var benchmarks_exports = {};
__export(benchmarks_exports, {
  BENCHMARK_SPLIT_SEED: () => BENCHMARK_SPLIT_SEED,
  buildStandardRetrievalItems: () => buildStandardRetrievalItems,
  calibrateBenchmarkMetric: () => calibrateBenchmarkMetric,
  createRetrievalIdBenchmarkAdapter: () => createRetrievalIdBenchmarkAdapter,
  deterministicSplit: () => deterministicSplit,
  evaluateStandardRetrieval: () => evaluateStandardRetrieval,
  normalizeRetrievedDocumentIds: () => normalizeRetrievedDocumentIds,
  parseBeirCorpusJsonl: () => parseBeirCorpusJsonl,
  parseBeirQueriesJsonl: () => parseBeirQueriesJsonl,
  parseJsonlRows: () => parseJsonlRows,
  parseQrels: () => parseQrels,
  parseTsvRows: () => parseTsvRows,
  renderBenchmarkReportMarkdown: () => renderBenchmarkReportMarkdown,
  retrievalMetricsAtCutoff: () => retrievalMetricsAtCutoff,
  routing: () => routing_exports,
  runBenchmarkAdapter: () => runBenchmarkAdapter,
  summarizeBenchmarkCampaign: () => summarizeBenchmarkCampaign
});

// src/benchmarks/calibration.ts
async function calibrateBenchmarkMetric(options) {
  const weak = await options.adapter.evaluate(options.item, options.weakArtifact);
  const strong = await options.adapter.evaluate(options.item, options.strongArtifact);
  const weakScore = clamp01(weak.score);
  const strongScore = clamp01(strong.score);
  const maxWeakScore = options.maxWeakScore ?? 0.3;
  const minStrongScore = options.minStrongScore ?? 0.7;
  const minGap = options.minGap ?? 0.4;
  const gap = strongScore - weakScore;
  const reasons = [
    weakScore <= maxWeakScore ? void 0 : `weak score ${weakScore.toFixed(3)} exceeds max ${maxWeakScore.toFixed(3)}`,
    strongScore >= minStrongScore ? void 0 : `strong score ${strongScore.toFixed(3)} below min ${minStrongScore.toFixed(3)}`,
    gap >= minGap ? void 0 : `gap ${gap.toFixed(3)} below min ${minGap.toFixed(3)}`
  ].filter((reason) => Boolean(reason));
  return {
    passed: reasons.length === 0,
    weak,
    strong,
    weakScore,
    strongScore,
    gap,
    reasons
  };
}
function clamp01(value) {
  if (!Number.isFinite(value)) return 0;
  if (value < 0) return 0;
  if (value > 1) return 1;
  return value;
}

// src/benchmarks/routing/index.ts
var routing_exports = {};
__export(routing_exports, {
  ROUTING_DATASET: () => ROUTING_DATASET,
  RoutingAdapter: () => RoutingAdapter,
  assignSplit: () => assignSplit,
  evaluate: () => evaluate,
  extractRouteTokens: () => extractRouteTokens,
  loadDataset: () => loadDataset
});

// src/benchmarks/types.ts
function fnv1a32(input) {
  let h = 2166136261;
  for (let i = 0; i < input.length; i++) {
    h ^= input.charCodeAt(i) & 255;
    h = h + ((h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24)) >>> 0;
  }
  return h >>> 0;
}
var BENCHMARK_SPLIT_SEED = "agent-eval-v1";
function deterministicSplit(itemId, seed = BENCHMARK_SPLIT_SEED) {
  const h = fnv1a32(`${seed}::${itemId}`);
  const pos = h / 4294967296;
  if (pos < 0.6) return "search";
  if (pos < 0.8) return "dev";
  return "holdout";
}

// src/benchmarks/routing/dataset.ts
var ROUTING_DATASET = [
  {
    id: "file_001",
    category: "file",
    prompt: "Save the meeting notes to /tmp/notes-2025-04.md as markdown.",
    route: "fs.write",
    synonyms: ["filesystem.write", "write_file"],
    hardNegatives: ["fs.read", "chat.reply"]
  },
  {
    id: "file_002",
    category: "file",
    prompt: "Read the contents of /etc/hosts and summarize the entries.",
    route: "fs.read",
    synonyms: ["filesystem.read", "read_file"],
    hardNegatives: ["fs.write", "search.web"]
  },
  {
    id: "file_003",
    category: "file",
    prompt: "List every Python file under src/ recursively.",
    route: "fs.list",
    synonyms: ["filesystem.list", "list_files"],
    hardNegatives: ["fs.read", "search.code"]
  },
  {
    id: "file_004",
    category: "file",
    prompt: "Delete the cached build at .turbo/cache.",
    route: "fs.delete",
    synonyms: ["filesystem.delete", "remove_file"],
    hardNegatives: ["fs.write", "fs.list"]
  },
  {
    id: "math_001",
    category: "math",
    prompt: "What is the integral of 3x^2 + 2x from 0 to 5?",
    route: "math.integral",
    synonyms: ["calculator.integral", "math.solve"],
    hardNegatives: ["math.derivative", "chat.reply"]
  },
  {
    id: "math_002",
    category: "math",
    prompt: "Compute the derivative of sin(x) * cos(x).",
    route: "math.derivative",
    synonyms: ["calculator.derivative", "math.solve"],
    hardNegatives: ["math.integral", "math.algebra"]
  },
  {
    id: "math_003",
    category: "math",
    prompt: "Solve 2x + 7 = 19 for x.",
    route: "math.algebra",
    synonyms: ["calculator.algebra", "math.solve"],
    hardNegatives: ["math.derivative", "math.integral"]
  },
  {
    id: "math_004",
    category: "math",
    prompt: "What is the prime factorization of 360?",
    route: "math.numbertheory",
    synonyms: ["calculator.factor", "math.solve"],
    hardNegatives: ["math.algebra", "search.web"]
  },
  {
    id: "search_001",
    category: "search",
    prompt: "Find recent papers on agent prompt optimization with held-out promotion gates.",
    route: "search.web",
    synonyms: ["web.search", "search.papers"],
    hardNegatives: ["search.code", "chat.reply"]
  },
  {
    id: "search_002",
    category: "search",
    prompt: "Search the codebase for every call site of `runProposeReview`.",
    route: "search.code",
    synonyms: ["code.search", "grep"],
    hardNegatives: ["search.web", "fs.read"]
  },
  {
    id: "search_003",
    category: "search",
    prompt: "What is the latest release of the Tangle network on GitHub?",
    route: "search.web",
    synonyms: ["web.search", "github.releases"],
    hardNegatives: ["search.code", "chat.reply"]
  },
  {
    id: "search_004",
    category: "search",
    prompt: "Find all TODO comments in the agent-eval src tree.",
    route: "search.code",
    synonyms: ["code.search", "grep"],
    hardNegatives: ["search.web", "fs.list"]
  },
  {
    id: "chat_001",
    category: "chat",
    prompt: "Hi there, how are you doing today?",
    route: "chat.reply",
    synonyms: ["conversation.reply"],
    hardNegatives: ["search.web", "fs.read"]
  },
  {
    id: "chat_002",
    category: "chat",
    prompt: "Please explain the difference between an LLM and a foundation model.",
    route: "chat.reply",
    synonyms: ["conversation.reply", "qa.answer"],
    hardNegatives: ["search.web", "math.algebra"]
  },
  {
    id: "chat_003",
    category: "chat",
    prompt: "Tell me a short joke about distributed systems.",
    route: "chat.reply",
    synonyms: ["conversation.reply"],
    hardNegatives: ["search.web", "fs.read"]
  },
  {
    id: "chat_004",
    category: "chat",
    prompt: "Acknowledge my last message with a thumbs up.",
    route: "chat.reply",
    synonyms: ["conversation.reply", "react"],
    hardNegatives: ["fs.write", "search.web"]
  }
];

// src/benchmarks/routing/index.ts
var RoutingAdapter = class {
  id = "first-party/routing";
  family = "first-party";
  taskKind = "routing";
  description = "Synthetic fixed-route classification smoke benchmark";
  defaultMetric = "route_exact_match";
  async loadDataset(split) {
    return ROUTING_DATASET.map((item) => ({ id: item.id, payload: item })).filter(
      (it) => assignSplitImpl(it.id) === split
    );
  }
  async evaluate(item, response) {
    const tokens = extractRouteTokens(response);
    const correct = new Set(
      [item.payload.route, ...item.payload.synonyms].map((s) => s.toLowerCase())
    );
    const hardNeg = new Set(item.payload.hardNegatives.map((s) => s.toLowerCase()));
    const firstMatch = tokens.find((t) => correct.has(t.toLowerCase())) ?? null;
    const firstHardNeg = tokens.find((t) => hardNeg.has(t.toLowerCase())) ?? null;
    const score = firstMatch ? 1 : 0;
    return {
      score,
      raw: {
        firstToken: tokens[0] ?? null,
        matchedRoute: firstMatch,
        hitHardNegative: Boolean(firstHardNeg),
        hardNegativeRoute: firstHardNeg,
        category: item.payload.category
      }
    };
  }
  assignSplit(itemId) {
    return assignSplitImpl(itemId);
  }
};
function assignSplitImpl(itemId) {
  return deterministicSplit(`routing::${itemId}`);
}
function extractRouteTokens(response) {
  const matches = response.match(/[a-z][a-z0-9_]*\.[a-z][a-z0-9_]*/gi);
  return matches ?? [];
}
var adapter = new RoutingAdapter();
var loadDataset = adapter.loadDataset.bind(adapter);
var evaluate = adapter.evaluate.bind(adapter);
var assignSplit = adapter.assignSplit.bind(adapter);

// src/benchmarks/runner.ts
import { join } from "path";
async function runBenchmarkAdapter(options) {
  const storage = options.storage ?? fsCampaignStorage();
  const benchmarkId = benchmarkIdFor(options.adapter);
  const scenarios = await loadBenchmarkScenarios(options.adapter, options.splits);
  const judge = benchmarkAdapterJudge(options.adapter);
  const dispatch = async (scenario, context) => {
    return options.respond({ scenario, item: scenario.item, context });
  };
  const campaign = await runCampaign({
    scenarios,
    dispatch,
    dispatchRef: `benchmark:${benchmarkId}`,
    judges: [judge],
    seed: options.seed,
    reps: options.reps,
    resumable: options.resumable,
    costCeiling: options.costCeiling,
    maxConcurrency: options.maxConcurrency,
    dispatchTimeoutMs: options.dispatchTimeoutMs,
    expectUsage: options.expectUsage ?? "off",
    runDir: options.runDir,
    repo: options.repo,
    storage,
    now: options.now
  });
  const report = summarizeBenchmarkCampaign({
    adapter: options.adapter,
    scenarios,
    campaign
  });
  storage.ensureDir(campaign.runDir);
  const reportJsonPath = join(campaign.runDir, "benchmark-report.json");
  const reportMarkdownPath = join(campaign.runDir, "benchmark-report.md");
  storage.write(reportJsonPath, `${JSON.stringify(report, null, 2)}
`);
  storage.write(reportMarkdownPath, renderBenchmarkReportMarkdown(report));
  return { scenarios, campaign, report, reportJsonPath, reportMarkdownPath };
}
async function loadBenchmarkScenarios(adapter2, splits = ["search", "dev", "holdout"]) {
  const benchmarkId = benchmarkIdFor(adapter2);
  const family = adapter2.family ?? "custom";
  const taskKind = adapter2.taskKind ?? "custom";
  const out = [];
  const seen = /* @__PURE__ */ new Set();
  for (const split of splits) {
    const items = await adapter2.loadDataset(split);
    for (const item of items) {
      const splitTag = item.split ?? adapter2.assignSplit(item.id);
      if (splitTag !== split) continue;
      const key = `${benchmarkId}:${item.id}`;
      if (seen.has(key)) continue;
      seen.add(key);
      out.push({
        id: key,
        kind: "benchmark",
        benchmarkId,
        family: item.family ?? family,
        taskKind: item.taskKind ?? taskKind,
        splitTag,
        tags: [.../* @__PURE__ */ new Set([splitTag, ...item.tags ?? []])],
        item
      });
    }
  }
  return out;
}
function benchmarkAdapterJudge(adapter2) {
  const benchmarkId = benchmarkIdFor(adapter2);
  return {
    name: `${benchmarkId}:score`,
    dimensions: [
      { key: "score", description: `Primary ${benchmarkId} benchmark score` },
      { key: "passed", description: "Binary pass projection for aggregate reporting" }
    ],
    appliesTo: (scenario) => {
      return scenario.kind === "benchmark" && scenario.id.startsWith(`${benchmarkId}:`);
    },
    async score({ artifact, scenario }) {
      const evaluation = await adapter2.evaluate(scenario.item, artifact);
      const dimensions = normalizeEvaluationDimensions(evaluation);
      return {
        dimensions,
        composite: clamp012(evaluation.score),
        notes: evaluation.notes ?? ""
      };
    }
  };
}
function summarizeBenchmarkCampaign(input) {
  const scenarioById = new Map(input.scenarios.map((scenario) => [scenario.id, scenario]));
  const rows = input.campaign.cells.map((cell) => {
    const scenario = scenarioById.get(cell.scenarioId);
    const judge = firstJudgeScore(cell.judgeScores);
    const score = judge?.composite ?? 0;
    return {
      cell,
      scenario,
      score,
      passed: (judge?.dimensions.passed ?? (score > 0 ? 1 : 0)) >= 1,
      dimensions: judge?.dimensions ?? {}
    };
  });
  const successful = rows.filter((row) => !row.cell.error);
  const scoreValues = successful.map((row) => row.score);
  return {
    benchmarkId: benchmarkIdFor(input.adapter),
    family: input.adapter.family ?? "custom",
    taskKind: input.adapter.taskKind ?? "custom",
    ...input.adapter.source ? { source: input.adapter.source } : {},
    runDir: input.campaign.runDir,
    manifestHash: input.campaign.manifestHash,
    seed: input.campaign.seed,
    startedAt: input.campaign.startedAt,
    endedAt: input.campaign.endedAt,
    durationMs: input.campaign.durationMs,
    totalItems: input.scenarios.length,
    totalCells: input.campaign.cells.length,
    cellsFailed: input.campaign.aggregates.cellsFailed,
    cellsCached: input.campaign.aggregates.cellsCached,
    totalCostUsd: input.campaign.aggregates.totalCostUsd,
    splits: summarizeSlices(successful, (row) => row.scenario?.splitTag ?? "unknown", [
      "search",
      "dev",
      "holdout"
    ]),
    tags: summarizeSlices(successful, (row) => row.scenario?.tags ?? []),
    dimensions: summarizeDimensions(successful.map((row) => row.dimensions)),
    score: distribution(scoreValues),
    costUsd: distribution(successful.map((row) => row.cell.costUsd)),
    latencyMs: distribution(successful.map((row) => row.cell.durationMs))
  };
}
function renderBenchmarkReportMarkdown(report) {
  const splitRows = Object.entries(report.splits).map(([split, summary]) => {
    return `| ${split} | ${summary.n} | ${fmt(summary.meanScore)} | ${fmt(summary.passRate)} | ${fmt(summary.score.p90)} | ${fmt(summary.costUsd.mean)} | ${fmt(summary.latencyMs.p90)} |`;
  }).join("\n");
  const dimRows = Object.entries(report.dimensions).sort(([a], [b]) => a.localeCompare(b)).map(([key, dist]) => `| ${key} | ${dist.n} | ${fmt(dist.mean)} | ${fmt(dist.p90)} |`).join("\n");
  return [
    `# Benchmark Report: ${report.benchmarkId}`,
    "",
    `- family: ${report.family}`,
    `- task kind: ${report.taskKind}`,
    `- run dir: ${report.runDir}`,
    `- manifest: ${report.manifestHash}`,
    `- cells: ${report.totalCells} total, ${report.cellsFailed} failed, ${report.cellsCached} cached`,
    `- cost: $${fmt(report.totalCostUsd)}`,
    `- score: mean ${fmt(report.score.mean)}, median ${fmt(report.score.median)}, p90 ${fmt(report.score.p90)}, n=${report.score.n}`,
    "",
    "## Splits",
    "",
    "| split | n | mean score | pass rate | score p90 | mean cost | latency p90 ms |",
    "| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
    splitRows || "| none | 0 | 0 | 0 | 0 | 0 | 0 |",
    "",
    "## Dimensions",
    "",
    "| dimension | n | mean | p90 |",
    "| --- | ---: | ---: | ---: |",
    dimRows || "| none | 0 | 0 | 0 |",
    ""
  ].join("\n");
}
function benchmarkIdFor(adapter2) {
  return adapter2.id ?? `${adapter2.family ?? "custom"}/${adapter2.taskKind ?? "custom"}`;
}
function normalizeEvaluationDimensions(evaluation) {
  const dimensions = { score: clamp012(evaluation.score) };
  for (const [key, value] of Object.entries(evaluation.dimensions ?? {})) {
    if (Number.isFinite(value)) dimensions[key] = value;
  }
  dimensions.passed = evaluation.passed ?? evaluation.score > 0 ? 1 : 0;
  return dimensions;
}
function summarizeDimensions(rows) {
  const values = /* @__PURE__ */ new Map();
  for (const row of rows) {
    for (const [key, value] of Object.entries(row)) {
      if (!Number.isFinite(value)) continue;
      const list = values.get(key) ?? [];
      list.push(value);
      values.set(key, list);
    }
  }
  return Object.fromEntries([...values.entries()].map(([key, vals]) => [key, distribution(vals)]));
}
function summarizeSlices(rows, keyOf, knownKeys = []) {
  const grouped = /* @__PURE__ */ new Map();
  for (const key of knownKeys) grouped.set(key, []);
  for (const row of rows) {
    const keys = keyOf(row);
    for (const key of Array.isArray(keys) ? keys : [keys]) {
      const list = grouped.get(key) ?? [];
      list.push(row);
      grouped.set(key, list);
    }
  }
  const out = {};
  for (const [key, list] of grouped) {
    const withShape = list;
    out[key] = {
      n: list.length,
      meanScore: mean(withShape.map((row) => row.score)),
      passRate: mean(withShape.map((row) => row.passed ? 1 : 0)),
      score: distribution(withShape.map((row) => row.score)),
      costUsd: distribution(withShape.map((row) => row.cell.costUsd)),
      latencyMs: distribution(withShape.map((row) => row.cell.durationMs))
    };
  }
  return out;
}
function firstJudgeScore(judgeScores) {
  return Object.values(judgeScores)[0];
}
function distribution(values) {
  const finite = [...values].filter(Number.isFinite).sort((a, b) => a - b);
  if (finite.length === 0) return { n: 0, min: 0, mean: 0, median: 0, p90: 0, max: 0 };
  return {
    n: finite.length,
    min: finite[0],
    mean: mean(finite),
    median: percentile(finite, 0.5),
    p90: percentile(finite, 0.9),
    max: finite[finite.length - 1]
  };
}
function percentile(sortedValues, p) {
  if (sortedValues.length === 0) return 0;
  const index = Math.min(
    sortedValues.length - 1,
    Math.max(0, Math.ceil(p * sortedValues.length) - 1)
  );
  return sortedValues[index];
}
function mean(values) {
  const finite = values.filter(Number.isFinite);
  if (finite.length === 0) return 0;
  return finite.reduce((sum, value) => sum + value, 0) / finite.length;
}
function clamp012(value) {
  if (!Number.isFinite(value)) return 0;
  if (value < 0) return 0;
  if (value > 1) return 1;
  return value;
}
function fmt(value) {
  if (!Number.isFinite(value)) return "0";
  return value.toFixed(value === 0 || Math.abs(value) >= 10 ? 0 : 3);
}

// src/benchmarks/standard-formats.ts
function parseJsonlRows(text) {
  return text.split(/\r?\n/).map((line) => line.trim()).filter(Boolean).map((line, index) => {
    try {
      return JSON.parse(line);
    } catch (error) {
      throw new Error(`invalid JSONL row ${index + 1}: ${error.message}`);
    }
  });
}
function parseTsvRows(text) {
  return text.split(/\r?\n/).map((line) => line.trim()).filter(Boolean).filter((line) => !line.startsWith("#")).map((line) => line.split(/\t|\s+/));
}
function parseQrels(text) {
  return parseTsvRows(text).flatMap((parts, index) => {
    if (parts.length < 3) return [];
    const [queryId, maybeZeroOrDocId, maybeDocIdOrScore, maybeScore] = parts;
    if (!queryId || !maybeZeroOrDocId || !maybeDocIdOrScore) return [];
    if (queryId.toLowerCase() === "query-id" || queryId.toLowerCase() === "qid") return [];
    const documentId = maybeScore === void 0 ? maybeZeroOrDocId : maybeDocIdOrScore;
    const scoreText = maybeScore === void 0 ? maybeDocIdOrScore : maybeScore;
    const score = Number(scoreText);
    if (!documentId || !Number.isFinite(score)) {
      throw new Error(`invalid qrels row ${index + 1}: expected query id, doc id, score`);
    }
    return [{ queryId, documentId, score }];
  });
}
function parseBeirCorpusJsonl(text) {
  return parseJsonlRows(text).map((row, index) => {
    const id = stringField(row, "_id") ?? stringField(row, "id");
    const body = stringField(row, "text") ?? stringField(row, "contents");
    if (!id || body === void 0) {
      throw new Error(`invalid BEIR corpus row ${index + 1}: expected _id/id and text/contents`);
    }
    return {
      id,
      title: stringField(row, "title"),
      text: body,
      metadata: stripKnown(row, ["_id", "id", "title", "text", "contents"])
    };
  });
}
function parseBeirQueriesJsonl(text) {
  return parseJsonlRows(text).map((row, index) => {
    const id = stringField(row, "_id") ?? stringField(row, "id") ?? stringField(row, "query_id");
    const query = stringField(row, "text") ?? stringField(row, "query");
    if (!id || !query) {
      throw new Error(`invalid BEIR query row ${index + 1}: expected _id/id and text/query`);
    }
    return {
      id,
      text: query,
      metadata: stripKnown(row, ["_id", "id", "query_id", "text", "query"])
    };
  });
}
function buildStandardRetrievalItems(options) {
  const qrelsByQuery = /* @__PURE__ */ new Map();
  for (const qrel of options.qrels) {
    if (qrel.score <= 0) continue;
    const list = qrelsByQuery.get(qrel.queryId) ?? [];
    list.push(qrel);
    qrelsByQuery.set(qrel.queryId, list);
  }
  const corpus = options.includeCorpusInPayload && options.corpus ? Object.fromEntries(options.corpus.map((document) => [document.id, document])) : void 0;
  return options.queries.flatMap((query) => {
    const qrels = qrelsByQuery.get(query.id) ?? [];
    if (qrels.length === 0) return [];
    const split = options.splitOf?.(query.id) ?? deterministicSplit(`${options.benchmarkId}:${query.id}`);
    return [
      {
        id: query.id,
        split,
        family: options.family,
        taskKind: "retrieval",
        tags: [.../* @__PURE__ */ new Set([...options.tags ?? [], split])],
        ...options.source ? { source: options.source } : {},
        ...query.metadata ? { metadata: query.metadata } : {},
        payload: {
          queryId: query.id,
          query: query.text,
          expectedDocumentIds: qrels.map((qrel) => qrel.documentId),
          expectedScores: Object.fromEntries(qrels.map((qrel) => [qrel.documentId, qrel.score])),
          ...corpus ? { corpus } : {},
          ...query.metadata ? { metadata: query.metadata } : {}
        }
      }
    ];
  });
}
function createRetrievalIdBenchmarkAdapter(options) {
  const items = buildStandardRetrievalItems(options);
  return {
    id: options.benchmarkId,
    family: options.family,
    taskKind: "retrieval",
    source: options.source,
    defaultMetric: options.primaryMetric ?? "ndcg@10",
    async loadDataset(split) {
      return items.filter((item) => item.split === split);
    },
    async evaluate(item, artifact) {
      return evaluateStandardRetrieval(item.payload, artifact, options);
    },
    assignSplit(itemId) {
      return options.splitOf?.(itemId) ?? deterministicSplit(`${options.benchmarkId}:${itemId}`);
    }
  };
}
function evaluateStandardRetrieval(payload, artifact, options = {}) {
  const rankedDocumentIds = normalizeRetrievedDocumentIds(
    artifact,
    options.responseIdPattern ?? /[A-Za-z0-9_.:/-]+/g
  );
  const cutoffs = normalizeCutoffs(options.cutoffs ?? [1, 3, 5, 10]);
  const dimensions = {
    expected_count: payload.expectedDocumentIds.length,
    returned_count: rankedDocumentIds.length
  };
  for (const cutoff of cutoffs) {
    Object.assign(
      dimensions,
      retrievalMetricsAtCutoff({
        rankedDocumentIds,
        expectedScores: payload.expectedScores,
        cutoff
      })
    );
  }
  const primaryMetric = options.primaryMetric ?? "ndcg@10";
  const passMetric = options.passMetric ?? "hit@10";
  const score = dimensions[primaryMetric] ?? dimensions[`ndcg@${cutoffs[cutoffs.length - 1]}`] ?? 0;
  const passScore = dimensions[passMetric] ?? dimensions[`hit@${cutoffs[cutoffs.length - 1]}`] ?? score;
  return {
    score,
    passed: passScore >= (options.passThreshold ?? 1),
    dimensions,
    raw: {
      rankedDocumentIds,
      expectedDocumentIds: payload.expectedDocumentIds,
      expectedScores: payload.expectedScores
    }
  };
}
function normalizeRetrievedDocumentIds(artifact, responseIdPattern = /[A-Za-z0-9_.:/-]+/g) {
  if (typeof artifact === "string") {
    return uniqueOrdered((artifact.match(responseIdPattern) ?? []).map((value) => value.trim()));
  }
  if (Array.isArray(artifact)) {
    const entries = artifact;
    return uniqueOrdered(
      entries.flatMap((entry) => {
        if (typeof entry === "string") return [entry];
        return [entry.documentId ?? entry.docId ?? entry.id ?? ""];
      })
    );
  }
  const objectArtifact = artifact;
  return uniqueOrdered(
    [
      ...objectArtifact.documentIds ?? [],
      ...objectArtifact.ids ?? [],
      ...(objectArtifact.results ?? []).map(
        (entry) => entry.documentId ?? entry.docId ?? entry.id ?? ""
      )
    ].map((value) => value.trim())
  );
}
function retrievalMetricsAtCutoff(input) {
  const top = input.rankedDocumentIds.slice(0, input.cutoff);
  const relevantIds = Object.keys(input.expectedScores).filter(
    (id) => input.expectedScores[id] > 0
  );
  const relevant = new Set(relevantIds);
  const hits = top.filter((id) => relevant.has(id)).length;
  const firstRelevantRank = top.findIndex((id) => relevant.has(id));
  const prefix = `@${input.cutoff}`;
  return {
    [`hit${prefix}`]: hits > 0 ? 1 : 0,
    [`recall${prefix}`]: relevant.size === 0 ? 1 : hits / relevant.size,
    [`precision${prefix}`]: hits / input.cutoff,
    [`mrr${prefix}`]: firstRelevantRank === -1 ? 0 : 1 / (firstRelevantRank + 1),
    [`ndcg${prefix}`]: ndcgAt(input.rankedDocumentIds, input.expectedScores, input.cutoff)
  };
}
function stringField(row, key) {
  const value = row[key];
  return typeof value === "string" ? value : void 0;
}
function stripKnown(row, keys) {
  const known = new Set(keys);
  return Object.fromEntries(Object.entries(row).filter(([key]) => !known.has(key)));
}
function normalizeCutoffs(cutoffs) {
  return [
    ...new Set(cutoffs.map((cutoff) => Math.trunc(cutoff)).filter((cutoff) => cutoff > 0))
  ].sort((a, b) => a - b);
}
function uniqueOrdered(values) {
  const seen = /* @__PURE__ */ new Set();
  const out = [];
  for (const value of values) {
    const trimmed = value.trim();
    if (!trimmed || seen.has(trimmed)) continue;
    seen.add(trimmed);
    out.push(trimmed);
  }
  return out;
}
function ndcgAt(rankedDocumentIds, expectedScores, cutoff) {
  const dcg = rankedDocumentIds.slice(0, cutoff).reduce(
    (sum, documentId, index) => sum + discountedGain(expectedScores[documentId] ?? 0, index),
    0
  );
  const ideal = Object.values(expectedScores).filter((score) => score > 0).sort((a, b) => b - a).slice(0, cutoff).reduce((sum, score, index) => sum + discountedGain(score, index), 0);
  return ideal === 0 ? 1 : dcg / ideal;
}
function discountedGain(relevance, zeroBasedRank) {
  if (relevance <= 0) return 0;
  return (2 ** relevance - 1) / Math.log2(zeroBasedRank + 2);
}

export {
  calibrateBenchmarkMetric,
  BENCHMARK_SPLIT_SEED,
  deterministicSplit,
  routing_exports,
  runBenchmarkAdapter,
  summarizeBenchmarkCampaign,
  renderBenchmarkReportMarkdown,
  parseJsonlRows,
  parseTsvRows,
  parseQrels,
  parseBeirCorpusJsonl,
  parseBeirQueriesJsonl,
  buildStandardRetrievalItems,
  createRetrievalIdBenchmarkAdapter,
  evaluateStandardRetrieval,
  normalizeRetrievedDocumentIds,
  retrievalMetricsAtCutoff,
  benchmarks_exports
};
//# sourceMappingURL=chunk-OZFTLMGS.js.map