import {
  rubricPredictiveValidity
} from "../chunk-YRZ4M5GS.js";
import {
  aggregateLlm,
  llmSpans
} from "../chunk-47X6LRCE.js";
import "../chunk-5BKGXME7.js";
import "../chunk-PZ5AY32C.js";

// src/meta-eval/calibration.ts
async function calibrationCurve(traceStore, outcomeStore, evalMetric, outcomeMetric, options = {}) {
  const runs = await traceStore.listRuns();
  const outcomes = await outcomeStore.list();
  const byRun = /* @__PURE__ */ new Map();
  for (const o of outcomes) {
    const arr = byRun.get(o.runId) ?? [];
    arr.push(o);
    byRun.set(o.runId, arr);
  }
  const extract = evalMetric.extract ?? defaultExtract(evalMetric.id);
  const pairs = [];
  for (const run of runs) {
    const os = byRun.get(run.runId);
    if (!os?.length) continue;
    const x = await extract(run, traceStore);
    if (x === null || !Number.isFinite(x)) continue;
    const latest = [...os].sort((a, b) => b.capturedAt - a.capturedAt)[0];
    const y = latest.metrics[outcomeMetric];
    if (typeof y !== "number" || !Number.isFinite(y)) continue;
    pairs.push({ x, y });
  }
  if (pairs.length < 2) return null;
  const numBins = options.bins ?? 10;
  const binning = options.binning ?? "equal-width";
  const xs = pairs.map((p) => p.x);
  const lo = options.range?.lo ?? Math.min(...xs);
  const hi = options.range?.hi ?? Math.max(...xs);
  const bins = [];
  if (binning === "equal-frequency") {
    const sorted = [...pairs].sort((a, b) => a.x - b.x);
    const perBin = Math.max(1, Math.floor(sorted.length / numBins));
    for (let i = 0; i < sorted.length; i += perBin) {
      const chunk = sorted.slice(i, i + perBin);
      if (chunk.length === 0) continue;
      bins.push(toBin(chunk));
    }
  } else {
    const width = (hi - lo) / numBins;
    if (width === 0) return null;
    for (let i = 0; i < numBins; i++) {
      const binLo = lo + i * width;
      const binHi = i === numBins - 1 ? hi + 1e-9 : lo + (i + 1) * width;
      const chunk = pairs.filter((p) => p.x >= binLo && p.x < binHi);
      if (chunk.length === 0) continue;
      bins.push(toBin(chunk, binLo, binHi));
    }
  }
  const total = bins.reduce((a, b) => a + b.n, 0);
  const ece = bins.reduce((a, b) => a + b.n / total * b.gap, 0);
  const maxGap = bins.reduce((a, b) => Math.max(a, b.gap), 0);
  return { evalMetric: evalMetric.id, outcomeMetric, n: pairs.length, bins, ece, maxGap };
}
function toBin(chunk, lower, upper) {
  const xs = chunk.map((c) => c.x);
  const ys = chunk.map((c) => c.y);
  const evalMean = mean(xs);
  const outcomeMean = mean(ys);
  return {
    lower: lower ?? Math.min(...xs),
    upper: upper ?? Math.max(...xs),
    n: chunk.length,
    evalMean,
    outcomeMean,
    gap: Math.abs(outcomeMean - evalMean)
  };
}
function mean(xs) {
  return xs.reduce((a, b) => a + b, 0) / xs.length;
}
function defaultExtract(metric) {
  return async (run) => run.outcome?.score ?? (metric === "pass" ? run.outcome?.pass === true ? 1 : 0 : null);
}

// src/meta-eval/correlation-study.ts
async function correlationStudy(traceStore, outcomeStore, evalMetrics, outcomeMetricNames, options = {}) {
  const runs = await traceStore.listRuns();
  const outcomes = await outcomeStore.list(options.outcomeFilter);
  const outcomesByRun = /* @__PURE__ */ new Map();
  for (const o of outcomes) {
    const arr = outcomesByRun.get(o.runId) ?? [];
    arr.push(o);
    outcomesByRun.set(o.runId, arr);
  }
  const reduction = options.reduction ?? "latest";
  const maxLag = options.maxCaptureLagMs ?? Infinity;
  const pairs = [];
  for (const em of evalMetrics) {
    for (const om of outcomeMetricNames) {
      pairs.push({ evalMetric: em.id, outcomeMetric: om, xs: [], ys: [] });
    }
  }
  let joined = 0;
  let skipped = 0;
  for (const run of runs) {
    const os = outcomesByRun.get(run.runId);
    if (!os || os.length === 0) {
      skipped++;
      continue;
    }
    const eligible = os.filter((o) => o.capturedAt - run.startedAt <= maxLag);
    if (eligible.length === 0) {
      skipped++;
      continue;
    }
    for (const em of evalMetrics) {
      const extract = em.extract ?? defaultExtract2(em.id);
      const x = await extract(run, traceStore);
      if (x === null || !Number.isFinite(x)) continue;
      for (const om of outcomeMetricNames) {
        const values = eligible.map((o) => o.metrics[om]).filter((v) => typeof v === "number" && Number.isFinite(v));
        if (values.length === 0) continue;
        const y = reduce(values, reduction, eligible);
        if (y === null) continue;
        const pair = pairs.find((p) => p.evalMetric === em.id && p.outcomeMetric === om);
        pair.xs.push(x);
        pair.ys.push(y);
      }
    }
    joined++;
  }
  const results = pairs.filter((p) => p.xs.length >= 3).map((p) => {
    const pearson = pearsonR(p.xs, p.ys);
    const spearman = pearsonR(ranks(p.xs), ranks(p.ys));
    const pearsonCi95 = bootstrapPearsonCi(p.xs, p.ys, options.bootstrapIterations ?? 500);
    const verdict = Math.abs(pearson) >= 0.7 ? "strong" : Math.abs(pearson) >= 0.4 ? "moderate" : "weak";
    return {
      evalMetric: p.evalMetric,
      outcomeMetric: p.outcomeMetric,
      n: p.xs.length,
      pearson,
      spearman,
      pearsonCi95,
      verdict
    };
  });
  return { pairs: results, joinedSamples: joined, skippedRuns: skipped };
}
function reduce(values, kind, outcomes) {
  if (values.length === 0) return null;
  if (kind === "mean") return values.reduce((a, b) => a + b, 0) / values.length;
  if (kind === "max") return Math.max(...values);
  const latest = [...outcomes].sort((a, b) => b.capturedAt - a.capturedAt)[0];
  if (!latest) return null;
  const latestKey = Object.keys(latest.metrics)[0];
  const v = latestKey !== void 0 ? latest.metrics[latestKey] : void 0;
  const paired = outcomes.map((o) => {
    const k = Object.keys(o.metrics)[0];
    return {
      at: o.capturedAt,
      v: k !== void 0 ? values.find((x) => o.metrics[k] === x) : void 0
    };
  }).filter((p) => p.v !== void 0);
  if (paired.length === 0) return v ?? null;
  return paired.sort((a, b) => b.at - a.at)[0]?.v ?? null;
}
function pearsonR(a, b) {
  if (a.length !== b.length || a.length < 2) return NaN;
  const mA = a.reduce((s, v) => s + v, 0) / a.length;
  const mB = b.reduce((s, v) => s + v, 0) / b.length;
  let num = 0, dA = 0, dB = 0;
  for (let i = 0; i < a.length; i++) {
    const da = a[i] - mA, db = b[i] - mB;
    num += da * db;
    dA += da * da;
    dB += db * db;
  }
  if (dA === 0 || dB === 0) return dA === 0 && dB === 0 ? 1 : 0;
  return num / Math.sqrt(dA * dB);
}
function ranks(xs) {
  const indexed = xs.map((v, i) => ({ v, i })).sort((x, y) => x.v - y.v);
  const r = new Array(xs.length);
  for (let i = 0; i < indexed.length; i++) {
    let j = i;
    while (j + 1 < indexed.length && indexed[j + 1].v === indexed[i].v) j++;
    const avg = (i + j + 2) / 2;
    for (let k = i; k <= j; k++) r[indexed[k].i] = avg;
    i = j;
  }
  return r;
}
function bootstrapPearsonCi(xs, ys, iterations) {
  const n = xs.length;
  if (n < 3) return { lower: NaN, upper: NaN };
  const rs = [];
  for (let b = 0; b < iterations; b++) {
    const rx = new Array(n);
    const ry = new Array(n);
    for (let i = 0; i < n; i++) {
      const idx = Math.floor(Math.random() * n);
      rx[i] = xs[idx];
      ry[i] = ys[idx];
    }
    const r = pearsonR(rx, ry);
    if (Number.isFinite(r)) rs.push(r);
  }
  rs.sort((a, b) => a - b);
  if (rs.length === 0) return { lower: NaN, upper: NaN };
  return {
    lower: rs[Math.floor(0.025 * rs.length)],
    upper: rs[Math.min(rs.length - 1, Math.floor(0.975 * rs.length))]
  };
}
function defaultExtract2(metric) {
  return async (run, store) => {
    switch (metric) {
      case "score":
      case "overallScore":
        return run.outcome?.score ?? null;
      case "pass":
        return run.outcome?.pass === true ? 1 : 0;
      case "durationMs":
        return run.endedAt && run.startedAt ? run.endedAt - run.startedAt : null;
      case "costUsd": {
        const llm = await llmSpans(store, run.runId);
        return aggregateLlm(llm).costUsd;
      }
      case "inputTokens": {
        const llm = await llmSpans(store, run.runId);
        return aggregateLlm(llm).inputTokens;
      }
      default:
        return null;
    }
  };
}

// src/meta-eval/outcome-store.ts
var InMemoryOutcomeStore = class {
  items = [];
  async append(outcome) {
    this.items.push({ ...outcome });
  }
  async forRun(runId) {
    return this.items.filter((o) => o.runId === runId).map((o) => ({ ...o }));
  }
  async list(filter = {}) {
    return this.items.filter((o) => matches(o, filter)).map((o) => ({ ...o }));
  }
};
var FileSystemOutcomeStore = class {
  dir;
  maxBytes;
  memo;
  loaded = false;
  constructor(options) {
    this.dir = options.dir;
    this.maxBytes = options.maxBytes ?? 32 * 1024 * 1024;
  }
  async ensureDir() {
    const fs = await import("fs/promises");
    await fs.mkdir(this.dir, { recursive: true });
  }
  async append(outcome) {
    await this.ensureDir();
    const fs = await import("fs/promises");
    const path = await import("path");
    const active = path.join(this.dir, "outcomes.ndjson");
    try {
      const stat = await fs.stat(active);
      if (stat.size >= this.maxBytes) {
        await fs.rename(active, path.join(this.dir, `outcomes.${Date.now()}.ndjson`));
      }
    } catch {
    }
    await fs.appendFile(active, `${JSON.stringify(outcome)}
`, "utf8");
    if (this.memo) await this.memo.append(outcome);
  }
  async load() {
    if (this.loaded && this.memo) return this.memo;
    const fs = await import("fs/promises");
    const path = await import("path");
    const memo = new InMemoryOutcomeStore();
    try {
      const entries = await fs.readdir(this.dir);
      for (const file of entries) {
        if (!file.endsWith(".ndjson")) continue;
        const content = await fs.readFile(path.join(this.dir, file), "utf8");
        for (const line of content.split("\n")) {
          if (!line.trim()) continue;
          await memo.append(JSON.parse(line));
        }
      }
    } catch {
    }
    this.memo = memo;
    this.loaded = true;
    return memo;
  }
  async forRun(runId) {
    return (await this.load()).forRun(runId);
  }
  async list(filter) {
    return (await this.load()).list(filter);
  }
};
function matches(o, f) {
  if (f.runIds && !f.runIds.includes(o.runId)) return false;
  if (f.since !== void 0 && o.capturedAt < f.since) return false;
  if (f.until !== void 0 && o.capturedAt > f.until) return false;
  if (f.source && o.source !== f.source) return false;
  if (f.label && o.labels?.[f.label.key] !== f.label.value) return false;
  return true;
}
export {
  FileSystemOutcomeStore,
  InMemoryOutcomeStore,
  calibrationCurve,
  correlationStudy,
  rubricPredictiveValidity
};
//# sourceMappingURL=index.js.map