import {
  calibrationCurve,
  calibrationFromPairs
} from "../chunk-NPCTHQIO.js";
import {
  analyzeSeries
} from "../chunk-BOD4O7OF.js";
import {
  FileSystemOutcomeStore,
  InMemoryOutcomeStore
} from "../chunk-3RF76KTD.js";
import {
  rubricPredictiveValidity
} from "../chunk-X4UCIOTZ.js";
import {
  pearsonR,
  spearmanR
} from "../chunk-PJQFMIOX.js";
import {
  aggregateLlm,
  llmSpans
} from "../chunk-ZET2UAYW.js";
import "../chunk-MA6HLL3S.js";
import {
  ValidationError
} from "../chunk-ONWEPEDO.js";
import "../chunk-PZ5AY32C.js";

// src/meta-eval/correlation-study.ts
async function correlationStudy(traceStore, outcomeStore, evalMetrics, outcomeMetricNames, options = {}) {
  const runs = await traceStore.listRuns();
  const outcomes = await outcomeStore.list(options.outcomeFilter);
  const outcomesByRun = /* @__PURE__ */ new Map();
  for (const o of outcomes) {
    const arr = outcomesByRun.get(o.runId) ?? [];
    arr.push(o);
    outcomesByRun.set(o.runId, arr);
  }
  const reduction = options.reduction ?? "latest";
  const maxLag = options.maxCaptureLagMs ?? Infinity;
  const pairs = [];
  for (const em of evalMetrics) {
    for (const om of outcomeMetricNames) {
      pairs.push({ evalMetric: em.id, outcomeMetric: om, xs: [], ys: [] });
    }
  }
  let joined = 0;
  let skipped = 0;
  for (const run of runs) {
    const os = outcomesByRun.get(run.runId);
    if (!os || os.length === 0) {
      skipped++;
      continue;
    }
    const eligible = os.filter((o) => o.capturedAt - run.startedAt <= maxLag);
    if (eligible.length === 0) {
      skipped++;
      continue;
    }
    for (const em of evalMetrics) {
      const extract = em.extract ?? defaultExtract(em.id);
      const x = await extract(run, traceStore);
      if (x === null || !Number.isFinite(x)) continue;
      for (const om of outcomeMetricNames) {
        const values = eligible.map((o) => o.metrics[om]).filter((v) => typeof v === "number" && Number.isFinite(v));
        if (values.length === 0) continue;
        const y = reduce(values, reduction, eligible);
        if (y === null) continue;
        const pair = pairs.find((p) => p.evalMetric === em.id && p.outcomeMetric === om);
        pair.xs.push(x);
        pair.ys.push(y);
      }
    }
    joined++;
  }
  const results = pairs.filter((p) => p.xs.length >= 3).map((p) => {
    const pearson = pearsonR(p.xs, p.ys);
    const spearman = spearmanR(p.xs, p.ys);
    const pearsonCi95 = bootstrapPearsonCi(p.xs, p.ys, options.bootstrapIterations ?? 500);
    const verdict = Math.abs(pearson) >= 0.7 ? "strong" : Math.abs(pearson) >= 0.4 ? "moderate" : "weak";
    return {
      evalMetric: p.evalMetric,
      outcomeMetric: p.outcomeMetric,
      n: p.xs.length,
      pearson,
      spearman,
      pearsonCi95,
      verdict
    };
  });
  return { pairs: results, joinedSamples: joined, skippedRuns: skipped };
}
function reduce(values, kind, outcomes) {
  if (values.length === 0) return null;
  if (kind === "mean") return values.reduce((a, b) => a + b, 0) / values.length;
  if (kind === "max") return Math.max(...values);
  const latest = [...outcomes].sort((a, b) => b.capturedAt - a.capturedAt)[0];
  if (!latest) return null;
  const latestKey = Object.keys(latest.metrics)[0];
  const v = latestKey !== void 0 ? latest.metrics[latestKey] : void 0;
  const paired = outcomes.map((o) => {
    const k = Object.keys(o.metrics)[0];
    return {
      at: o.capturedAt,
      v: k !== void 0 ? values.find((x) => o.metrics[k] === x) : void 0
    };
  }).filter((p) => p.v !== void 0);
  if (paired.length === 0) return v ?? null;
  return paired.sort((a, b) => b.at - a.at)[0]?.v ?? null;
}
function bootstrapPearsonCi(xs, ys, iterations) {
  const n = xs.length;
  if (n < 3) return { lower: NaN, upper: NaN };
  const rs = [];
  for (let b = 0; b < iterations; b++) {
    const rx = new Array(n);
    const ry = new Array(n);
    for (let i = 0; i < n; i++) {
      const idx = Math.floor(Math.random() * n);
      rx[i] = xs[idx];
      ry[i] = ys[idx];
    }
    const r = pearsonR(rx, ry);
    if (Number.isFinite(r)) rs.push(r);
  }
  rs.sort((a, b) => a - b);
  if (rs.length === 0) return { lower: NaN, upper: NaN };
  return {
    lower: rs[Math.floor(0.025 * rs.length)],
    upper: rs[Math.min(rs.length - 1, Math.floor(0.975 * rs.length))]
  };
}
function defaultExtract(metric) {
  return async (run, store) => {
    switch (metric) {
      case "score":
      case "overallScore":
        return run.outcome?.score ?? null;
      case "pass":
        return run.outcome?.pass === true ? 1 : 0;
      case "durationMs":
        return run.endedAt && run.startedAt ? run.endedAt - run.startedAt : null;
      case "costUsd": {
        const llm = await llmSpans(store, run.runId);
        return aggregateLlm(llm).costUsd;
      }
      case "inputTokens": {
        const llm = await llmSpans(store, run.runId);
        return aggregateLlm(llm).inputTokens;
      }
      default:
        return null;
    }
  };
}

// src/meta-eval/sentinel.ts
var SENTINEL_METRIC_NAMES = ["irr", "calibrationKappa", "sentinelPassRate"];
function parseIso(value, label) {
  const ms = typeof value === "string" ? Date.parse(value) : NaN;
  if (!Number.isFinite(ms)) {
    throw new ValidationError(
      `judge-sentinel: ${label} is not a parseable ISO timestamp: ${JSON.stringify(value)}`
    );
  }
  return ms;
}
function validateSentinelSnapshot(snapshot, source = "snapshot") {
  parseIso(snapshot.at, `${source}.at`);
  if (typeof snapshot.judgeId !== "string" || snapshot.judgeId.length === 0) {
    throw new ValidationError(`judge-sentinel: ${source}.judgeId must be a non-empty string`);
  }
  if (typeof snapshot.judgeModel !== "string" || snapshot.judgeModel.length === 0) {
    throw new ValidationError(`judge-sentinel: ${source}.judgeModel must be a non-empty string`);
  }
  if (snapshot.metrics === null || typeof snapshot.metrics !== "object") {
    throw new ValidationError(`judge-sentinel: ${source}.metrics must be an object (may be empty)`);
  }
  for (const [key, value] of Object.entries(snapshot.metrics)) {
    if (!SENTINEL_METRIC_NAMES.includes(key)) {
      throw new ValidationError(
        `judge-sentinel: ${source}.metrics carries unknown key "${key}" \u2014 known: ${SENTINEL_METRIC_NAMES.join(", ")}`
      );
    }
    if (value !== void 0 && !Number.isFinite(value)) {
      throw new ValidationError(
        `judge-sentinel: ${source}.metrics.${key} is not a finite number: ${String(value)}`
      );
    }
  }
}
function snapshotFromCalibration(report, meta) {
  const kappa = "weightedKappaContinuous" in report ? report.weightedKappaContinuous : report.kappa;
  if (!Number.isFinite(kappa)) {
    throw new ValidationError(
      `snapshotFromCalibration: \u03BA is not finite (n=${report.n}) \u2014 calibrate against \u22652 joined golden items before snapshotting`
    );
  }
  const snapshot = { ...meta, metrics: { calibrationKappa: kappa } };
  validateSentinelSnapshot(snapshot);
  return snapshot;
}
function snapshotFromAgreement(report, meta) {
  const irr = "overallIcc" in report ? report.overallIcc : report.icc;
  if (!Number.isFinite(irr)) {
    throw new ValidationError(
      "snapshotFromAgreement: ICC is not finite \u2014 agreement needs \u22652 raters and \u22652 complete items"
    );
  }
  const snapshot = { ...meta, metrics: { irr } };
  validateSentinelSnapshot(snapshot);
  return snapshot;
}
function snapshotFromSentinelSet(scores, golden, meta, options = {}) {
  const tolerance = options.tolerance ?? 0.1;
  if (!Number.isFinite(tolerance) || tolerance < 0) {
    throw new ValidationError(
      `snapshotFromSentinelSet: tolerance must be a finite number \u2265 0, got ${tolerance}`
    );
  }
  const goldenById = /* @__PURE__ */ new Map();
  for (const item of golden) {
    if (!Number.isFinite(item.humanScore)) {
      throw new ValidationError(
        `snapshotFromSentinelSet: golden item "${item.itemId}" has a non-finite humanScore`
      );
    }
    if (goldenById.has(item.itemId)) {
      throw new ValidationError(`snapshotFromSentinelSet: duplicate golden itemId "${item.itemId}"`);
    }
    goldenById.set(item.itemId, item.humanScore);
  }
  let joined = 0;
  let passed = 0;
  for (const s of scores) {
    const human = goldenById.get(s.itemId);
    if (human === void 0) continue;
    if (!Number.isFinite(s.score)) {
      throw new ValidationError(
        `snapshotFromSentinelSet: judge score for item "${s.itemId}" is not finite`
      );
    }
    joined++;
    if (Math.abs(s.score - human) <= tolerance) passed++;
  }
  if (joined === 0) {
    throw new ValidationError(
      "snapshotFromSentinelSet: no judge scores joined the golden set \u2014 itemIds mismatch or empty sentinel run"
    );
  }
  const snapshot = { ...meta, metrics: { sentinelPassRate: passed / joined } };
  validateSentinelSnapshot(snapshot);
  return snapshot;
}
function inMemorySentinelStore(initial = []) {
  for (const s of initial) validateSentinelSnapshot(s);
  const items = initial.map((s) => ({ ...s, metrics: { ...s.metrics } }));
  return {
    async append(snapshot) {
      validateSentinelSnapshot(snapshot);
      items.push({ ...snapshot, metrics: { ...snapshot.metrics } });
    },
    async history(judgeId) {
      const all = items.map((s) => ({ ...s, metrics: { ...s.metrics } }));
      return judgeId === void 0 ? all : all.filter((s) => s.judgeId === judgeId);
    }
  };
}
function fileSentinelStore(path) {
  return {
    async append(snapshot) {
      validateSentinelSnapshot(snapshot);
      const fs = await import("fs/promises");
      const pathMod = await import("path");
      await fs.mkdir(pathMod.dirname(path), { recursive: true });
      await fs.appendFile(path, `${JSON.stringify(snapshot)}
`, "utf8");
    },
    async history(judgeId) {
      const fs = await import("fs/promises");
      let raw;
      try {
        raw = await fs.readFile(path, "utf8");
      } catch (err) {
        if (err.code === "ENOENT") return [];
        throw err;
      }
      const lines = raw.split("\n");
      const snapshots = [];
      for (let i = 0; i < lines.length; i++) {
        const line = lines[i];
        if (line.trim().length === 0) continue;
        let parsed;
        try {
          parsed = JSON.parse(line);
        } catch (err) {
          throw new ValidationError(
            `judge-sentinel: store at ${path} has a corrupt JSONL line ${i + 1}: ${err.message}`
          );
        }
        const snapshot = parsed;
        validateSentinelSnapshot(snapshot, `${path}:${i + 1}`);
        snapshots.push(snapshot);
      }
      return judgeId === void 0 ? snapshots : snapshots.filter((s) => s.judgeId === judgeId);
    }
  };
}
var MS_PER_DAY = 864e5;
function judgeSentinelReport(history, opts) {
  const asOfMs = parseIso(opts.asOf, "asOf");
  const minIrr = opts.thresholds?.minIrr ?? 0.6;
  const maxKappaDrop = opts.thresholds?.maxKappaDrop ?? 0.15;
  const maxSentinelDrop = opts.thresholds?.maxSentinelDrop ?? 0.1;
  const staleAfterDays = opts.thresholds?.staleAfterDays ?? 30;
  const byJudge = /* @__PURE__ */ new Map();
  for (const snapshot of history) {
    validateSentinelSnapshot(snapshot);
    const atMs = parseIso(snapshot.at, `snapshot.at (judge "${snapshot.judgeId}")`);
    const arr = byJudge.get(snapshot.judgeId) ?? [];
    arr.push({ ...snapshot, atMs });
    byJudge.set(snapshot.judgeId, arr);
  }
  const perJudge = [];
  const alarms = [];
  const insufficientHistory = [];
  for (const judgeId of [...byJudge.keys()].sort()) {
    const snaps = byJudge.get(judgeId).sort((a, b) => a.atMs - b.atMs);
    const newest = snaps[snaps.length - 1];
    const ageDays = (asOfMs - newest.atMs) / MS_PER_DAY;
    if (ageDays > staleAfterDays) {
      alarms.push(
        `judge "${judgeId}": stale \u2014 newest snapshot ${newest.at} is ${ageDays.toFixed(1)}d old vs asOf (limit ${staleAfterDays}d)`
      );
    }
    let changeIdx = -1;
    for (let i = 1; i < snaps.length; i++) {
      if (snaps[i].judgeModel !== snaps[i - 1].judgeModel) changeIdx = i;
    }
    if (changeIdx >= 0) {
      const recalibrated = snaps.slice(changeIdx).some(
        (s) => Number.isFinite(s.metrics.calibrationKappa) || Number.isFinite(s.metrics.sentinelPassRate)
      );
      if (!recalibrated) {
        alarms.push(
          `judge "${judgeId}": model changed "${snaps[changeIdx - 1].judgeModel}" \u2192 "${snaps[changeIdx].judgeModel}" at ${snaps[changeIdx].at} with no post-change calibration snapshot \u2014 silent-upgrade trap; rerun calibrateJudge or the sentinel set against the new model`
        );
      }
    }
    for (const metric of SENTINEL_METRIC_NAMES) {
      const values = snaps.filter((s) => typeof s.metrics[metric] === "number").map((s) => s.metrics[metric]);
      if (values.length === 0) continue;
      const convergence = analyzeSeries(values, opts.convergence);
      if (convergence.state === "insufficient-data")
        insufficientHistory.push(`${judgeId}:${metric}`);
      const current = values[values.length - 1];
      const baseline = values[0];
      const drift = current - baseline;
      const reasons = [];
      if (convergence.state === "drifting-down") {
        reasons.push(
          `convergence state drifting-down (tail run ${convergence.tailRun}, window mean ${convergence.windowMean.toFixed(3)})`
        );
      }
      if (metric === "irr" && current < minIrr) {
        reasons.push(`irr ${current.toFixed(3)} below floor ${minIrr}`);
      }
      if (metric === "calibrationKappa" && baseline - current > maxKappaDrop) {
        reasons.push(
          `calibrationKappa dropped ${(baseline - current).toFixed(3)} from baseline ${baseline.toFixed(3)} (limit ${maxKappaDrop})`
        );
      }
      if (metric === "sentinelPassRate" && baseline - current > maxSentinelDrop) {
        reasons.push(
          `sentinelPassRate dropped ${(baseline - current).toFixed(3)} from baseline ${baseline.toFixed(3)} (limit ${maxSentinelDrop})`
        );
      }
      const alarmed = reasons.length > 0;
      const trend = {
        judgeId,
        metric,
        state: convergence.state,
        current,
        baseline,
        drift,
        alarmed
      };
      if (alarmed) {
        trend.reason = reasons.join("; ");
        alarms.push(`judge "${judgeId}" ${metric}: ${trend.reason}`);
      }
      perJudge.push(trend);
    }
  }
  return { perJudge, alarms, healthy: alarms.length === 0, insufficientHistory };
}
function evalHealthStamp(report) {
  return { healthy: report.healthy, alarms: [...report.alarms] };
}
export {
  FileSystemOutcomeStore,
  InMemoryOutcomeStore,
  SENTINEL_METRIC_NAMES,
  calibrationCurve,
  calibrationFromPairs,
  correlationStudy,
  evalHealthStamp,
  fileSentinelStore,
  inMemorySentinelStore,
  judgeSentinelReport,
  rubricPredictiveValidity,
  snapshotFromAgreement,
  snapshotFromCalibration,
  snapshotFromSentinelSet,
  validateSentinelSnapshot
};
//# sourceMappingURL=index.js.map