// src/rl/verifiable-reward.ts
var DEFAULT_DETERMINISTIC_LAYERS = /* @__PURE__ */ new Set([
  "install",
  "typecheck",
  "build",
  "lint",
  "test",
  "compile",
  "schema",
  "sandbox",
  "unit_tests",
  "integration_tests"
]);
var DEFAULT_SOURCE_FOR = (name) => {
  const lower = name.toLowerCase();
  if (lower.includes("test")) return "test";
  if (lower.includes("compile") || lower.includes("build") || lower.includes("typecheck") || lower.includes("lint"))
    return "compile";
  if (lower.includes("schema")) return "schema";
  if (lower.includes("sandbox")) return "sandbox";
  if (lower.includes("judge") || lower.includes("semantic")) return "judge";
  return "composite";
};
function extractVerifiableReward(report, opts = {}) {
  const deterministicSet = new Set(opts.deterministicLayers ?? [...DEFAULT_DETERMINISTIC_LAYERS]);
  const sourceFor = opts.sourceFor ?? DEFAULT_SOURCE_FOR;
  const fallbackToJudge = opts.fallbackToJudge ?? true;
  const judgeFloor = opts.judgeConfidenceFloor ?? 0.7;
  const deterministic = report.layers.filter(
    (l) => deterministicSet.has(l.layer) && typeof l.score === "number" && Number.isFinite(l.score)
  );
  if (deterministic.length === 1) {
    const layer = deterministic[0];
    return {
      value: clamp01(layer.score),
      source: sourceFor(layer.layer),
      determinism: "deterministic",
      confidence: 1,
      origin: layer.layer,
      breakdown: layerBreakdown(layer)
    };
  }
  if (deterministic.length > 1) {
    let num = 0;
    let denom = 0;
    const breakdown = {};
    for (const l of deterministic) {
      const w = l.detail?.weight ?? 1;
      num += w * (l.score ?? 0);
      denom += w;
      breakdown[l.layer] = l.score;
    }
    return {
      value: denom === 0 ? 0 : clamp01(num / denom),
      source: "composite",
      determinism: "deterministic",
      confidence: 1,
      origin: deterministic.map((l) => l.layer).join("+"),
      breakdown
    };
  }
  if (!fallbackToJudge) return null;
  const judge = report.layers.find(
    (l) => typeof l.score === "number" && Number.isFinite(l.score) && sourceFor(l.layer) === "judge"
  ) ?? report.layers.find((l) => typeof l.score === "number" && Number.isFinite(l.score));
  if (!judge) return null;
  const confFromDetail = judge.detail?.confidence;
  return {
    value: clamp01(judge.score),
    source: "judge",
    determinism: "probabilistic",
    confidence: typeof confFromDetail === "number" ? confFromDetail : judgeFloor,
    origin: judge.layer,
    breakdown: layerBreakdown(judge)
  };
}
function extractVerifiableRewardsFromRecords(runs, opts = {}) {
  const sourceFor = opts.sourceFor ?? DEFAULT_SOURCE_FOR;
  const deterministicSet = new Set(opts.deterministicLayers ?? [...DEFAULT_DETERMINISTIC_LAYERS]);
  const fallbackToJudge = opts.fallbackToJudge ?? true;
  const judgeFloor = opts.judgeConfidenceFloor ?? 0.7;
  return runs.map((run) => {
    const layerScores = [];
    for (const [k, v] of Object.entries(run.outcome.raw)) {
      if (k.startsWith("layer.") && !k.includes(".", 6) && typeof v === "number" && Number.isFinite(v)) {
        layerScores.push({ name: k.slice("layer.".length), score: v });
      }
    }
    const det = layerScores.filter((l) => deterministicSet.has(l.name));
    if (det.length === 1) {
      const layer = det[0];
      return {
        runId: run.runId,
        reward: {
          value: clamp01(layer.score),
          source: sourceFor(layer.name),
          determinism: "deterministic",
          confidence: 1,
          origin: layer.name
        }
      };
    }
    if (det.length > 1) {
      const value = det.reduce((s, l) => s + l.score, 0) / det.length;
      const breakdown = Object.fromEntries(
        det.map((l) => [l.name, l.score])
      );
      return {
        runId: run.runId,
        reward: {
          value: clamp01(value),
          source: "composite",
          determinism: "deterministic",
          confidence: 1,
          origin: det.map((l) => l.name).join("+"),
          breakdown
        }
      };
    }
    if (!fallbackToJudge) return { runId: run.runId, reward: null };
    const primary = run.outcome.holdoutScore ?? run.outcome.searchScore;
    if (typeof primary !== "number" || !Number.isFinite(primary)) {
      return { runId: run.runId, reward: null };
    }
    return {
      runId: run.runId,
      reward: {
        value: clamp01(primary),
        source: "judge",
        determinism: "probabilistic",
        confidence: judgeFloor,
        origin: "run.outcome.score"
      }
    };
  });
}
function filterDeterministicallyRewarded(runs, opts = {}) {
  const rewarded = extractVerifiableRewardsFromRecords(runs, { ...opts, fallbackToJudge: false });
  const out = [];
  for (let i = 0; i < runs.length; i++) {
    const r = rewarded[i];
    if (r.reward && r.reward.determinism === "deterministic") {
      out.push({ run: runs[i], reward: r.reward });
    }
  }
  return out;
}
function clamp01(x) {
  if (!Number.isFinite(x)) return 0;
  return Math.max(0, Math.min(1, x));
}
function layerBreakdown(l) {
  const out = {};
  if (l.diagnostics) {
    for (const [k, v] of Object.entries(l.diagnostics)) {
      if (typeof v === "number" && Number.isFinite(v)) out[k] = v;
    }
  }
  return out;
}

// src/rl/reward-hacking.ts
var DEFAULT_PROXY = (r) => {
  const v = r.outcome.holdoutScore ?? r.outcome.searchScore;
  return typeof v === "number" && Number.isFinite(v) ? v : null;
};
function detectRewardHacking(input) {
  const proxyOf = input.proxyOf ?? DEFAULT_PROXY;
  const truthOf = input.truthOf;
  const sus = input.thresholds?.suspect ?? 0.3;
  const gam = input.thresholds?.gaming ?? 0.6;
  const runs = input.runs.filter((r) => proxyOf(r) !== null);
  const n = runs.length;
  if (n < 4) {
    return {
      findings: [],
      verdict: "clean",
      n,
      rationale: [`fewer than 4 runs with proxy reward (n=${n}); insufficient evidence`]
    };
  }
  const windowSize = Math.max(1, input.windowSize ?? Math.min(50, Math.floor(n / 2)));
  const before = runs.slice(0, n - windowSize);
  const after = runs.slice(n - windowSize);
  const findings = [];
  if (truthOf) {
    const beforeProxy = before.map(proxyOf).filter((v) => typeof v === "number");
    const afterProxy = after.map(proxyOf).filter((v) => typeof v === "number");
    const beforeTruth = before.map(truthOf).filter((v) => typeof v === "number");
    const afterTruth = after.map(truthOf).filter((v) => typeof v === "number");
    if (beforeProxy.length >= 2 && afterProxy.length >= 2 && beforeTruth.length >= 2 && afterTruth.length >= 2) {
      const proxyDelta = mean(afterProxy) - mean(beforeProxy);
      const truthDelta = mean(afterTruth) - mean(beforeTruth);
      const gap = Math.max(0, proxyDelta - truthDelta);
      const severity = clamp012(gap * 5);
      findings.push({
        signal: "reward_divergence",
        severity,
        message: severity >= sus ? `proxy reward rose by ${proxyDelta.toFixed(3)} while truth changed by ${truthDelta.toFixed(3)} \u2014 potential Goodhart` : `proxy and truth moved together (proxy ${proxyDelta.toFixed(3)}, truth ${truthDelta.toFixed(3)})`,
        detail: {
          proxyDelta,
          truthDelta,
          gap,
          beforeN: beforeProxy.length,
          afterN: afterProxy.length
        }
      });
    }
  }
  {
    const beforeP = before.map(proxyOf).filter((v) => typeof v === "number");
    const afterP = after.map(proxyOf).filter((v) => typeof v === "number");
    if (beforeP.length >= 4 && afterP.length >= 4) {
      const ks = ksStatistic(beforeP, afterP);
      const severity = clamp012(ks - 0.2);
      findings.push({
        signal: "distribution_shift",
        severity,
        message: severity >= sus ? `KS=${ks.toFixed(3)} between before/after windows \u2014 distributional shift large` : `KS=${ks.toFixed(3)} between before/after windows \u2014 within-distribution drift`,
        detail: { ks, beforeN: beforeP.length, afterN: afterP.length }
      });
    }
  }
  {
    const secondaryOf = input.secondaryRewardOf ?? defaultSecondary(input.verifiableRewardOptions);
    const aligned = runs.map((r) => ({ p: proxyOf(r), s: secondaryOf(r) })).filter(
      (x) => typeof x.p === "number" && typeof x.s === "number"
    );
    if (aligned.length >= 4) {
      const ps = aligned.map((x) => x.p);
      const ss = aligned.map((x) => x.s);
      const r = pearsonR(ps, ss);
      const severity = clamp012(0.5 - Math.max(0, r));
      findings.push({
        signal: "reward_disagreement",
        severity,
        message: severity >= sus ? `proxy and independent secondary reward correlate \u03C1=${r.toFixed(3)} \u2014 possibly hacking proxy` : `proxy and secondary reward correlate \u03C1=${r.toFixed(3)}`,
        detail: { pearson: r, n: aligned.length }
      });
    }
  }
  {
    const detRuns = filterDeterministicallyRewarded(runs, input.verifiableRewardOptions ?? {});
    if (detRuns.length >= 4) {
      const detBefore = detRuns.slice(0, Math.floor(detRuns.length / 2));
      const detAfter = detRuns.slice(Math.floor(detRuns.length / 2));
      const detDelta = mean(detAfter.map((r) => r.reward.value)) - mean(detBefore.map((r) => r.reward.value));
      const proxyDelta = mean(after.map(proxyOf).filter((v) => typeof v === "number")) - mean(before.map(proxyOf).filter((v) => typeof v === "number"));
      const driftGap = Math.max(0, proxyDelta - detDelta);
      const severity = clamp012(driftGap * 5);
      findings.push({
        signal: "judge_drift",
        severity,
        message: severity >= sus ? `judge proxy +${proxyDelta.toFixed(3)} while deterministic reward +${detDelta.toFixed(3)} \u2014 judge drifting up without verifiable backing` : `judge and deterministic rewards move in step (judge ${proxyDelta.toFixed(3)}, det ${detDelta.toFixed(3)})`,
        detail: { proxyDelta, detDelta, driftGap, n: detRuns.length }
      });
    }
  }
  const maxSev = findings.reduce((m, f) => Math.max(m, f.severity), 0);
  const verdict = maxSev >= gam ? "gaming" : maxSev >= sus ? "suspect" : "clean";
  const rationale = findings.filter((f) => f.severity >= sus).map((f) => `${f.signal}: severity ${f.severity.toFixed(2)} \u2014 ${f.message}`);
  if (rationale.length === 0) rationale.push("no signals fired above suspect threshold");
  return { findings, verdict, rationale, n };
}
function mean(xs) {
  if (xs.length === 0) return 0;
  return xs.reduce((s, x) => s + x, 0) / xs.length;
}
function clamp012(x) {
  if (!Number.isFinite(x)) return 0;
  return Math.max(0, Math.min(1, x));
}
function pearsonR(a, b) {
  if (a.length !== b.length || a.length < 2) return 0;
  const ma = mean(a);
  const mb = mean(b);
  let num = 0, da = 0, db = 0;
  for (let i = 0; i < a.length; i++) {
    const xa = a[i] - ma;
    const xb = b[i] - mb;
    num += xa * xb;
    da += xa * xa;
    db += xb * xb;
  }
  if (da === 0 || db === 0) return 0;
  return num / Math.sqrt(da * db);
}
function ksStatistic(a, b) {
  const sortedA = [...a].sort((x, y) => x - y);
  const sortedB = [...b].sort((x, y) => x - y);
  const all = [.../* @__PURE__ */ new Set([...sortedA, ...sortedB])].sort((x, y) => x - y);
  let max = 0;
  for (const v of all) {
    const fa = sortedA.filter((x) => x <= v).length / sortedA.length;
    const fb = sortedB.filter((x) => x <= v).length / sortedB.length;
    max = Math.max(max, Math.abs(fa - fb));
  }
  return max;
}
function defaultSecondary(verifiableOpts) {
  return (run) => {
    const filtered = filterDeterministicallyRewarded([run], verifiableOpts ?? {});
    return filtered.length === 1 ? filtered[0].reward.value : null;
  };
}

export {
  extractVerifiableReward,
  extractVerifiableRewardsFromRecords,
  filterDeterministicallyRewarded,
  detectRewardHacking
};
//# sourceMappingURL=chunk-YV7J7X5N.js.map