import { createHash } from 'node:crypto';
function extractFirstJsonObject(text) {
    const start = text.indexOf('{');
    if (start === -1)
        return null;
    let depth = 0;
    let inString = false;
    let escaped = false;
    for (let i = start; i < text.length; i++) {
        const ch = text[i];
        if (escaped) {
            escaped = false;
            continue;
        }
        if (ch === '\\') {
            escaped = true;
            continue;
        }
        if (ch === '"') {
            inString = !inString;
            continue;
        }
        if (inString)
            continue;
        if (ch === '{')
            depth++;
        if (ch === '}') {
            depth--;
            if (depth === 0)
                return text.slice(start, i + 1);
        }
    }
    return null;
}
function salvageJudgeResponse(text) {
    const scoreMatch = text.match(/"score"\s*:\s*([0-5](?:\.\d+)?)(?![\d.])/i)
        ?? text.match(/\bscore\b\s*[:=]\s*([0-5](?:\.\d+)?)(?![\d.])/i);
    if (!scoreMatch)
        return null;
    const score = Number(scoreMatch[1]);
    if (!Number.isFinite(score) || score < 0 || score > 5)
        return null;
    return {
        score,
        reason: 'judge returned malformed JSON; score salvaged',
        reasoning: text.trim().slice(0, 2000),
    };
}
/**
 * Judge prompt template version.
 *
 *  - 'v2-cot'         — legacy; CoT scoring without explicit length-debias instruction.
 *                       Kept for `--no-debias-length` so users can reproduce historical
 *                       reports byte-for-byte.
 *  - 'v3-cot-length'  — adds a paragraph telling the judge that length is not a quality
 *                       signal. Default on (research consistently shows
 *                       LLM judges over-weight verbosity; explicit instruction mitigates).
 *
 * Bump when the prompt's intent or structure changes meaningfully — reports tagged
 * with the same hash are score-comparable; mismatched hashes mean "we changed how we
 * ask the judge to think" and should not be compared blind.
 */
// 版本字符串内嵌"判官能看到什么"的语义。bump 时机:
//   v2-cot           : 仅看 output + rubric + 工具名分布(早期 buildTraceSummary)
//   v3-cot-length    : 加了 length-debias 指令(v0.21)
//   v3-cot-toolargs  : 加了 tool input 预览(本次,BREAKING-COMPARABILITY)
//   v4-cot-len-args  : v3-cot-length 的同步升级(BREAKING-COMPARABILITY)
//   bump 原因:之前 trace 只给 tool 名 + 分布,wrapper-style skill(mcporter / code-host CLI 等)
//   被判官当成"只调了 Bash,没用 skill 指定的 MCP 工具"——结论事实错误。
//   tool input 预览让判官能识别 `Bash: mcporter --tool skylark_xxx` 内的真实语义调用。
//
// ⚠️ 命名约定(下次 bump 时执行):
//   当前 debias OFF: v3-* / debias ON: v4-* 编号不对称(历史上 debias OFF 跑 v2→v3,
//   debias ON 跑 v3→v4)。这次再 bump 会让两条线持续偏移 v4/v5、v5/v6、…
//
//   下次 bump 时统一为单一主序号 + features 后缀,如:
//     JUDGE_PROMPT_VERSION_DEBIAS_OFF = 'v5-cot-<features>'
//     JUDGE_PROMPT_VERSION_DEBIAS_ON  = 'v5-cot-<features>-len'
//   这样 hash 测试自然分两条,主序号清晰对齐,features 字段独立描述差异。
//   本 PR 不在意命名,只是把约定写下来,避免下次又跟着错误偏移走。
const JUDGE_PROMPT_VERSION_DEBIAS_OFF = 'v3-cot-toolargs';
const JUDGE_PROMPT_VERSION_DEBIAS_ON = 'v4-cot-len-args';
const JUDGE_SYSTEM_PROMPT = '你是一个严格的 AI 输出质量评审员。先逐条对照评分标准做推理，再给最终分数。只返回 JSON，不要其他内容。';
const LENGTH_DEBIAS_INSTRUCTION = [
    '## 重要：长度不是质量信号',
    '评分时聚焦内容实质与正确性。回答的篇幅、行文丰富度、结构复杂度本身不是质量指标 ——',
    '简洁正确的回答不应因短而扣分；冗长但偏题或重复的回答不应因长而加分。',
    '研究显示 LLM 评委容易隐性偏向更长的回答，请在打分前先警觉这一点。',
].join('\n');
export function buildJudgePrompt(prompt, rubric, output, traceSummary, lengthDebias = true) {
    const version = lengthDebias ? JUDGE_PROMPT_VERSION_DEBIAS_ON : JUDGE_PROMPT_VERSION_DEBIAS_OFF;
    const traceSection = traceSummary
        ? ['', '## Agent 执行过程', traceSummary, '', '请同时考虑执行过程的合理性（工具选择、步骤效率、错误恢复）。']
        : [];
    const debiasSection = lengthDebias ? ['', LENGTH_DEBIAS_INSTRUCTION] : [];
    return [
        `请对以下 AI 输出进行质量评分（template ${version}）。`,
        '',
        '## 原始任务',
        prompt,
        '',
        '## 评分标准',
        rubric,
        '',
        '## AI 输出',
        output,
        ...traceSection,
        ...debiasSection,
        '',
        '## 评分流程',
        '1. 逐条对照评分标准，先做推理（reasoning）：列出 AI 输出哪些点对应哪条标准，哪些缺失，哪些有歧义。',
        '2. 基于推理给出最终分数（1-5 的整数）和简短理由。',
        '',
        '请返回 JSON（不要包含 markdown 代码块标记）：',
        '{"reasoning": "<对照标准的逐条推理>", "score": <1-5的整数>, "reason": "<最终结论的简短理由>"}',
        '',
        '评分标准：1=完全不达标, 2=部分涉及, 3=基本达标, 4=较好, 5=优秀',
    ].join('\n');
}
/**
 * Stable hash of the judge prompt template. Saved into ReportMeta.judgePromptHash so
 * downstream readers can detect "the judge prompt changed between these two reports".
 *
 * `lengthDebias` defaults to true (v0.21+ default). Pass false when running under
 * `--no-debias-length` so the hash matches historical v2-cot reports.
 */
export function getJudgePromptHash(lengthDebias = true) {
    const version = lengthDebias ? JUDGE_PROMPT_VERSION_DEBIAS_ON : JUDGE_PROMPT_VERSION_DEBIAS_OFF;
    // Hash the template-shaping function source + the version tag together. We hash a
    // deterministic stringified form of the template (with placeholder inputs) so any
    // structural edit shows up.
    const sample = buildJudgePrompt('<P>', '<R>', '<O>', '<T>', lengthDebias);
    return createHash('sha256').update(version + '\n' + sample).digest('hex').slice(0, 12);
}
function getErrorMessage(err) {
    return err instanceof Error ? err.message : String(err);
}
/**
 * Tool input 单行预览。Bash 命令(input.command)直接拿;其它 tool 走 JSON 序列化。
 * 截断到 INPUT_PREVIEW_MAX 字符,避免 prompt 膨胀过头。
 *
 * 关键:wrapper-style skill(mcporter / code-host CLI / git CLI 等)的真实语义调用
 * 编码在 Bash 命令字符串里(`mcporter --tool X` / `code-host pr show Y`),判官看不到
 * 就会下"没调指定工具"的错误结论。这个预览是判官识别这层语义的唯一通道。
 */
const INPUT_PREVIEW_MAX = 280;
const TOOL_DETAIL_MAX_CALLS = 12;
function previewToolInput(tc) {
    const inp = tc.input;
    if (inp == null)
        return '';
    // Bash 是 wrapper 大头,优先显示 command 字段
    if (typeof inp === 'object' && 'command' in inp && typeof inp.command === 'string') {
        const cmd = inp.command;
        return cmd.length > INPUT_PREVIEW_MAX ? cmd.slice(0, INPUT_PREVIEW_MAX) + '…' : cmd;
    }
    const repr = typeof inp === 'string' ? inp : JSON.stringify(inp);
    return repr.length > INPUT_PREVIEW_MAX ? repr.slice(0, INPUT_PREVIEW_MAX) + '…' : repr;
}
export function buildTraceSummary(turns, toolCalls) {
    if ((!turns || turns.length === 0) && (!toolCalls || toolCalls.length === 0))
        return null;
    const lines = [];
    if (toolCalls && toolCalls.length > 0) {
        lines.push(`共调用 ${toolCalls.length} 个工具：`);
        const successCount = toolCalls.filter((tc) => tc.success).length;
        const failureCount = toolCalls.length - successCount;
        lines.push(`  成功 ${successCount}/${toolCalls.length}`);
        if (failureCount > 0)
            lines.push(`  失败 ${failureCount}/${toolCalls.length}`);
        const dist = {};
        for (const tc of toolCalls) {
            dist[tc.tool] = (dist[tc.tool] || 0) + 1;
        }
        lines.push(`  工具分布：${Object.entries(dist).map(([k, v]) => `${k}(${v})`).join(', ')}`);
        const failedTools = toolCalls.filter((tc) => !tc.success).map((tc) => tc.tool);
        if (failedTools.length > 0) {
            lines.push(`  失败工具：${[...new Set(failedTools)].join(', ')}`);
        }
        // Tool input 详情(critical for wrapper-style skill 的判官准确性,见 INPUT_PREVIEW_MAX 注释)。
        // 截 TOOL_DETAIL_MAX_CALLS 条防止长 trace 膨胀 prompt。
        lines.push(`  调用详情(注意:Bash 命令内的 \`mcporter --tool X\` / \`code-host Y\` 等才是真实语义调用):`);
        const detailCap = Math.min(toolCalls.length, TOOL_DETAIL_MAX_CALLS);
        for (let i = 0; i < detailCap; i++) {
            const tc = toolCalls[i];
            const status = tc.success ? '' : ' [失败]';
            lines.push(`    [${i + 1}] ${tc.tool}${status} → ${previewToolInput(tc)}`);
        }
        if (toolCalls.length > TOOL_DETAIL_MAX_CALLS) {
            lines.push(`    ... 还有 ${toolCalls.length - TOOL_DETAIL_MAX_CALLS} 次调用`);
        }
    }
    if (turns && turns.length > 0) {
        lines.push('');
        lines.push('执行轨迹摘要：');
        const assistantTurns = turns.filter((turn) => turn.role === 'assistant').length;
        const toolTurns = turns.filter((turn) => turn.role === 'tool').length;
        lines.push(`  共 ${turns.length} 步（assistant ${assistantTurns} / tool ${toolTurns}）`);
        const maxTurns = Math.min(turns.length, 10);
        for (let i = 0; i < maxTurns; i++) {
            const t = turns[i];
            const preview = t.content.slice(0, 100) + (t.content.length > 100 ? '...' : '');
            if (t.role === 'assistant' && t.toolCalls?.length) {
                lines.push(`  [${i + 1}] assistant: 调用 ${t.toolCalls.map((tc) => tc.tool).join(', ')}`);
            }
            else if (t.role === 'tool') {
                lines.push(`  [${i + 1}] tool: ${preview}`);
            }
            else {
                lines.push(`  [${i + 1}] ${t.role}: ${preview}`);
            }
        }
        if (turns.length > maxTurns)
            lines.push(`  ... 还有 ${turns.length - maxTurns} 步`);
    }
    return lines.join('\n');
}
export async function llmJudge({ output, rubric, prompt, executor, model, traceSummary, lengthDebias }) {
    const judgePrompt = buildJudgePrompt(prompt, rubric, output, traceSummary || null, lengthDebias ?? true);
    const result = await executor({
        model,
        system: JUDGE_SYSTEM_PROMPT,
        prompt: judgePrompt,
    });
    // 透传 executor 是否报告 cost。codex executor `costReportedByExecutor: false`,
    // 让 grade / VariantResult / renderer 跨 judge 路径仍能识别 cost 非真值。
    const reportedField = result.costReportedByExecutor === false ? { judgeCostReportedByExecutor: false } : {};
    if (!result.ok)
        return { score: 0, reason: `judge error: ${result.error}`, judgeCostUSD: result.costUSD, ...reportedField };
    try {
        const text = result.output.trim();
        const jsonText = extractFirstJsonObject(text);
        if (!jsonText) {
            process.stderr.write(`[omk] LLM judge returned non-JSON: ${text.slice(0, 100)}\n`);
            return { score: 0, reason: 'judge returned non-JSON', judgeCostUSD: result.costUSD, ...reportedField };
        }
        const parsed = JSON.parse(jsonText);
        return {
            score: Number(parsed.score) || 0,
            reason: String(parsed.reason || ''),
            reasoning: parsed.reasoning ? String(parsed.reasoning) : undefined,
            judgeCostUSD: result.costUSD,
            ...reportedField,
        };
    }
    catch (parseErr) {
        const salvaged = salvageJudgeResponse(result.output || '');
        if (salvaged) {
            process.stderr.write(`[omk] LLM judge malformed JSON salvaged: ${getErrorMessage(parseErr)}\n`);
            return {
                score: Number(salvaged.score) || 0,
                reason: String(salvaged.reason || ''),
                reasoning: salvaged.reasoning ? String(salvaged.reasoning) : undefined,
                judgeCostUSD: result.costUSD,
                ...reportedField,
            };
        }
        process.stderr.write(`[omk] LLM judge parse error: ${getErrorMessage(parseErr)}\n`);
        return { score: 0, reason: 'failed to parse judge response', judgeCostUSD: result.costUSD, ...reportedField };
    }
}
/**
 * Max concurrent judge calls within a single sample × dimension. Each judge call
 * is one API request; running N=3 sequentially means 3× latency, but going wide
 * open risks hitting per-account RPM limits. 3 is a conservative middle ground —
 * for N=3 (the common case) it's fully parallel; for N=10 it's 4 batches.
 */
const JUDGE_REPEAT_CONCURRENCY = 3;
async function runInChunks(items, chunkSize, fn) {
    const results = new Array(items);
    for (let start = 0; start < items; start += chunkSize) {
        const end = Math.min(start + chunkSize, items);
        const batch = await Promise.all(Array.from({ length: end - start }, (_, k) => fn(start + k)));
        for (let k = 0; k < batch.length; k++)
            results[start + k] = batch[k];
    }
    return results;
}
/**
 * Judge a single (output, rubric) pair N times and aggregate. Returns mean score,
 * stddev across runs, raw score samples, and the first-call reasoning (we keep one
 * reasoning sample, not N — only the score distribution matters for stability).
 *
 * Calls run with bounded concurrency (JUDGE_REPEAT_CONCURRENCY) to amortize latency
 * without burning rate limit. N=3 finishes in ~1 round-trip; N=10 in ~4. Cost is
 * summed across all N calls. When N <= 1 this is equivalent to llmJudge() with
 * scoreSamples = [score], scoreStddev = 0. The repeat value is clamped to >= 1
 * here as well as at the CLI layer — library callers shouldn't see surprises.
 *
 * Failures: any call returning score=0 (non-JSON / executor error / parse error) is
 * counted in `judgeFailureCount`. stddev is computed only over successful calls. If
 * stddev is 0 but judgeFailureCount > 0, that's NOT "judge agreed perfectly" — it
 * means most calls failed and one happened to succeed. Always inspect both fields.
 */
export async function llmJudgeRepeat(options, repeat) {
    const n = Math.max(1, Math.floor(repeat) || 1);
    if (n === 1) {
        const single = await llmJudge(options);
        const failed = single.score <= 0 ? 1 : 0;
        return { ...single, scoreSamples: [single.score], scoreStddev: 0, judgeFailureCount: failed };
    }
    // Run N judge calls with bounded concurrency. Result array preserves input order
    // (call 0 → results[0]) so "first call reasoning" is well-defined regardless of
    // which physical call returned first.
    const calls = await runInChunks(n, JUDGE_REPEAT_CONCURRENCY, () => llmJudge(options));
    const samples = calls.map((c) => c.score);
    const totalCost = calls.reduce((sum, c) => sum + (c.judgeCostUSD || 0), 0);
    // 任一 call 的 executor 不报 cost → 整个 repeat 的 totalCost 不可信
    const anyCostUnreported = calls.some((c) => c.judgeCostReportedByExecutor === false);
    const firstReasoning = calls[0]?.reasoning;
    const firstReason = calls.find((c) => c.reason)?.reason || '';
    const validSamples = samples.filter((s) => s > 0);
    const failures = samples.length - validSamples.length;
    const mean = validSamples.length > 0 ? validSamples.reduce((a, b) => a + b, 0) / validSamples.length : 0;
    const variance = validSamples.length > 1
        ? validSamples.reduce((s, x) => s + (x - mean) ** 2, 0) / (validSamples.length - 1)
        : 0;
    const stddev = Math.sqrt(variance);
    return {
        score: Number(mean.toFixed(2)),
        reason: firstReason,
        reasoning: firstReasoning,
        judgeCostUSD: totalCost,
        ...(anyCostUnreported && { judgeCostReportedByExecutor: false }),
        scoreSamples: samples,
        scoreStddev: Number(stddev.toFixed(3)),
        judgeFailureCount: failures,
    };
}
// ===========================================================================
// Multi-judge ensemble — cross-model agreement
// ===========================================================================
/** Format a JudgeConfig as "executor:model" identifier for reports / logs. */
export function judgeId(config) {
    return `${config.executor}:${config.model}`;
}
/**
 * Pearson correlation between two number arrays of equal length. Returns null when
 * either array has zero variance (constant scores) — Pearson is undefined in that
 * case (division by zero), and reporting 0 would be misleading.
 */
function pearson(a, b) {
    if (a.length !== b.length || a.length < 2)
        return null;
    const n = a.length;
    const meanA = a.reduce((s, x) => s + x, 0) / n;
    const meanB = b.reduce((s, x) => s + x, 0) / n;
    let num = 0;
    let denomA = 0;
    let denomB = 0;
    for (let i = 0; i < n; i++) {
        const da = a[i] - meanA;
        const db = b[i] - meanB;
        num += da * db;
        denomA += da * da;
        denomB += db * db;
    }
    if (denomA === 0 || denomB === 0)
        return null;
    return num / Math.sqrt(denomA * denomB);
}
/** Mean absolute difference between two equal-length number arrays. */
function meanAbsDiff(a, b) {
    if (a.length !== b.length || a.length === 0)
        return 0;
    let sum = 0;
    for (let i = 0; i < a.length; i++)
        sum += Math.abs(a[i] - b[i]);
    return sum / a.length;
}
/**
 * Compute pairwise inter-judge agreement metrics across an ensemble. For N judges
 * we get N*(N-1)/2 pairs; metrics are pairwise-averaged.
 *
 * Each judge contributes ONE score per sample (its mean if judge-repeat > 1). We
 * then compute Pearson and mean-abs-diff over the N-judge × M-sample score matrix.
 *
 * NOTE: Within a single (sample × dimension) call, each judge gives ONE aggregated
 * score. So `pairwise` here means "two judges' scores on this one sample". With a
 * single sample point Pearson is undefined (need ≥ 2). For per-sample agreement we
 * fall back to mean-abs-diff alone; Pearson kicks in at the report-aggregate level
 * (across many samples).
 */
export function computeJudgeAgreement(judgeScores) {
    // judgeScores[i][j] = score from judge i on sample j. All rows same length.
    const n = judgeScores.length;
    if (n < 2)
        return { meanAbsDiff: 0, pairCount: 0 };
    let madSum = 0;
    let pearsonSum = 0;
    let pearsonCount = 0;
    let pairs = 0;
    for (let i = 0; i < n; i++) {
        for (let j = i + 1; j < n; j++) {
            pairs++;
            madSum += meanAbsDiff(judgeScores[i], judgeScores[j]);
            const p = pearson(judgeScores[i], judgeScores[j]);
            if (p !== null) {
                pearsonSum += p;
                pearsonCount++;
            }
        }
    }
    const result = {
        meanAbsDiff: Number((madSum / pairs).toFixed(3)),
        pairCount: pairs,
    };
    if (pearsonCount > 0) {
        result.pearson = Number((pearsonSum / pearsonCount).toFixed(3));
    }
    return result;
}
/**
 * Judge a single (output, rubric) pair with N judge models in parallel. Each judge
 * may use a different executor (e.g. claude:opus + openai:gpt-4o + gemini:pro). Each
 * judge can also be repeated `judgeRepeat` times — final per-judge score is its mean.
 *
 * Returns: aggregate DimensionResult (score = mean across judges; this is the "consensus"
 * score), per-judge breakdown in `ensemble`, and agreement metrics in `agreement`.
 *
 * The aggregate score is the mean of per-judge scores. This is a defensible default
 * but not the only choice — one could argue median (robust to outlier judges) or
 * majority vote (if scores are categorical). Mean is what most papers report; we
 * provide the raw ensemble so downstream can recompute.
 */
export async function llmJudgeEnsemble(options, judges, executorByName, judgeRepeat = 1) {
    if (judges.length === 0) {
        throw new Error('llmJudgeEnsemble called with empty judges array');
    }
    if (judges.length === 1) {
        // Degenerate case — fall through to non-ensemble path.
        return llmJudgeRepeat({ ...options, executor: executorByName(judges[0].executor), model: judges[0].model }, judgeRepeat);
    }
    // Run all judges in parallel — they're independent. Each judge internally handles
    // its judge-repeat sequence.
    const perJudge = await Promise.all(judges.map(async (jc) => {
        const r = await llmJudgeRepeat({ ...options, executor: executorByName(jc.executor), model: jc.model }, judgeRepeat);
        const entry = {
            judge: judgeId(jc),
            score: r.score,
            scoreStddev: r.scoreStddev,
            scoreSamples: r.scoreSamples,
            judgeFailureCount: r.judgeFailureCount,
            reasoning: r.reasoning,
            costUSD: r.judgeCostUSD,
            ...(r.judgeCostReportedByExecutor === false && { costReportedByExecutor: false }),
        };
        return { entry, raw: r };
    }));
    const ensemble = perJudge.map((p) => p.entry);
    const totalCost = perJudge.reduce((s, p) => s + (p.raw.judgeCostUSD || 0), 0);
    // 任一 judge 的 executor 不报 cost → ensemble totalCost 不可信
    const anyCostUnreported = perJudge.some((p) => p.raw.judgeCostReportedByExecutor === false);
    // Aggregate score = mean of per-judge means (consensus).
    const validScores = ensemble.map((e) => e.score).filter((s) => s > 0);
    const consensusScore = validScores.length > 0
        ? Number((validScores.reduce((a, b) => a + b, 0) / validScores.length).toFixed(2))
        : 0;
    // Per-judge agreement: only one sample point here (this single output), so Pearson
    // is undefined and we report mean-abs-diff. Aggregate-level Pearson (across all
    // samples in the run) is computed by the report-level aggregator, not here.
    const judgeScoreMatrix = ensemble.map((e) => [e.score]);
    const agreement = computeJudgeAgreement(judgeScoreMatrix);
    // Pick the "spokesperson" reasoning from the first judge that produced one.
    const spokesperson = ensemble.find((e) => e.reasoning);
    return {
        score: consensusScore,
        reason: `consensus across ${judges.length} judges`,
        reasoning: spokesperson?.reasoning,
        judgeCostUSD: totalCost,
        ...(anyCostUnreported && { judgeCostReportedByExecutor: false }),
        ensemble,
        agreement,
    };
}
//# sourceMappingURL=judge.js.map