import { resolve } from 'node:path';
import _Ajv from 'ajv';
const Ajv = _Ajv.default ?? _Ajv;
const ajv = new Ajv();
const CUSTOM_ASSERTION_TIMEOUT_MS = 30_000;
export const ASYNC_ASSERTION_TYPES = new Set([
    'semantic_similarity',
    'custom',
    // RAG-specific judge metrics. All three go through the LLM-judge path
    // and inherit the same length-debias instruction as the main rubric judge.
    'faithfulness',
    'answer_relevancy',
    'context_recall',
]);
function getErrorMessage(err) {
    return err instanceof Error ? err.message : String(err);
}
export function ratioToScore(ratio) {
    return Number((1 + ratio * 4).toFixed(2));
}
// ===========================================================================
// Deterministic similarity metrics
// ===========================================================================
/** Tokenizer: Latin words/numbers as units; CJK chars as single-char units. */
function tokenize(s) {
    const m = s.toLowerCase().match(/[a-z0-9]+|[一-龥]/g);
    return m ?? [];
}
function ngrams(tokens, n) {
    if (n <= 0 || tokens.length < n)
        return [];
    const out = new Array(tokens.length - n + 1);
    for (let i = 0; i <= tokens.length - n; i++) {
        out[i] = tokens.slice(i, i + n).join(' ');
    }
    return out;
}
function clippedOverlap(candNgrams, refNgrams) {
    if (candNgrams.length === 0 || refNgrams.length === 0)
        return 0;
    const candCount = new Map();
    const refCount = new Map();
    for (const g of candNgrams)
        candCount.set(g, (candCount.get(g) ?? 0) + 1);
    for (const g of refNgrams)
        refCount.set(g, (refCount.get(g) ?? 0) + 1);
    let total = 0;
    for (const [g, cc] of candCount) {
        total += Math.min(cc, refCount.get(g) ?? 0);
    }
    return total;
}
/**
 * ROUGE-N (recall-oriented). N-gram overlap divided by the reference's
 * n-gram count. Multi-set form per Lin 2004 — repeated n-grams are clipped
 * to their reference count, so a candidate that repeats a single matching
 * n-gram many times can't game the score above 1.
 */
export function rougeN(candidate, reference, n) {
    const cn = ngrams(tokenize(candidate), n);
    const rn = ngrams(tokenize(reference), n);
    if (rn.length === 0)
        return 0;
    return clippedOverlap(cn, rn) / rn.length;
}
/** Standard Levenshtein edit distance with O(min(m,n)) memory. */
export function levenshtein(a, b) {
    if (a === b)
        return 0;
    if (a.length === 0)
        return b.length;
    if (b.length === 0)
        return a.length;
    // Ensure b is the shorter one for memory.
    if (a.length < b.length)
        [a, b] = [b, a];
    const m = a.length, n = b.length;
    let prev = new Array(n + 1);
    let curr = new Array(n + 1);
    for (let j = 0; j <= n; j++)
        prev[j] = j;
    for (let i = 1; i <= m; i++) {
        curr[0] = i;
        for (let j = 1; j <= n; j++) {
            const cost = a[i - 1] === b[j - 1] ? 0 : 1;
            curr[j] = Math.min(prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + cost);
        }
        [prev, curr] = [curr, prev];
    }
    return prev[n];
}
/**
 * BLEU score (corpus-level form, single reference). Geometric mean of
 * precision at n=1..maxN, multiplied by the brevity penalty for short
 * candidates. Unsmoothed — returns 0 if any n-gram precision is 0, which
 * is a known sharpness of the original BLEU; threshold accordingly.
 */
export function bleu(candidate, reference, maxN = 4) {
    const ct = tokenize(candidate);
    const rt = tokenize(reference);
    if (ct.length === 0 || rt.length === 0)
        return 0;
    let logSum = 0;
    for (let n = 1; n <= maxN; n++) {
        const cn = ngrams(ct, n);
        const rn = ngrams(rt, n);
        if (cn.length === 0)
            return 0;
        const overlap = clippedOverlap(cn, rn);
        if (overlap === 0)
            return 0;
        logSum += Math.log(overlap / cn.length);
    }
    const bp = ct.length >= rt.length ? 1 : Math.exp(1 - rt.length / ct.length);
    return bp * Math.exp(logSum / maxN);
}
export function validateJsonSchema(data, schema) {
    if (!schema || typeof schema !== 'object')
        return true;
    try {
        const validate = ajv.compile(schema);
        return validate(data);
    }
    catch {
        return false;
    }
}
/**
 * Evaluate a single assertion against the output. Returns the raw pass/fail
 * BEFORE applying `not`. Pulled out of `runAssertions` so `assert-set` can
 * recurse on children without rebuilding the context.
 *
 * Universal negation (`not: true`) and `assert-set` (any/all combinator) are
 * supported here. Legacy `not_contains` / `not_equals` / etc. still work;
 * their negation is hard-coded inline as before.
 */
function evalAssertion(output, assertion, ctx) {
    const { outputLower, toolCalls, toolNames } = ctx;
    if (assertion.type === 'assert-set') {
        const children = assertion.children ?? [];
        if (children.length === 0)
            return false;
        const mode = assertion.mode ?? 'all';
        const childPasses = children.map((c) => {
            const raw = evalAssertion(output, c, ctx);
            return c.not ? !raw : raw;
        });
        return mode === 'any' ? childPasses.some(Boolean) : childPasses.every(Boolean);
    }
    switch (assertion.type) {
        case 'contains':
            return outputLower.includes(String(assertion.value).toLowerCase());
        case 'not_contains':
            return !outputLower.includes(String(assertion.value).toLowerCase());
        case 'regex': {
            const flags = assertion.flags || 'i';
            const re = new RegExp(assertion.pattern, flags);
            return re.test(output);
        }
        case 'min_length':
            return output.length >= assertion.value;
        case 'max_length':
            return output.length <= assertion.value;
        case 'json_valid':
            try {
                JSON.parse(output);
                return true;
            }
            catch {
                return false;
            }
        case 'json_schema':
            try {
                const data = JSON.parse(output);
                return validateJsonSchema(data, assertion.schema);
            }
            catch {
                return false;
            }
        case 'starts_with':
            return outputLower.startsWith(String(assertion.value).toLowerCase());
        case 'ends_with':
            return outputLower.endsWith(String(assertion.value).toLowerCase());
        case 'equals':
            return output.trim() === String(assertion.value).trim();
        case 'not_equals':
            return output.trim() !== String(assertion.value).trim();
        case 'word_count_min':
            return output.split(/\s+/).filter(Boolean).length >= assertion.value;
        case 'word_count_max':
            return output.split(/\s+/).filter(Boolean).length <= assertion.value;
        case 'contains_all':
            return (assertion.values || []).every((v) => outputLower.includes(String(v).toLowerCase()));
        case 'contains_any':
            return (assertion.values || []).some((v) => outputLower.includes(String(v).toLowerCase()));
        case 'cost_max':
            return (ctx.costUSD ?? Infinity) <= assertion.value;
        case 'latency_max':
            return (ctx.durationMs ?? Infinity) <= assertion.value;
        case 'turns_max':
            return (ctx.numTurns ?? Infinity) <= assertion.value;
        case 'turns_min':
            return (ctx.numTurns ?? 0) >= assertion.value;
        case 'tools_called':
            return (assertion.values || []).every((v) => toolNames.includes(String(v).toLowerCase()));
        case 'tools_not_called':
            return (assertion.values || []).every((v) => !toolNames.includes(String(v).toLowerCase()));
        case 'tools_count_max':
            return toolCalls.length <= assertion.value;
        case 'tools_count_min':
            return toolCalls.length >= assertion.value;
        case 'tool_output_contains': {
            const sep = String(assertion.value).indexOf(':');
            if (sep <= 0)
                return false;
            const targetTool = String(assertion.value).slice(0, sep).toLowerCase();
            const expected = String(assertion.value).slice(sep + 1).toLowerCase();
            return toolCalls.some((tc) => tc.tool.toLowerCase() === targetTool &&
                String(tc.output || '').toLowerCase().includes(expected));
        }
        case 'tool_input_contains': {
            const sep = String(assertion.value).indexOf(':');
            if (sep <= 0)
                return false;
            const targetTool = String(assertion.value).slice(0, sep).toLowerCase();
            const expected = String(assertion.value).slice(sep + 1).toLowerCase();
            return toolCalls.some((tc) => tc.tool.toLowerCase() === targetTool &&
                JSON.stringify(tc.input || '').toLowerCase().includes(expected));
        }
        case 'tool_input_not_contains': {
            const sep = String(assertion.value).indexOf(':');
            // value 必须是 "Tool:needle" 格式。无冒号(generator 偶尔产 "--force" / "lastTaskPatrol"
            // 这种裸 needle)在 _contains 语义里算 false(没法判定哪个工具),但 _not_contains 语义
            // 应该 trivially pass(没有匹配 = 没传 needle = 满足"不应包含"约束)。否则会变成
            // 假阳性失败,扭曲通过率。loader 已加格式校验拒绝这种 value,这里是 grader 兜底。
            if (sep <= 0)
                return true;
            const targetTool = String(assertion.value).slice(0, sep).toLowerCase();
            const expected = String(assertion.value).slice(sep + 1).toLowerCase();
            return !toolCalls.some((tc) => tc.tool.toLowerCase() === targetTool &&
                JSON.stringify(tc.input || '').toLowerCase().includes(expected));
        }
        // mock_hit:校验"驱动流程"——指定 mock 是否被命中至少 N 次
        // value 是 mockStats.perMock 的 key,格式 "Tool:N"(N 为 sample.mocks 数组里的 1-based 序号)
        // threshold 可选,默认 >=1
        // 例: { type: 'mock_hit', value: 'Bash:2' } 检查第 2 条 Bash mock 被调过
        case 'mock_hit': {
            const stats = ctx.mockStats;
            if (!stats)
                return false;
            const key = String(assertion.value);
            const hits = stats.perMock[key] ?? 0;
            const min = assertion.threshold ?? 1;
            return hits >= min;
        }
        case 'rouge_n_min':
            return rougeN(output, String(assertion.reference ?? assertion.value ?? ''), assertion.n ?? 1)
                >= (assertion.threshold ?? 0.5);
        case 'levenshtein_max':
            return levenshtein(output, String(assertion.reference ?? assertion.value ?? ''))
                <= (assertion.value ?? Infinity);
        case 'bleu_min':
            return bleu(output, String(assertion.reference ?? assertion.value ?? ''))
                >= (assertion.threshold ?? 0.5);
        default:
            return false;
    }
}
export function runAssertions(output, assertions, context = {}) {
    const outputLower = output.toLowerCase();
    const toolCalls = context.toolCalls || [];
    const toolNames = toolCalls.map((tc) => tc.tool.toLowerCase());
    const ctx = { outputLower, toolCalls, toolNames, costUSD: context.costUSD, durationMs: context.durationMs, numTurns: context.numTurns, mockStats: context.mockStats };
    const details = [];
    for (const assertion of assertions) {
        const weight = assertion.weight ?? 1;
        const raw = evalAssertion(output, assertion, ctx);
        const passed = assertion.not ? !raw : raw;
        details.push({
            type: assertion.type,
            value: assertion.value ?? assertion.pattern ?? assertion.values?.join(', ') ?? '',
            weight,
            passed,
        });
    }
    const totalWeight = details.reduce((s, d) => s + d.weight, 0);
    const passedWeight = details.filter((d) => d.passed).reduce((s, d) => s + d.weight, 0);
    const passedCount = details.filter((d) => d.passed).length;
    const ratio = totalWeight > 0 ? passedWeight / totalWeight : 0;
    return {
        passed: passedCount,
        total: details.length,
        score: ratioToScore(ratio),
        details,
    };
}
export async function runAsyncAssertions(output, assertions, { executor, judgeModel, sample, samplesDir }) {
    const details = [];
    let asyncCostUSD = 0;
    let anyCostUnreported = false;
    for (const assertion of assertions) {
        const weight = assertion.weight ?? 1;
        let passed = false;
        let message = '';
        if (assertion.type === 'semantic_similarity') {
            const reference = assertion.reference || '';
            const judgePrompt = [
                '请判断以下两段文本的语义相似度。',
                '',
                '## 参考文本',
                reference,
                '',
                '## 待评估文本',
                output,
                '',
                '请返回 JSON（不要包含 markdown 代码块标记）：',
                '{"score": <1-5的整数>, "reason": "<简短理由>"}',
                '',
                '评分：1=完全无关, 2=略有关联, 3=部分相似, 4=大致相同, 5=高度一致',
            ].join('\n');
            const result = await executor({
                model: judgeModel,
                system: '你是语义相似度评审员。只返回 JSON，不要其他内容。',
                prompt: judgePrompt,
            });
            asyncCostUSD += result.costUSD || 0;
            if (result.costReportedByExecutor === false)
                anyCostUnreported = true;
            if (result.ok) {
                try {
                    const jsonMatch = result.output.trim().match(/\{[\s\S]*\}/);
                    if (jsonMatch) {
                        const parsed = JSON.parse(jsonMatch[0]);
                        const score = Number(parsed.score) || 0;
                        const threshold = assertion.threshold ?? 3;
                        passed = score >= threshold;
                        message = parsed.reason || '';
                    }
                    else {
                        process.stderr.write(`[omk] semantic_similarity judge returned non-JSON: ${result.output.slice(0, 100)}\n`);
                    }
                }
                catch (parseErr) {
                    process.stderr.write(`[omk] semantic_similarity judge parse error: ${getErrorMessage(parseErr)}\n`);
                }
            }
        }
        else if (assertion.type === 'faithfulness' ||
            assertion.type === 'answer_relevancy' ||
            assertion.type === 'context_recall') {
            const ragResult = await runRagJudge(assertion, output, sample, executor, judgeModel);
            asyncCostUSD += ragResult.costUSD;
            if (ragResult.costReportedByExecutor === false)
                anyCostUnreported = true;
            passed = ragResult.passed;
            message = ragResult.message;
        }
        else if (assertion.type === 'custom') {
            try {
                const fnPath = resolve(samplesDir, assertion.fn);
                const mod = await import(fnPath);
                const fn = mod.default || mod.check;
                if (!fn)
                    throw new Error('custom assertion module must export default or check');
                const result = await Promise.race([
                    fn(output, { sample, assertion }),
                    new Promise((_, reject) => setTimeout(() => reject(new Error(`custom assertion timed out (${CUSTOM_ASSERTION_TIMEOUT_MS / 1000}s)`)), CUSTOM_ASSERTION_TIMEOUT_MS)),
                ]);
                passed = Boolean(result.pass);
                message = result.message || '';
            }
            catch (err) {
                passed = false;
                message = `custom assertion error: ${getErrorMessage(err)}`;
            }
        }
        details.push({
            type: assertion.type,
            value: assertion.reference || assertion.fn || '',
            weight,
            passed,
            ...(message && { message }),
        });
    }
    const totalWeight = details.reduce((s, d) => s + d.weight, 0);
    const passedWeight = details.filter((d) => d.passed).reduce((s, d) => s + d.weight, 0);
    const passedCount = details.filter((d) => d.passed).length;
    const ratio = totalWeight > 0 ? passedWeight / totalWeight : 0;
    return {
        passed: passedCount,
        total: details.length,
        score: ratioToScore(ratio),
        details,
        judgeCostUSD: asyncCostUSD,
        ...(anyCostUnreported && { judgeCostReportedByExecutor: false }),
    };
}
// ===========================================================================
// RAG-specific judge metrics
// ===========================================================================
//
// Three metrics, all running through the LLM judge:
//
//  - faithfulness:      does the output's content stay grounded in the
//                       reference context? Anti-hallucination check.
//  - answer_relevancy:  does the output directly answer the user's question?
//                       Catches verbose dodges and topic drift.
//  - context_recall:    are the key facts from the gold context actually
//                       used in the output? Catches retrieved-but-ignored
//                       context (a common RAG bug).
//
// Implementation notes:
//
//  1. Each prompt is a SINGLE-CALL judge (1-5 score) rather than the multi-step
//     statement-decomposition that RAGAS uses. This is honest tradeoff: simpler,
//     faster, less rigorous than RAGAS but consistent with omk's other LLM-judge
//     assertions. Users who need RAGAS-grade decomposition can drop down to a
//     custom assertion.
//  2. The prompt includes the SAME length-debias paragraph as the main judge
//     prompt (v3-cot-length) — output verbosity is not a quality signal here
//     either. This is the "auto-inherit length-debias" claim from the  plan.
//  3. Reference resolution:
//       faithfulness:    sample.context  (or assertion.reference override)
//       context_recall:  assertion.reference (or sample.context fallback)
//       answer_relevancy: sample.prompt — no reference needed
//  4. Threshold defaults to 3 (same as semantic_similarity). User can override.
const RAG_LENGTH_DEBIAS = [
    '## 重要:长度不是质量信号',
    '评分时聚焦内容实质,不要因输出更长就给更高分。',
    '简洁正确的回答与冗长正确的回答应得相同分数。',
].join('\n');
async function runRagJudge(assertion, output, sample, executor, judgeModel) {
    const threshold = assertion.threshold ?? 3;
    let prompt;
    let system;
    if (assertion.type === 'faithfulness') {
        const context = assertion.reference || sample.context || '';
        if (!context) {
            return { passed: false, message: 'faithfulness: 缺少 sample.context 或 assertion.reference', costUSD: 0 };
        }
        system = '你是 RAG 评审员,专注判断输出是否被参考 context 支持。只返回 JSON。';
        prompt = [
            '请判断"待评估输出"中的事实性陈述是否被"参考 context"支持。',
            '',
            '## 参考 context',
            context,
            '',
            '## 待评估输出',
            output,
            '',
            RAG_LENGTH_DEBIAS,
            '',
            '## 评分流程',
            '1. 列出待评估输出中所有事实性陈述',
            '2. 逐条判断是否能在 context 中找到支持',
            '3. 给出 1-5 分:',
            '   5 = 全部陈述都有 context 支持,无编造',
            '   4 = 多数有支持,有 1-2 处不重要的编造',
            '   3 = 一半有支持',
            '   2 = 多数无支持',
            '   1 = 完全编造或与 context 矛盾',
            '',
            '请返回 JSON(不要 markdown 代码块):',
            '{"score": <1-5的整数>, "reason": "<简短理由>"}',
        ].join('\n');
    }
    else if (assertion.type === 'answer_relevancy') {
        system = '你是答题切题度评审员。只返回 JSON。';
        prompt = [
            '请判断"AI 输出"是否直接、切题地回答了"用户问题"。',
            '',
            '## 用户问题',
            sample.prompt,
            '',
            '## AI 输出',
            output,
            '',
            RAG_LENGTH_DEBIAS,
            '',
            '## 评分',
            '5 = 完整切题回答,无冗余无遗漏',
            '4 = 切题但有少量冗余或小遗漏',
            '3 = 部分切题,部分跑题或避而不答',
            '2 = 大部分跑题',
            '1 = 完全跑题或拒答',
            '',
            '请返回 JSON(不要 markdown 代码块):',
            '{"score": <1-5的整数>, "reason": "<简短理由>"}',
        ].join('\n');
    }
    else {
        // context_recall
        const reference = assertion.reference || sample.context || '';
        if (!reference) {
            return { passed: false, message: 'context_recall: 缺少 assertion.reference 或 sample.context', costUSD: 0 };
        }
        system = '你是 context 覆盖率评审员。只返回 JSON。';
        prompt = [
            '请判断"参考 gold"中的关键事实在"AI 输出"中被覆盖的程度。',
            '',
            '## 参考 gold',
            reference,
            '',
            '## AI 输出',
            output,
            '',
            RAG_LENGTH_DEBIAS,
            '',
            '## 评分流程',
            '1. 列出参考中的关键事实(忽略修饰性内容)',
            '2. 检查每条是否在输出中被提及/使用',
            '3. 给出 1-5 分:',
            '   5 = 全部关键事实被覆盖',
            '   4 = 大部分覆盖,缺 1-2 条次要事实',
            '   3 = 一半覆盖',
            '   2 = 仅覆盖少量',
            '   1 = 完全未覆盖',
            '',
            '请返回 JSON(不要 markdown 代码块):',
            '{"score": <1-5的整数>, "reason": "<简短理由>"}',
        ].join('\n');
    }
    const result = await executor({ model: judgeModel, system, prompt });
    const reported = result.costReportedByExecutor === false ? { costReportedByExecutor: false } : {};
    if (!result.ok) {
        return {
            passed: false,
            message: `${assertion.type} judge error: ${result.error}`,
            costUSD: result.costUSD || 0,
            ...reported,
        };
    }
    try {
        const text = result.output.trim();
        const jsonMatch = text.match(/\{[\s\S]*\}/);
        if (!jsonMatch) {
            process.stderr.write(`[omk] ${assertion.type} judge returned non-JSON: ${text.slice(0, 100)}\n`);
            return { passed: false, message: 'judge returned non-JSON', costUSD: result.costUSD || 0, ...reported };
        }
        const parsed = JSON.parse(jsonMatch[0]);
        const score = Number(parsed.score) || 0;
        return {
            passed: score >= threshold,
            message: parsed.reason ? String(parsed.reason) : '',
            costUSD: result.costUSD || 0,
            ...reported,
        };
    }
    catch (parseErr) {
        process.stderr.write(`[omk] ${assertion.type} judge parse error: ${getErrorMessage(parseErr)}\n`);
        return { passed: false, message: 'failed to parse judge response', costUSD: result.costUSD || 0, ...reported };
    }
}
//# sourceMappingURL=assertions.js.map