import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';
import { join, dirname } from 'node:path';
import { execFileSync } from 'node:child_process';
import { homedir } from 'node:os';
import { createHash } from 'node:crypto';
import { fileURLToPath } from 'node:url';
import { buildVariantSummary } from './schema.js';
import { buildVariantConfig, resolveExecutionStrategy } from './execution-strategy.js';
import { getJudgePromptHash } from '../grading/judge.js';
import { bootstrapMeanCI, bootstrapDiffCI } from './bootstrap.js';
import { getExecutorRuntimeFingerprint } from '../executors/runtime-fingerprint.js';
const __dirname = dirname(fileURLToPath(import.meta.url));
function findPackageJson(startDir) {
    let dir = startDir;
    for (let i = 0; i < 5; i++) {
        const candidate = join(dir, 'package.json');
        if (existsSync(candidate))
            return candidate;
        dir = dirname(dir);
    }
    return join(startDir, '..', 'package.json');
}
const PKG = JSON.parse(readFileSync(findPackageJson(__dirname), 'utf-8'));
export const DEFAULT_OUTPUT_DIR = join(homedir(), '.oh-my-knowledge', 'reports');
export function hashString(str) {
    return createHash('sha256').update(str).digest('hex').slice(0, 12);
}
/**
 * Canonical (key-sorted, recursive) JSON serialization. Required for cross-run hash
 * stability — JS object key iteration order is implementation-defined for objects
 * built by spread / Object.assign / yaml.parse, so naive JSON.stringify can produce
 * different bytes for the "same" sample on different runs.
 */
function canonicalStringify(value) {
    if (value === null || typeof value !== 'object')
        return JSON.stringify(value);
    if (Array.isArray(value))
        return '[' + value.map(canonicalStringify).join(',') + ']';
    const entries = Object.keys(value).sort();
    return '{' + entries.map((k) => JSON.stringify(k) + ':' + canonicalStringify(value[k])).join(',') + '}';
}
/**
 * Stable content hash of a sample. Hashes the prompt + assertions + dimensions/rubric
 * (the parts that determine what's being measured). Two samples with the same hash
 * across runs measure the same thing; mismatched hashes mean the sample changed.
 */
export function hashSample(sample) {
    const stableForm = canonicalStringify({
        prompt: sample.prompt,
        rubric: sample.rubric ?? null,
        dimensions: sample.dimensions ?? null,
        assertions: sample.assertions ?? null,
        schema: sample.schema ?? null,
    });
    return hashString(stableForm);
}
export function getCliVersion() {
    return PKG.version;
}
export function getGitInfo() {
    try {
        const commit = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf-8' }).trim();
        const branch = execFileSync('git', ['rev-parse', '--abbrev-ref', 'HEAD'], { encoding: 'utf-8' }).trim();
        const dirty = execFileSync('git', ['status', '--porcelain'], { encoding: 'utf-8' }).trim().length > 0;
        return { commit, commitShort: commit.slice(0, 7), branch, dirty };
    }
    catch {
        return null;
    }
}
function commonRuntime(runtimes) {
    const values = Object.values(runtimes);
    if (values.length === 0)
        return undefined;
    const first = values[0];
    return values.every((runtime) => runtime.fingerprint === first.fingerprint) ? first : undefined;
}
function representativeRuntime(runtimes) {
    return Object.values(runtimes)[0];
}
function buildExecutorRuntimesByVariant({ variants, model, executorName, tasks, artifacts, request, }) {
    const runtimes = {};
    for (const task of tasks) {
        if (runtimes[task.variant])
            continue;
        const executionPlan = resolveExecutionStrategy(task, model, request?.timeoutMs, false);
        runtimes[task.variant] = getExecutorRuntimeFingerprint(executorName, model, {
            skillDir: executionPlan.input.skillDir,
        });
    }
    for (const variant of variants) {
        if (runtimes[variant])
            continue;
        const artifact = artifacts.find((a) => a.name === variant);
        const fallbackSkillDir = artifact?.kind === 'baseline'
            ? null
            : artifact?.locator && artifact.source !== 'git'
                ? dirname(artifact.locator)
                : request?.skillDir;
        runtimes[variant] = getExecutorRuntimeFingerprint(executorName, model, {
            skillDir: fallbackSkillDir,
        });
    }
    return runtimes;
}
export function aggregateReport({ runId, variants, model, judgeModel, noJudge, executorName, samples, tasks, results, totalCostUSD, artifacts, request, run, job, layeredStats, }) {
    const summary = {};
    for (const variant of variants) {
        const entries = Object.values(results).map((result) => result[variant]).filter(Boolean);
        summary[variant] = buildVariantSummary(entries);
    }
    // Bootstrap CI (per-variant mean) when --bootstrap requested. Adds bootstrapCI to
    // each VariantSummary; legacy t-interval (in summary's other fields) is preserved.
    const bootstrapEnabled = request?.bootstrap === true;
    const bootstrapSamples = request?.bootstrapSamples ?? 1000;
    let pairComparisons;
    if (bootstrapEnabled) {
        for (const variant of variants) {
            const entries = Object.values(results).map((r) => r[variant]).filter(Boolean);
            const compositeScores = entries
                .filter((e) => typeof e.compositeScore === 'number' && e.compositeScore > 0)
                .map((e) => e.compositeScore);
            if (compositeScores.length >= 2) {
                summary[variant].bootstrapCI = bootstrapMeanCI(compositeScores, 0.05, bootstrapSamples);
            }
        }
        // Pairwise treatment-vs-control comparisons. Convention: variants[0] is control;
        // each variants[i>0] is a treatment compared against control.
        if (variants.length >= 2) {
            pairComparisons = [];
            const controlName = variants[0];
            const controlEntries = Object.values(results).map((r) => r[controlName]).filter(Boolean);
            const controlScores = controlEntries
                .filter((e) => typeof e.compositeScore === 'number' && e.compositeScore > 0)
                .map((e) => e.compositeScore);
            for (let i = 1; i < variants.length; i++) {
                const treatmentName = variants[i];
                const treatmentEntries = Object.values(results).map((r) => r[treatmentName]).filter(Boolean);
                const treatmentScores = treatmentEntries
                    .filter((e) => typeof e.compositeScore === 'number' && e.compositeScore > 0)
                    .map((e) => e.compositeScore);
                if (controlScores.length >= 2 && treatmentScores.length >= 2) {
                    pairComparisons.push({
                        control: controlName,
                        treatment: treatmentName,
                        diffBootstrapCI: bootstrapDiffCI(controlScores, treatmentScores, 0.05, bootstrapSamples),
                    });
                }
            }
        }
    }
    const artifactHashes = Object.fromEntries(artifacts.map((artifact) => [artifact.name, artifact.content ? hashString(artifact.content) : 'no-skill']));
    const sampleHashes = Object.fromEntries(samples.map((s) => [s.sample_id, hashSample(s)]));
    const judgeRepeat = request?.judgeRepeat && request.judgeRepeat > 1 ? request.judgeRepeat : undefined;
    const runtimeOptions = { skillDir: request?.skillDir };
    const executorRuntimes = buildExecutorRuntimesByVariant({ variants, model, executorName, tasks, artifacts, request });
    const executorRuntime = commonRuntime(executorRuntimes)
        ?? representativeRuntime(executorRuntimes)
        ?? getExecutorRuntimeFingerprint(executorName, model, runtimeOptions);
    // request.judgeModels is the authoritative source (always non-empty in new schema).
    // Fallback synthesizes a 1-entry from positional judgeModel/executorName for any
    // legacy caller not yet migrated to the array. noJudge ⇒ runtime undefined per entry.
    const requestJudges = request?.judgeModels ?? [{ executor: executorName, model: judgeModel }];
    const judgeModelsMeta = requestJudges.map((jc) => ({
        executor: jc.executor,
        model: jc.model,
        ...(noJudge ? {} : { runtime: getExecutorRuntimeFingerprint(jc.executor, jc.model, runtimeOptions) }),
    }));
    // length-debias is on by default; the request only sets it
    // false when the user passed --no-debias-length. The hash differs between
    // v3-cot-length (on) and v2-cot (off) so readers can detect the divergence.
    const lengthDebiasOn = request?.lengthDebias !== false;
    const debiasModeList = [];
    if (lengthDebiasOn)
        debiasModeList.push('length');
    const totalCostReported = Object.values(summary).every((variant) => variant.execCostReported !== false && variant.judgeCostReported !== false);
    return {
        kind: 'evaluation',
        id: runId,
        meta: {
            variants,
            model,
            executor: executorName,
            ...(request?.effort ? { effort: request.effort } : {}),
            sampleCount: samples.length,
            taskCount: tasks.length,
            totalCostUSD: Number(totalCostUSD.toFixed(6)),
            ...(totalCostReported ? {} : { totalCostReported: false }),
            timestamp: new Date().toISOString(),
            cliVersion: getCliVersion(),
            nodeVersion: process.version,
            artifactHashes,
            sampleHashes,
            ...(noJudge ? {} : { judgePromptHash: getJudgePromptHash(lengthDebiasOn) }),
            executorRuntime,
            executorRuntimes,
            judgeModels: judgeModelsMeta,
            ...(noJudge ? { noJudge: true } : {}),
            ...(judgeRepeat ? { judgeRepeat } : {}),
            ...(debiasModeList.length > 0 ? { debiasMode: debiasModeList } : {}),
            ...(bootstrapEnabled ? { evaluationFramework: 'both' } : {}),
            ...(pairComparisons ? { pairComparisons } : {}),
            variantConfigs: artifacts.map((artifact) => buildVariantConfig(artifact)),
            //  — Skill isolation snapshot per variant. variantName → allowedSkills (null
            // 表示该 variant 没声明 = SDK 默认全发现)。跨报告对比 verdict / Δ 时,isolation
            // 不一致(一个报告有 [],另一个没字段)即不可比。renderer 可据此打提示。
            skillIsolation: Object.fromEntries(artifacts.map((a) => [a.name, a.allowedSkills ?? null])),
            request,
            run,
            job,
            gitInfo: getGitInfo(),
            ...(layeredStats ? { layeredStats: true } : {}),
        },
        summary,
        results: Object.entries(results).map(([sample_id, variantData]) => ({
            sample_id,
            variants: variantData,
        })),
        // 用例设计快照,供单测视角渲染。只挑渲染需要的字段,跳过 cwd / allowedTools /
        // expectedTools / dimensions / environment(对单测视图无附加价值)。Sample 字段
        // 全选会让 report 体积接近翻倍,选子集 size 增长 ~10-20%。
        sampleSnapshots: Object.fromEntries(samples.map((s) => [s.sample_id, {
                sample_id: s.sample_id,
                prompt: s.prompt,
                ...(s.rubric ? { rubric: s.rubric } : {}),
                ...(s.context ? { context: s.context } : {}),
                ...(s.assertions && s.assertions.length > 0 ? { assertions: s.assertions } : {}),
                ...(s.mocks && s.mocks.length > 0 ? { mocks: s.mocks } : {}),
                ...(s.capability && s.capability.length > 0 ? { capability: s.capability } : {}),
                ...(s.difficulty ? { difficulty: s.difficulty } : {}),
                ...(s.construct ? { construct: s.construct } : {}),
                ...(s.provenance ? { provenance: s.provenance } : {}),
                ...(s.tripwire ? { tripwire: true } : {}),
            }])),
    };
}
export function applyBlindMode(report, variants, blindSeed) {
    const labels = variants.map((_, i) => String.fromCharCode(65 + i));
    let seed = parseInt(hashString(blindSeed).slice(0, 8), 16) | 0;
    const seededRandom = () => {
        seed |= 0;
        seed = seed + 0x6D2B79F5 | 0;
        let value = Math.imul(seed ^ seed >>> 15, 1 | seed);
        value ^= value + Math.imul(value ^ value >>> 7, 61 | value);
        return ((value ^ value >>> 14) >>> 0) / 4294967296;
    };
    const shuffled = [...variants];
    for (let i = shuffled.length - 1; i > 0; i--) {
        const j = Math.floor(seededRandom() * (i + 1));
        [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
    }
    const blindMap = Object.fromEntries(shuffled.map((variant, i) => [labels[i], variant]));
    const reverseMap = Object.fromEntries(Object.entries(blindMap).map(([label, variant]) => [variant, label]));
    report.meta.blind = true;
    report.meta.blindMap = blindMap;
    report.meta.variants = labels;
    if (report.meta.executorRuntimes) {
        report.meta.executorRuntimes = Object.fromEntries(Object.entries(report.meta.executorRuntimes).map(([variant, runtime]) => [reverseMap[variant] ?? variant, runtime]));
    }
    const newSummary = {};
    for (const [variant, stats] of Object.entries(report.summary)) {
        newSummary[reverseMap[variant]] = stats;
    }
    report.summary = newSummary;
    for (const result of report.results) {
        const newVariants = {};
        for (const [variant, data] of Object.entries(result.variants)) {
            newVariants[reverseMap[variant]] = data;
        }
        result.variants = newVariants;
    }
}
export function persistReport(report, outputDir) {
    if (!outputDir)
        return null;
    if (!existsSync(outputDir))
        mkdirSync(outputDir, { recursive: true });
    const filePath = join(outputDir, `${report.id}.json`);
    writeFileSync(filePath, JSON.stringify(report, null, 2));
    return filePath;
}
export function generateRunId(variants) {
    const d = new Date();
    const pad = (n) => String(n).padStart(2, '0');
    const date = `${d.getFullYear()}${pad(d.getMonth() + 1)}${pad(d.getDate())}`;
    const time = `${pad(d.getHours())}${pad(d.getMinutes())}`;
    const variantPart = variants
        .map((variant) => variant.replaceAll(/[\\/:]/g, '-').replaceAll(/[^a-zA-Z0-9._@-]/g, '_'))
        .join('-vs-');
    return `${variantPart}-${date}-${time}`;
}
//# sourceMappingURL=evaluation-reporting.js.map