"use strict";
// Confidence calibration — tracks model prediction confidence vs actual
// correctness, producing Brier-score-style calibration metrics per model.
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
    var ownKeys = function(o) {
        ownKeys = Object.getOwnPropertyNames || function (o) {
            var ar = [];
            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
            return ar;
        };
        return ownKeys(o);
    };
    return function (mod) {
        if (mod && mod.__esModule) return mod;
        var result = {};
        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
        __setModuleDefault(result, mod);
        return result;
    };
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.recordSample = recordSample;
exports.loadSamples = loadSamples;
exports.computeCalibration = computeCalibration;
exports.generateReport = generateReport;
exports.generateAllReports = generateAllReports;
exports.renderCalibrationBlock = renderCalibrationBlock;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
// ─── Helpers ─────────────────────────────────────────────────────────
function sanitizeModelName(model) {
    return model.replace(/[^a-zA-Z0-9]/g, '-');
}
function calibrationDir(projectDir) {
    return path.join(projectDir, '.harness', 'calibration');
}
function modelFilePath(projectDir, model) {
    return path.join(calibrationDir(projectDir), `${sanitizeModelName(model)}.jsonl`);
}
// ─── Core functions ──────────────────────────────────────────────────
const MAX_SAMPLES_PER_MODEL = 10_000;
/** Append a sample to `<projectDir>/.harness/calibration/<model>.jsonl`. */
async function recordSample(projectDir, sample) {
    const dir = calibrationDir(projectDir);
    await fs.promises.mkdir(dir, { recursive: true });
    const filePath = modelFilePath(projectDir, sample.model);
    // Check current line count to avoid unbounded growth
    try {
        const existing = await fs.promises.readFile(filePath, 'utf-8');
        const lines = existing.split('\n').filter((l) => l.trim().length > 0);
        if (lines.length >= MAX_SAMPLES_PER_MODEL) {
            // Keep the most recent MAX_SAMPLES_PER_MODEL - 1 lines + append new one
            const trimmed = lines.slice(lines.length - (MAX_SAMPLES_PER_MODEL - 1)).join('\n');
            await fs.promises.writeFile(filePath, trimmed + '\n' + JSON.stringify(sample) + '\n', 'utf-8');
            return;
        }
    }
    catch {
        // File doesn't exist yet — that's fine, just append below
    }
    await fs.promises.appendFile(filePath, JSON.stringify(sample) + '\n', 'utf-8');
}
/** Read all samples for a model. Returns [] if file absent. */
async function loadSamples(projectDir, model) {
    const filePath = modelFilePath(projectDir, model);
    try {
        const content = await fs.promises.readFile(filePath, 'utf-8');
        return content
            .split('\n')
            .filter((line) => line.trim().length > 0)
            .map((line) => JSON.parse(line));
    }
    catch {
        return [];
    }
}
/** Pure computation. Default 10 buckets. */
function computeCalibration(samples, model, bucketCount = 10) {
    const N = samples.length;
    // Build empty buckets
    const buckets = [];
    const step = 1.0 / bucketCount;
    for (let i = 0; i < bucketCount; i++) {
        buckets.push({
            rangeStart: parseFloat((i * step).toFixed(10)),
            rangeEnd: parseFloat(((i + 1) * step).toFixed(10)),
            count: 0,
            avgPredicted: 0,
            actualAccuracy: 0,
            gap: 0,
        });
    }
    if (N === 0) {
        return {
            model,
            totalSamples: 0,
            brierScore: 0,
            ece: 0,
            overconfidenceRatio: 0,
            buckets,
            generatedAt: new Date().toISOString(),
        };
    }
    // Accumulate per-bucket sums
    const bucketPredSum = new Array(bucketCount).fill(0);
    const bucketCorrectSum = new Array(bucketCount).fill(0);
    for (const s of samples) {
        const idx = Math.min(Math.floor(s.predictedConfidence * bucketCount), bucketCount - 1);
        buckets[idx].count++;
        bucketPredSum[idx] += s.predictedConfidence;
        bucketCorrectSum[idx] += s.actuallyCorrect ? 1 : 0;
    }
    for (let i = 0; i < bucketCount; i++) {
        const c = buckets[i].count;
        if (c > 0) {
            buckets[i].avgPredicted = bucketPredSum[i] / c;
            buckets[i].actualAccuracy = bucketCorrectSum[i] / c;
            buckets[i].gap = buckets[i].avgPredicted - buckets[i].actualAccuracy;
        }
    }
    // Brier score
    let brierSum = 0;
    for (const s of samples) {
        const actual = s.actuallyCorrect ? 1.0 : 0.0;
        brierSum += (s.predictedConfidence - actual) ** 2;
    }
    const brierScore = brierSum / N;
    // ECE
    let ece = 0;
    for (const b of buckets) {
        if (b.count > 0) {
            ece += (b.count / N) * Math.abs(b.gap);
        }
    }
    // Overconfidence ratio
    const highConfSamples = samples.filter((s) => s.predictedConfidence > 0.5);
    let overconfidenceRatio = 0;
    if (highConfSamples.length > 0) {
        const wrongHighConf = highConfSamples.filter((s) => !s.actuallyCorrect).length;
        overconfidenceRatio = wrongHighConf / highConfSamples.length;
    }
    return {
        model,
        totalSamples: N,
        brierScore,
        ece,
        overconfidenceRatio,
        buckets,
        generatedAt: new Date().toISOString(),
    };
}
/** Load samples for a model and compute its calibration report. */
async function generateReport(projectDir, model) {
    const samples = await loadSamples(projectDir, model);
    return computeCalibration(samples, model);
}
/** Scan calibration dir and generate a report for every model found. */
async function generateAllReports(projectDir) {
    const dir = calibrationDir(projectDir);
    try {
        const files = await fs.promises.readdir(dir);
        const reports = [];
        for (const file of files) {
            if (!file.endsWith('.jsonl'))
                continue;
            const model = file.replace(/\.jsonl$/, '');
            const filePath = path.join(dir, file);
            const content = await fs.promises.readFile(filePath, 'utf-8');
            const samples = content
                .split('\n')
                .filter((l) => l.trim().length > 0)
                .map((l) => JSON.parse(l));
            reports.push(computeCalibration(samples, model));
        }
        return reports;
    }
    catch {
        return [];
    }
}
/** Render a compact Markdown block for system prompt injection. */
function renderCalibrationBlock(report) {
    const lines = [];
    lines.push(`## Calibration: ${report.model}`);
    lines.push('');
    lines.push(`- **Samples:** ${report.totalSamples}`);
    lines.push(`- **Brier Score:** ${report.brierScore.toFixed(4)}`);
    lines.push(`- **ECE:** ${report.ece.toFixed(4)}`);
    lines.push(`- **Overconfidence Ratio:** ${report.overconfidenceRatio.toFixed(4)}`);
    lines.push('');
    lines.push('| Bucket | Count | Avg Predicted | Actual Accuracy | Gap |');
    lines.push('|--------|-------|---------------|-----------------|-----|');
    for (const b of report.buckets) {
        if (b.count === 0)
            continue;
        lines.push(`| ${b.rangeStart.toFixed(1)}–${b.rangeEnd.toFixed(1)} | ${b.count} | ${b.avgPredicted.toFixed(3)} | ${b.actualAccuracy.toFixed(3)} | ${b.gap >= 0 ? '+' : ''}${b.gap.toFixed(3)} |`);
    }
    return lines.join('\n');
}
//# sourceMappingURL=confidenceCalibration.js.map